466 lines
16 KiB
JavaScript
466 lines
16 KiB
JavaScript
import { spawn } from "node:child_process";
|
|
import { createHash } from "node:crypto";
|
|
import {
|
|
copyFile,
|
|
cp,
|
|
lstat,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
realpath,
|
|
rename,
|
|
rm,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import {
|
|
verifyChecksummedEnginePack,
|
|
writeEngineChecksums,
|
|
} from "./checksummed-engine-pack.mjs";
|
|
import { assertNoUnexpectedLocalPaths } from "./release-path-hygiene.mjs";
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
|
|
export const DOTNET_LOCK = Object.freeze({
|
|
sdkVersion: "10.0.302",
|
|
sdkCommit: "35b593bebf",
|
|
runtimeVersion: "10.0.10",
|
|
runtimeCommit: "f7d90799ce",
|
|
engineVersion: ".NET 10.0.10 System.Text.RegularExpressions",
|
|
sourceDateEpoch: 1_785_110_400,
|
|
bridgeAbi: 1,
|
|
legalFiles: Object.freeze({
|
|
"LICENSE-dotnet.txt":
|
|
"cfc21f5e8bd655ae997eec916138b707b1d290b83272c02a95c9f821b8c87310",
|
|
"THIRD-PARTY-NOTICES.txt":
|
|
"2dc8f8c5a39401e928b5784ab564eb8b3ceb99ead3df8f260e0cab7e0bbecc7a",
|
|
}),
|
|
});
|
|
|
|
const SOURCE_FILES = Object.freeze([
|
|
"engines/dotnet/Program.cs",
|
|
"engines/dotnet/README.md",
|
|
"engines/dotnet/RegexTools.DotNet.csproj",
|
|
"engines/dotnet/wwwroot/main.js",
|
|
]);
|
|
|
|
function compareText(left, right) {
|
|
return left < right ? -1 : left > right ? 1 : 0;
|
|
}
|
|
|
|
function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
async function fileRecord(directory, name) {
|
|
const value = await readFile(path.join(directory, name));
|
|
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
|
}
|
|
|
|
async function regularFiles(directory, relative = "") {
|
|
const entries = (
|
|
await readdir(path.join(directory, relative), {
|
|
withFileTypes: true,
|
|
})
|
|
).sort((left, right) => compareText(left.name, right.name));
|
|
const files = [];
|
|
for (const entry of entries) {
|
|
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
|
const candidate = path.join(directory, name);
|
|
const details = await lstat(candidate);
|
|
if (details.isSymbolicLink()) {
|
|
throw new Error(`.NET engine assets may not be symlinks: ${candidate}`);
|
|
}
|
|
if (entry.isDirectory()) {
|
|
files.push(...(await regularFiles(directory, name)));
|
|
} else if (entry.isFile()) {
|
|
files.push(name);
|
|
} else {
|
|
throw new Error(`Unsupported .NET engine asset: ${candidate}`);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
async function normalizeFrameworkPaths(
|
|
framework,
|
|
{ dotnetRoot, projectRoot, workspace },
|
|
) {
|
|
const replacements = [
|
|
[workspace, "dotnet-build"],
|
|
[dotnetRoot, `dotnet-sdk-${DOTNET_LOCK.sdkVersion}`],
|
|
[path.resolve(projectRoot), "regex-tools-source"],
|
|
].sort(
|
|
([left], [right]) => Buffer.byteLength(right) - Buffer.byteLength(left),
|
|
);
|
|
const javascriptFiles = (await regularFiles(framework)).filter((name) =>
|
|
name.endsWith(".js"),
|
|
);
|
|
for (const name of javascriptFiles) {
|
|
const filename = path.join(framework, name);
|
|
let source = await readFile(filename, "utf8");
|
|
for (const [localPath, stableName] of replacements) {
|
|
source = source
|
|
.replaceAll(localPath, stableName)
|
|
.replaceAll(localPath.replaceAll("\\", "\\\\"), stableName);
|
|
}
|
|
await writeFile(filename, source, "utf8");
|
|
}
|
|
|
|
const nativeModules = javascriptFiles.filter((name) =>
|
|
/^dotnet\.native\.[a-z0-9]+\.js$/u.test(name),
|
|
);
|
|
if (nativeModules.length !== 1) {
|
|
throw new Error(
|
|
"The .NET publish must produce exactly one fingerprinted native JavaScript module.",
|
|
);
|
|
}
|
|
const originalName = nativeModules[0];
|
|
const nativeSource = await readFile(path.join(framework, originalName));
|
|
const stableName = `dotnet.native.${sha256(nativeSource).slice(0, 12)}.js`;
|
|
if (stableName === originalName) return;
|
|
|
|
for (const name of javascriptFiles) {
|
|
if (name === originalName) continue;
|
|
const filename = path.join(framework, name);
|
|
const source = await readFile(filename, "utf8");
|
|
if (source.includes(originalName)) {
|
|
await writeFile(
|
|
filename,
|
|
source.replaceAll(originalName, stableName),
|
|
"utf8",
|
|
);
|
|
}
|
|
}
|
|
await rename(
|
|
path.join(framework, originalName),
|
|
path.join(framework, stableName),
|
|
);
|
|
}
|
|
|
|
async function run(command, arguments_, options = {}) {
|
|
const child = spawn(command, arguments_, {
|
|
cwd: options.cwd,
|
|
env: options.env,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
const stdout = [];
|
|
const stderr = [];
|
|
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
|
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
|
const result = await new Promise((resolve, reject) => {
|
|
child.once("error", reject);
|
|
child.once("close", (code, signal) => resolve({ code, signal }));
|
|
});
|
|
const output = Buffer.concat(stdout).toString("utf8");
|
|
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
|
if (result.code !== 0) {
|
|
throw new Error(
|
|
`${command} failed with ${
|
|
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
|
}:\n${[output, errorOutput].filter(Boolean).join("\n").trim()}`,
|
|
);
|
|
}
|
|
return { stdout: output, stderr: errorOutput };
|
|
}
|
|
|
|
async function exactDotNet(candidate) {
|
|
const details = await lstat(candidate).catch(() => null);
|
|
if (!details?.isFile() || details.isSymbolicLink()) {
|
|
throw new Error(`dotnet must be a regular file: ${candidate}`);
|
|
}
|
|
const executable = await realpath(candidate);
|
|
const version = (await run(executable, ["--version"])).stdout.trim();
|
|
const information = (await run(executable, ["--info"])).stdout;
|
|
if (
|
|
version !== DOTNET_LOCK.sdkVersion ||
|
|
!information.includes(`Commit: ${DOTNET_LOCK.sdkCommit}`) ||
|
|
!information.includes(`Version: ${DOTNET_LOCK.runtimeVersion}`) ||
|
|
!information.includes(`Commit: ${DOTNET_LOCK.runtimeCommit}`) ||
|
|
!information.includes("[wasm-tools]")
|
|
) {
|
|
throw new Error(
|
|
`The .NET pack requires SDK ${DOTNET_LOCK.sdkVersion}, runtime ${DOTNET_LOCK.runtimeVersion}, and its wasm-tools workload.`,
|
|
);
|
|
}
|
|
const sdkRoot = path.dirname(executable);
|
|
const license = path.join(sdkRoot, "LICENSE.txt");
|
|
const notices = path.join(sdkRoot, "ThirdPartyNotices.txt");
|
|
for (const file of [license, notices]) {
|
|
const legalDetails = await lstat(file).catch(() => null);
|
|
if (!legalDetails?.isFile() || legalDetails.isSymbolicLink()) {
|
|
throw new Error(`Required .NET legal file is missing: ${file}`);
|
|
}
|
|
}
|
|
for (const [name, file] of [
|
|
["LICENSE-dotnet.txt", license],
|
|
["THIRD-PARTY-NOTICES.txt", notices],
|
|
]) {
|
|
if (sha256(await readFile(file)) !== DOTNET_LOCK.legalFiles[name]) {
|
|
throw new Error(`The pinned .NET SDK legal file drifted: ${name}.`);
|
|
}
|
|
}
|
|
return { executable, license, notices };
|
|
}
|
|
|
|
function buildEnvironment(dotnetRoot) {
|
|
const environment = { ...process.env };
|
|
for (const variable of [
|
|
"CFLAGS",
|
|
"CPPFLAGS",
|
|
"CXXFLAGS",
|
|
"DOTNET_ROOT",
|
|
"MSBuildSDKsPath",
|
|
]) {
|
|
delete environment[variable];
|
|
}
|
|
environment.DOTNET_ROOT = dotnetRoot;
|
|
environment.DOTNET_CLI_TELEMETRY_OPTOUT = "1";
|
|
environment.DOTNET_NOLOGO = "1";
|
|
environment.LANG = "C";
|
|
environment.LC_ALL = "C";
|
|
environment.SOURCE_DATE_EPOCH = String(DOTNET_LOCK.sourceDateEpoch);
|
|
environment.TZ = "UTC";
|
|
return environment;
|
|
}
|
|
|
|
async function sourceRecords(projectRoot) {
|
|
return Promise.all(
|
|
SOURCE_FILES.map(async (name) => {
|
|
const value = await readFile(path.join(projectRoot, name));
|
|
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function writeMetadata(pack, projectRoot) {
|
|
const assetNames = (await regularFiles(pack))
|
|
.filter((name) => name !== "SHA256SUMS" && name !== "engine-metadata.json")
|
|
.sort(compareText);
|
|
const metadata = {
|
|
schemaVersion: 1,
|
|
engine: "dotnet",
|
|
engineName: "System.Text.RegularExpressions",
|
|
engineVersion: DOTNET_LOCK.engineVersion,
|
|
semanticIdentity:
|
|
".NET 10.0.10 System.Text.RegularExpressions; invariant culture; UTF-16 offsets",
|
|
status: "production-native-runtime",
|
|
bridge: {
|
|
abiVersion: DOTNET_LOCK.bridgeAbi,
|
|
nativeOffsetUnit: "UTF-16 code units",
|
|
requestEncoding: "bounded JSON passed to fixed JSExport methods",
|
|
userSourceEvaluation: false,
|
|
replacement:
|
|
"Fixed streaming parser for .NET replacement tokens ($$, $n, ${name}, $&, $`, $', $+, $_), verified against Match.Result at load",
|
|
replacementOutputTransport:
|
|
"bounded UTF-8 bytes encoded as canonical base64 in the JSON response",
|
|
supportedApplicationFlags: ["g", "i", "m", "s", "n", "x", "r", "c", "b"],
|
|
matchTimeoutMilliseconds: 9_000,
|
|
limits: {
|
|
maximumPatternUtf16: 65_536,
|
|
maximumSubjectUtf16: 16_777_216,
|
|
maximumReplacementUtf16: 65_536,
|
|
maximumMatches: 10_000,
|
|
maximumCaptureRows: 100_000,
|
|
maximumCaptureGroups: 1_000,
|
|
maximumOutputBytes: 67_108_864,
|
|
},
|
|
sourceFiles: await sourceRecords(projectRoot),
|
|
},
|
|
source: {
|
|
project: ".NET",
|
|
repository: "https://github.com/dotnet/runtime",
|
|
license: "MIT",
|
|
runtimeVersion: DOTNET_LOCK.runtimeVersion,
|
|
runtimeCommit: DOTNET_LOCK.runtimeCommit,
|
|
},
|
|
toolchain: {
|
|
sdkVersion: DOTNET_LOCK.sdkVersion,
|
|
sdkCommit: DOTNET_LOCK.sdkCommit,
|
|
workload: "wasm-tools",
|
|
target: "browser-wasm",
|
|
command:
|
|
"dotnet publish engines/dotnet/RegexTools.DotNet.csproj -c Release --no-restore -p:PathMap=<source>=/regex-tools-source -p:EmccExtraCFlags=-ffile-prefix-map=<sdk>=/dotnet-sdk-10.0.302",
|
|
},
|
|
sourceDateEpoch: DOTNET_LOCK.sourceDateEpoch,
|
|
files: await Promise.all(assetNames.map((name) => fileRecord(pack, name))),
|
|
};
|
|
await writeFile(
|
|
path.join(pack, "engine-metadata.json"),
|
|
`${JSON.stringify(metadata, null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
await writeEngineChecksums(pack);
|
|
return metadata;
|
|
}
|
|
|
|
export async function verifyDotNetPack(pack, projectRoot = root) {
|
|
const metadata = await verifyChecksummedEnginePack(pack, "dotnet");
|
|
const files = (await regularFiles(pack)).sort(compareText);
|
|
const framework = files.filter((name) => name.startsWith("_framework/"));
|
|
const requiredPatterns = [
|
|
/^_framework\/RegexTools\.DotNet\.[a-z0-9]+\.wasm$/u,
|
|
/^_framework\/System\.Text\.RegularExpressions\.[a-z0-9]+\.wasm$/u,
|
|
/^_framework\/dotnet\.native\.[a-z0-9]+\.wasm$/u,
|
|
/^_framework\/dotnet\.native\.[a-z0-9]+\.js$/u,
|
|
/^_framework\/dotnet\.runtime\.[a-z0-9]+\.js$/u,
|
|
];
|
|
if (
|
|
metadata.engineVersion !== DOTNET_LOCK.engineVersion ||
|
|
metadata.source?.runtimeVersion !== DOTNET_LOCK.runtimeVersion ||
|
|
metadata.source?.runtimeCommit !== DOTNET_LOCK.runtimeCommit ||
|
|
metadata.toolchain?.sdkVersion !== DOTNET_LOCK.sdkVersion ||
|
|
metadata.toolchain?.sdkCommit !== DOTNET_LOCK.sdkCommit ||
|
|
metadata.bridge?.abiVersion !== DOTNET_LOCK.bridgeAbi ||
|
|
metadata.bridge?.userSourceEvaluation !== false ||
|
|
!String(metadata.bridge?.replacement).includes("Match.Result") ||
|
|
!String(metadata.bridge?.replacementOutputTransport).includes("base64") ||
|
|
JSON.stringify(metadata.bridge?.sourceFiles) !==
|
|
JSON.stringify(await sourceRecords(projectRoot)) ||
|
|
Object.entries(DOTNET_LOCK.legalFiles).some(
|
|
([name, digest]) =>
|
|
metadata.files?.find((file) => file.path === name)?.sha256 !== digest,
|
|
) ||
|
|
!files.includes("_framework/dotnet.js") ||
|
|
!files.includes("LICENSE-dotnet.txt") ||
|
|
!files.includes("THIRD-PARTY-NOTICES.txt") ||
|
|
files.some((name) => /\.(?:br|gz|map|pdb)$/u.test(name)) ||
|
|
requiredPatterns.some(
|
|
(pattern) => framework.filter((name) => pattern.test(name)).length !== 1,
|
|
)
|
|
) {
|
|
throw new Error(
|
|
".NET engine pack drifted from its pinned runtime contract.",
|
|
);
|
|
}
|
|
const mainModule = await readFile(
|
|
path.join(pack, "_framework/dotnet.js"),
|
|
"utf8",
|
|
);
|
|
if (!mainModule.includes("dotnet") || !mainModule.includes("./")) {
|
|
throw new Error(".NET engine entry module has an unexpected shape.");
|
|
}
|
|
for (const name of framework) {
|
|
assertNoUnexpectedLocalPaths(
|
|
await readFile(path.join(pack, name)),
|
|
`.NET engine asset ${name}`,
|
|
{
|
|
allowedVirtualHomes:
|
|
/^_framework\/dotnet\.native\.[a-z0-9]+\.js$/u.test(name)
|
|
? ["/home/web_user"]
|
|
: [],
|
|
},
|
|
);
|
|
}
|
|
return metadata;
|
|
}
|
|
|
|
export async function buildDotNetPack(
|
|
dotnet,
|
|
output = path.join(root, ".engine-build", "dotnet"),
|
|
projectRoot = root,
|
|
) {
|
|
const toolchain = await exactDotNet(dotnet);
|
|
const outputParent = path.dirname(output);
|
|
await mkdir(outputParent, { recursive: true });
|
|
const workspace = await mkdtemp(path.join(outputParent, ".dotnet-build-"));
|
|
const staging = path.join(workspace, "pack");
|
|
const publish = path.join(workspace, "publish");
|
|
await mkdir(staging);
|
|
try {
|
|
const dotnetRoot = path.dirname(toolchain.executable);
|
|
const projectPath = path.join(
|
|
projectRoot,
|
|
"engines",
|
|
"dotnet",
|
|
"RegexTools.DotNet.csproj",
|
|
);
|
|
await run(
|
|
toolchain.executable,
|
|
[
|
|
"publish",
|
|
projectPath,
|
|
"-c",
|
|
"Release",
|
|
"--no-restore",
|
|
"--output",
|
|
publish,
|
|
`-p:PathMap=${path.resolve(projectRoot)}=/regex-tools-source`,
|
|
`-p:EmccExtraCFlags=-ffile-prefix-map=${dotnetRoot}=/dotnet-sdk-${DOTNET_LOCK.sdkVersion}`,
|
|
],
|
|
{
|
|
cwd: projectRoot,
|
|
env: buildEnvironment(dotnetRoot),
|
|
},
|
|
);
|
|
const publishedFramework = path.join(publish, "wwwroot", "_framework");
|
|
const frameworkDetails = await lstat(publishedFramework).catch(() => null);
|
|
if (!frameworkDetails?.isDirectory() || frameworkDetails.isSymbolicLink()) {
|
|
throw new Error(
|
|
"The .NET publish did not produce a WebAssembly framework.",
|
|
);
|
|
}
|
|
await cp(publishedFramework, path.join(staging, "_framework"), {
|
|
recursive: true,
|
|
filter: (source) => !/\.(?:br|gz)$/u.test(source),
|
|
});
|
|
for (const name of await regularFiles(path.join(staging, "_framework"))) {
|
|
if (/\.(?:map|pdb)$/u.test(name)) {
|
|
await rm(path.join(staging, "_framework", name), { force: true });
|
|
}
|
|
}
|
|
await normalizeFrameworkPaths(path.join(staging, "_framework"), {
|
|
dotnetRoot: path.dirname(toolchain.executable),
|
|
projectRoot,
|
|
workspace,
|
|
});
|
|
await copyFile(toolchain.license, path.join(staging, "LICENSE-dotnet.txt"));
|
|
await copyFile(
|
|
toolchain.notices,
|
|
path.join(staging, "THIRD-PARTY-NOTICES.txt"),
|
|
);
|
|
await writeMetadata(staging, projectRoot);
|
|
await verifyDotNetPack(staging, projectRoot);
|
|
await rm(output, { recursive: true, force: true });
|
|
await rename(staging, output);
|
|
await rm(workspace, { recursive: true, force: true });
|
|
return { output, metadata: await verifyDotNetPack(output, projectRoot) };
|
|
} catch (error) {
|
|
await rm(workspace, { recursive: true, force: true });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function installDotNetPack(source, projectRoot = root) {
|
|
await verifyDotNetPack(source, projectRoot);
|
|
const destination = path.join(projectRoot, "public", "engines", "dotnet");
|
|
const parent = path.dirname(destination);
|
|
const staging = await mkdtemp(path.join(parent, ".dotnet-install-"));
|
|
try {
|
|
await cp(source, staging, { recursive: true });
|
|
await verifyDotNetPack(staging, projectRoot);
|
|
await rm(destination, { recursive: true, force: true });
|
|
await rename(staging, destination);
|
|
} catch (error) {
|
|
await rm(staging, { recursive: true, force: true });
|
|
throw error;
|
|
}
|
|
return {
|
|
output: destination,
|
|
metadata: await verifyDotNetPack(destination, projectRoot),
|
|
};
|
|
}
|
|
|
|
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
const arguments_ = process.argv.slice(2);
|
|
if (arguments_.length !== 2 || arguments_[0] !== "--dotnet") {
|
|
throw new Error(
|
|
"Usage: node scripts/dotnet-engine-pack.mjs --dotnet /absolute/path/to/dotnet",
|
|
);
|
|
}
|
|
const result = await buildDotNetPack(arguments_[1]);
|
|
console.log(`Built verified .NET engine pack at ${result.output}.`);
|
|
}
|