feat: add multi-engine regex flavour support
This commit is contained in:
@@ -37,20 +37,34 @@ const packageJson = JSON.parse(
|
||||
);
|
||||
const engineDirectory = path.join(root, "public", "engines");
|
||||
const entries = (await readdir(engineDirectory)).sort();
|
||||
const expectedEntries = [
|
||||
"README.md",
|
||||
"cpp",
|
||||
"dotnet",
|
||||
"go",
|
||||
"java",
|
||||
"pcre2",
|
||||
"perl",
|
||||
"php",
|
||||
"python",
|
||||
"ruby",
|
||||
"rust",
|
||||
];
|
||||
|
||||
if (
|
||||
typeof packageJson.version !== "string" ||
|
||||
entries.length !== 4 ||
|
||||
entries[0] !== "README.md" ||
|
||||
entries[1] !== "java" ||
|
||||
entries[2] !== "pcre2" ||
|
||||
entries[3] !== "python"
|
||||
entries.length !== expectedEntries.length ||
|
||||
entries.some((entry, index) => entry !== expectedEntries[index])
|
||||
) {
|
||||
throw new Error(
|
||||
"The production build requires the documented Java, PCRE2 and Python runtime packs.",
|
||||
"The production build requires every documented checked-in runtime pack.",
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
"The checked-in Java, PCRE2 and Python runtime packs are present; no release-time download or engine compilation is needed.",
|
||||
`The checked-in ${expectedEntries
|
||||
.filter((entry) => entry !== "README.md")
|
||||
.join(
|
||||
", ",
|
||||
)} runtime packs are present; no release-time download or engine compilation is needed.`,
|
||||
);
|
||||
|
||||
15
scripts/build-go-engine.mjs
Normal file
15
scripts/build-go-engine.mjs
Normal file
@@ -0,0 +1,15 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { buildGoPack } from "./go-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length < 1 || arguments_.length > 2) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/build-go-engine.mjs GO_ROOT [.engine-build/go]",
|
||||
);
|
||||
}
|
||||
const result = await buildGoPack(root, arguments_[0], arguments_[1]);
|
||||
console.log(
|
||||
`Built and verified ${result.metadata.semanticIdentity} at ${result.output}.`,
|
||||
);
|
||||
15
scripts/build-rust-engine.mjs
Normal file
15
scripts/build-rust-engine.mjs
Normal file
@@ -0,0 +1,15 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { buildRustPack } from "./rust-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length < 1 || arguments_.length > 2) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/build-rust-engine.mjs RUST_TOOLCHAIN_ROOT [.engine-build/rust]",
|
||||
);
|
||||
}
|
||||
const result = await buildRustPack(root, arguments_[0], arguments_[1]);
|
||||
console.log(
|
||||
`Built and verified ${result.metadata.semanticIdentity} at ${result.output}.`,
|
||||
);
|
||||
163
scripts/checksummed-engine-pack.mjs
Normal file
163
scripts/checksummed-engine-pack.mjs
Normal file
@@ -0,0 +1,163 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
lstat,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
function compareText(left, right) {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function regularFiles(directory, relative = "") {
|
||||
const current = path.join(directory, relative);
|
||||
const entries = (await readdir(current, { withFileTypes: true })).sort(
|
||||
(left, right) => compareText(left.name, right.name),
|
||||
);
|
||||
const result = [];
|
||||
for (const entry of entries) {
|
||||
const entryRelative = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
const candidate = path.join(directory, entryRelative);
|
||||
const details = await lstat(candidate);
|
||||
if (details.isSymbolicLink()) {
|
||||
throw new Error(`Engine packs may not contain symlinks: ${candidate}`);
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
result.push(...(await regularFiles(directory, entryRelative)));
|
||||
} else if (entry.isFile()) {
|
||||
result.push(entryRelative);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Engine packs may contain only directories and regular files: ${candidate}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function writeEngineChecksums(directory) {
|
||||
const pack = await realpath(directory);
|
||||
const details = await lstat(pack);
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`Engine pack must be a real directory: ${directory}`);
|
||||
}
|
||||
const files = (await regularFiles(pack))
|
||||
.filter((name) => name !== "SHA256SUMS")
|
||||
.sort(compareText);
|
||||
const records = await Promise.all(
|
||||
files.map(async (name) => ({
|
||||
name,
|
||||
digest: sha256(await readFile(path.join(pack, name))),
|
||||
})),
|
||||
);
|
||||
const text = `${records
|
||||
.map(({ digest, name }) => `${digest} ${name}`)
|
||||
.join("\n")}\n`;
|
||||
await writeFile(path.join(pack, "SHA256SUMS"), text, {
|
||||
encoding: "utf8",
|
||||
mode: 0o644,
|
||||
});
|
||||
return records;
|
||||
}
|
||||
|
||||
export async function verifyChecksummedEnginePack(directory, expectedEngine) {
|
||||
const packDetails = await lstat(directory).catch(() => null);
|
||||
if (!packDetails?.isDirectory() || packDetails.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine pack must be a real directory: ${directory}`,
|
||||
);
|
||||
}
|
||||
const pack = await realpath(directory);
|
||||
const files = (await regularFiles(pack)).sort(compareText);
|
||||
if (
|
||||
!files.includes("SHA256SUMS") ||
|
||||
!files.includes("engine-metadata.json")
|
||||
) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine pack requires SHA256SUMS and engine-metadata.json.`,
|
||||
);
|
||||
}
|
||||
const checksumText = await readFile(path.join(pack, "SHA256SUMS"), "utf8");
|
||||
const lines = checksumText.endsWith("\n")
|
||||
? checksumText.slice(0, -1).split("\n")
|
||||
: [];
|
||||
const records = lines.map((line) => {
|
||||
const match = /^([0-9a-f]{64}) ([A-Za-z0-9._+/@-]+)$/u.exec(line);
|
||||
if (
|
||||
!match ||
|
||||
match[2] === "SHA256SUMS" ||
|
||||
match[2].startsWith("/") ||
|
||||
match[2].split("/").includes("..")
|
||||
) {
|
||||
throw new Error(
|
||||
`${expectedEngine} SHA256SUMS contains a malformed entry.`,
|
||||
);
|
||||
}
|
||||
return { digest: match[1], name: match[2] };
|
||||
});
|
||||
const names = records.map(({ name }) => name);
|
||||
if (
|
||||
new Set(names).size !== names.length ||
|
||||
[...names].sort(compareText).some((name, index) => name !== names[index])
|
||||
) {
|
||||
throw new Error(`${expectedEngine} SHA256SUMS must be unique and sorted.`);
|
||||
}
|
||||
const expectedFiles = files
|
||||
.filter((name) => name !== "SHA256SUMS")
|
||||
.sort(compareText);
|
||||
if (
|
||||
expectedFiles.length !== names.length ||
|
||||
expectedFiles.some((name, index) => name !== names[index])
|
||||
) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine pack closed file set does not match SHA256SUMS.`,
|
||||
);
|
||||
}
|
||||
for (const record of records) {
|
||||
const actual = sha256(await readFile(path.join(pack, record.name)));
|
||||
if (actual !== record.digest) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine asset ${record.name} SHA-256 mismatch.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
if (
|
||||
metadata.schemaVersion !== 1 ||
|
||||
metadata.engine !== expectedEngine ||
|
||||
!Array.isArray(metadata.files)
|
||||
) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine metadata has an unexpected identity.`,
|
||||
);
|
||||
}
|
||||
for (const record of metadata.files) {
|
||||
if (
|
||||
!record ||
|
||||
typeof record.path !== "string" ||
|
||||
typeof record.sha256 !== "string" ||
|
||||
!Number.isSafeInteger(record.bytes) ||
|
||||
!names.includes(record.path)
|
||||
) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine metadata contains an invalid file record.`,
|
||||
);
|
||||
}
|
||||
const value = await readFile(path.join(pack, record.path));
|
||||
if (value.byteLength !== record.bytes || sha256(value) !== record.sha256) {
|
||||
throw new Error(
|
||||
`${expectedEngine} engine metadata does not match ${record.path}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
645
scripts/cpp-engine-pack.mjs
Normal file
645
scripts/cpp-engine-pack.mjs
Normal file
@@ -0,0 +1,645 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
verifyChecksummedEnginePack,
|
||||
writeEngineChecksums,
|
||||
} from "./checksummed-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
export const CPP_LOCK = Object.freeze({
|
||||
engineVersion: "Emscripten 6.0.4 libc++ std::wregex",
|
||||
emscriptenVersion: "6.0.4",
|
||||
emscriptenCommit: "fe5be6afdff43ad58860d821fcc8572a23f92d19",
|
||||
emsdkCommit: "224ec5f9f2f72f09f9ce0e26d66bae7dbd8b692f",
|
||||
releaseCommit: "b23272fac5d05617cd36dc451356dca0f79bf22d",
|
||||
llvmVersion: "24.0.0git",
|
||||
llvmCommit: "c050c487e9edb97ef44f53cb29fe1d8bcddb8f76",
|
||||
sourceDateEpoch: 1_785_110_400,
|
||||
bridgeAbi: 1,
|
||||
legalFiles: Object.freeze({
|
||||
"LICENSE-Emscripten.txt":
|
||||
"620a78084fc7ca97c0b5dea9abf891f3ffcadfdbf305276f099c9c4e12fc1d86",
|
||||
"LICENSE-compiler-rt.txt":
|
||||
"1a8f1058753f1ba890de984e48f0242a3a5c29a6a8f2ed9fd813f36985387e8d",
|
||||
"LICENSE-libcxx.txt":
|
||||
"539dd7aed86e8a4f12cbdd0e6c50c189c7d74847e4fecc64ce2c6ee3a01da38b",
|
||||
"LICENSE-libcxxabi.txt":
|
||||
"e2b35be49f7284a45b7baca8fc7b3ab7440e7902392b2528a457816b5bb2a15c",
|
||||
"LICENSE-libunwind.txt":
|
||||
"b5efebcaca80879234098e52d1725e6d9eb8fb96a19fce625d39184b705f7b6d",
|
||||
"LICENSE-musl.txt":
|
||||
"b870108ec5e7790e9f9919064f1b9421d62d5f9b0e6c230c6adf7ea2da62e97b",
|
||||
}),
|
||||
sourceEvidence: Object.freeze({
|
||||
"emscripten-revision.txt":
|
||||
"33d347ba6aa1de448bc1727d46912c1abcbddac9a32dcf4794d7098dea16e1ad",
|
||||
"emscripten-version.txt":
|
||||
"1ae2390b7da62b8488a9f596a39ed978b22356de0f267db9ae9dd43d0d3faa6e",
|
||||
"system/lib/compiler-rt/README.md":
|
||||
"50648d4da5c5c88468da064bd9ba29d68d47597607852330423da0a1cf987946",
|
||||
"system/lib/libc/README.md":
|
||||
"69aeb7c1991af58c547f8da100c57ff38f53c4015f4df579731c5abab84654b9",
|
||||
"system/lib/libcxx/README.md":
|
||||
"da90136512d04c06e2bd27ef18d71be34bfc64c06bc29b12ddcf66bb76ae193b",
|
||||
"system/lib/libcxxabi/README.md":
|
||||
"dcfec7b2c2987ce00e513cac89933f4d30329070570c4efdb4d1d00297182736",
|
||||
"system/lib/libunwind/README.md":
|
||||
"26b3b06b2640cee75b2cf4e01b0956a0fd6cc5ca2054998e18d66ff060762410",
|
||||
"system/lib/emmalloc.c":
|
||||
"00eb577cbef51a3359c309f45a7c99a814b8c6151ab75c9a3c78a56f0169146b",
|
||||
}),
|
||||
linkedArchives: Object.freeze([
|
||||
"libc++.a",
|
||||
"libc++abi.a",
|
||||
"libc.a",
|
||||
"libclang_rt.builtins.a",
|
||||
"libembind-rtti.a",
|
||||
"libemmalloc.a",
|
||||
"libnoexit.a",
|
||||
"libstubs.a",
|
||||
]),
|
||||
});
|
||||
|
||||
const SOURCE_FILES = Object.freeze([
|
||||
"engines/cpp/README.md",
|
||||
"engines/cpp/cpp_regex_bridge.cpp",
|
||||
"scripts/cpp-engine-pack.mjs",
|
||||
]);
|
||||
|
||||
const PACK_FILES = Object.freeze(
|
||||
[
|
||||
...Object.keys(CPP_LOCK.legalFiles),
|
||||
"SHA256SUMS",
|
||||
"SOURCE-MANIFEST.json",
|
||||
"cpp-regex.mjs",
|
||||
"cpp-regex.wasm",
|
||||
"engine-metadata.json",
|
||||
].sort(),
|
||||
);
|
||||
|
||||
const SOURCE_COMPONENTS = Object.freeze([
|
||||
Object.freeze({
|
||||
id: "emscripten-runtime",
|
||||
version: CPP_LOCK.emscriptenVersion,
|
||||
linked: true,
|
||||
linkedArchives: ["libembind-rtti.a", "libnoexit.a", "libstubs.a"],
|
||||
sourcePath: "src, system/lib and tools",
|
||||
sourceEvidencePath: "emscripten-revision.txt",
|
||||
license: "MIT OR NCSA",
|
||||
legalFiles: ["LICENSE-Emscripten.txt"],
|
||||
evidence:
|
||||
"Emscripten-generated JavaScript/runtime imports plus exact linker-trace archive members",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "musl",
|
||||
version: "1.2.6 with Emscripten changes",
|
||||
linked: true,
|
||||
linkedArchives: ["libc.a"],
|
||||
sourcePath: "system/lib/libc",
|
||||
sourceEvidencePath: "system/lib/libc/README.md",
|
||||
license: "MIT and file-level compatible terms",
|
||||
legalFiles: ["LICENSE-musl.txt"],
|
||||
evidence: "exact linker trace selects members of libc.a",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "emmalloc",
|
||||
version: `Emscripten ${CPP_LOCK.emscriptenVersion} snapshot`,
|
||||
linked: true,
|
||||
linkedArchives: ["libemmalloc.a"],
|
||||
sourcePath: "system/lib/emmalloc.c",
|
||||
sourceEvidencePath: "system/lib/emmalloc.c",
|
||||
license: "MIT OR NCSA",
|
||||
legalFiles: ["LICENSE-Emscripten.txt"],
|
||||
evidence:
|
||||
"the build fixes -sMALLOC=emmalloc and the exact linker trace selects emmalloc.o and sbrk.o",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "compiler-rt",
|
||||
version: "22.1.8 with Emscripten changes",
|
||||
linked: true,
|
||||
linkedArchives: ["libclang_rt.builtins.a"],
|
||||
sourcePath: "system/lib/compiler-rt",
|
||||
sourceEvidencePath: "system/lib/compiler-rt/README.md",
|
||||
license: "Apache-2.0 WITH LLVM-exception and legacy component terms",
|
||||
legalFiles: ["LICENSE-compiler-rt.txt"],
|
||||
evidence:
|
||||
"exact linker trace selects compiler-rt builtins, including Emscripten exception support",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "libc++",
|
||||
version: "21.1.8 with Emscripten changes",
|
||||
linked: true,
|
||||
linkedArchives: ["libc++.a"],
|
||||
sourcePath: "system/lib/libcxx",
|
||||
sourceEvidencePath: "system/lib/libcxx/README.md",
|
||||
license: "Apache-2.0 WITH LLVM-exception and legacy component terms",
|
||||
legalFiles: ["LICENSE-libcxx.txt"],
|
||||
evidence:
|
||||
"exact linker trace selects libc++ regex, locale, string, exception and support objects",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "libc++abi",
|
||||
version: "21.1.8 with Emscripten changes",
|
||||
linked: true,
|
||||
linkedArchives: ["libc++abi.a"],
|
||||
sourcePath: "system/lib/libcxxabi",
|
||||
sourceEvidencePath: "system/lib/libcxxabi/README.md",
|
||||
license: "Apache-2.0 WITH LLVM-exception and legacy component terms",
|
||||
legalFiles: ["LICENSE-libcxxabi.txt"],
|
||||
evidence:
|
||||
"exact linker trace selects the Emscripten C++ exception ABI and type-information objects",
|
||||
}),
|
||||
Object.freeze({
|
||||
id: "libunwind",
|
||||
version: "22.1.8 with Emscripten changes",
|
||||
linked: false,
|
||||
linkedArchives: [],
|
||||
consideredArchive: "libunwind.a",
|
||||
sourcePath: "system/lib/libunwind",
|
||||
sourceEvidencePath: "system/lib/libunwind/README.md",
|
||||
license: "Apache-2.0 WITH LLVM-exception and legacy component terms",
|
||||
legalFiles: ["LICENSE-libunwind.txt"],
|
||||
evidence:
|
||||
"audited exclusion: the exact JavaScript-exception linker trace selects no libunwind.a member; its legal text is retained to make that boundary reviewable",
|
||||
}),
|
||||
]);
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function fileRecord(directory, name) {
|
||||
const value = await readFile(path.join(directory, name));
|
||||
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
throw new Error(
|
||||
`${command} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}:\n${[output, errorOutput].filter(Boolean).join("\n").trim()}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
async function exactEmpp(candidate) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`em++ must be a regular file: ${candidate}`);
|
||||
}
|
||||
const executable = await realpath(candidate);
|
||||
const version = await run(executable, ["--version"]);
|
||||
if (
|
||||
!version.stdout.startsWith(
|
||||
`emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) ${CPP_LOCK.emscriptenVersion} (${CPP_LOCK.emscriptenCommit})`,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`The C++ pack requires Emscripten ${CPP_LOCK.emscriptenVersion} at commit ${CPP_LOCK.emscriptenCommit}.`,
|
||||
);
|
||||
}
|
||||
const upstreamRoot = path.dirname(path.dirname(executable));
|
||||
const sourceRoot = path.join(upstreamRoot, "emscripten");
|
||||
const legalFiles = {
|
||||
"LICENSE-Emscripten.txt": path.join(sourceRoot, "LICENSE"),
|
||||
"LICENSE-compiler-rt.txt": path.join(
|
||||
sourceRoot,
|
||||
"system",
|
||||
"lib",
|
||||
"compiler-rt",
|
||||
"LICENSE.TXT",
|
||||
),
|
||||
"LICENSE-libcxx.txt": path.join(
|
||||
sourceRoot,
|
||||
"system",
|
||||
"lib",
|
||||
"libcxx",
|
||||
"LICENSE.TXT",
|
||||
),
|
||||
"LICENSE-libcxxabi.txt": path.join(
|
||||
sourceRoot,
|
||||
"system",
|
||||
"lib",
|
||||
"libcxxabi",
|
||||
"LICENSE.TXT",
|
||||
),
|
||||
"LICENSE-libunwind.txt": path.join(
|
||||
sourceRoot,
|
||||
"system",
|
||||
"lib",
|
||||
"libunwind",
|
||||
"LICENSE.TXT",
|
||||
),
|
||||
"LICENSE-musl.txt": path.join(
|
||||
sourceRoot,
|
||||
"system",
|
||||
"lib",
|
||||
"libc",
|
||||
"musl",
|
||||
"COPYRIGHT",
|
||||
),
|
||||
};
|
||||
for (const [name, file] of Object.entries(legalFiles)) {
|
||||
const licenseDetails = await lstat(file).catch(() => null);
|
||||
if (!licenseDetails?.isFile() || licenseDetails.isSymbolicLink()) {
|
||||
throw new Error(`Required toolchain licence is missing: ${file}`);
|
||||
}
|
||||
if (sha256(await readFile(file)) !== CPP_LOCK.legalFiles[name]) {
|
||||
throw new Error(
|
||||
`The pinned Emscripten toolchain legal file drifted: ${name}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const [name, digest] of Object.entries(CPP_LOCK.sourceEvidence)) {
|
||||
const file = path.join(sourceRoot, name);
|
||||
const sourceDetails = await lstat(file).catch(() => null);
|
||||
if (!sourceDetails?.isFile() || sourceDetails.isSymbolicLink()) {
|
||||
throw new Error(`Required toolchain source evidence is missing: ${file}`);
|
||||
}
|
||||
if (sha256(await readFile(file)) !== digest) {
|
||||
throw new Error(
|
||||
`The pinned Emscripten source evidence drifted: ${name}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const clang = await realpath(path.join(upstreamRoot, "bin", "clang++"));
|
||||
const clangDetails = await lstat(clang).catch(() => null);
|
||||
if (!clangDetails?.isFile()) {
|
||||
throw new Error(`Required pinned clang++ is missing: ${clang}`);
|
||||
}
|
||||
const clangVersion = await run(clang, ["--version"]);
|
||||
if (
|
||||
!clangVersion.stdout.startsWith(`clang version ${CPP_LOCK.llvmVersion} `) ||
|
||||
!clangVersion.stdout.includes(CPP_LOCK.llvmCommit)
|
||||
) {
|
||||
throw new Error(
|
||||
`The C++ pack requires LLVM ${CPP_LOCK.llvmVersion} at commit ${CPP_LOCK.llvmCommit}.`,
|
||||
);
|
||||
}
|
||||
return { executable, legalFiles };
|
||||
}
|
||||
|
||||
function buildEnvironment() {
|
||||
const environment = { ...process.env };
|
||||
for (const variable of [
|
||||
"CFLAGS",
|
||||
"CPPFLAGS",
|
||||
"CXXFLAGS",
|
||||
"EMCC_CFLAGS",
|
||||
"EM_CONFIG",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.SOURCE_DATE_EPOCH = String(CPP_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceRecords(projectRoot) {
|
||||
return Promise.all(
|
||||
SOURCE_FILES.map(async (name) => {
|
||||
const value = await readFile(path.join(projectRoot, name));
|
||||
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function linkedArchives(buildOutput) {
|
||||
const archives = new Set();
|
||||
for (const match of buildOutput.matchAll(
|
||||
/(?:^|\n)[^\n]*[\\/](lib[^/\\()\s]+\.a)\([^)\n]+\)/gu,
|
||||
)) {
|
||||
archives.add(match[1]);
|
||||
}
|
||||
return [...archives].sort();
|
||||
}
|
||||
|
||||
function assertLinkedArchives(buildOutput) {
|
||||
const actual = linkedArchives(buildOutput);
|
||||
if (
|
||||
actual.length !== CPP_LOCK.linkedArchives.length ||
|
||||
actual.some((name, index) => name !== CPP_LOCK.linkedArchives[index]) ||
|
||||
/[\\/]libunwind\.a\(/u.test(buildOutput)
|
||||
) {
|
||||
throw new Error(
|
||||
`C++ exact linker inventory drifted: ${actual.join(", ") || "(none)"}.`,
|
||||
);
|
||||
}
|
||||
return actual;
|
||||
}
|
||||
|
||||
async function expectedSourceManifest(pack) {
|
||||
const wasm = await fileRecord(pack, "cpp-regex.wasm");
|
||||
const legalFiles = await Promise.all(
|
||||
Object.keys(CPP_LOCK.legalFiles).map((name) => fileRecord(pack, name)),
|
||||
);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "cpp",
|
||||
scope: {
|
||||
asset: wasm.path,
|
||||
assetSha256: wasm.sha256,
|
||||
assetBytes: wasm.bytes,
|
||||
purpose:
|
||||
"Complete linked-component, preferred-source and legal inventory for the exact C++ WebAssembly module",
|
||||
},
|
||||
sourceSnapshot: {
|
||||
project: "Emscripten",
|
||||
version: CPP_LOCK.emscriptenVersion,
|
||||
tag: CPP_LOCK.emscriptenVersion,
|
||||
commitSha1: CPP_LOCK.emscriptenCommit,
|
||||
repository: "https://github.com/emscripten-core/emscripten",
|
||||
preferredSourceUrl: `https://github.com/emscripten-core/emscripten/tree/${CPP_LOCK.emscriptenCommit}`,
|
||||
archiveUrl: `https://github.com/emscripten-core/emscripten/archive/${CPP_LOCK.emscriptenCommit}.tar.gz`,
|
||||
emsdk: {
|
||||
tag: CPP_LOCK.emscriptenVersion,
|
||||
commitSha1: CPP_LOCK.emsdkCommit,
|
||||
releaseCommitSha1: CPP_LOCK.releaseCommit,
|
||||
repository: "https://github.com/emscripten-core/emsdk",
|
||||
},
|
||||
llvmToolchain: {
|
||||
version: CPP_LOCK.llvmVersion,
|
||||
commitSha1: CPP_LOCK.llvmCommit,
|
||||
repository: "https://github.com/llvm/llvm-project",
|
||||
},
|
||||
},
|
||||
buildSelection: {
|
||||
exceptionMode: "Emscripten JavaScript exceptions (-fexceptions)",
|
||||
allocator: "emmalloc (-sMALLOC=emmalloc)",
|
||||
exactLinkedArchives: [...CPP_LOCK.linkedArchives],
|
||||
inventoryMethod:
|
||||
"the pack builder adds wasm-ld --trace, accepts this exact archive set and fails on any addition, removal or libunwind selection",
|
||||
},
|
||||
sourceEvidence: Object.entries(CPP_LOCK.sourceEvidence).map(
|
||||
([sourcePath, sourceSha256]) => ({ sourcePath, sourceSha256 }),
|
||||
),
|
||||
linkedComponents: SOURCE_COMPONENTS,
|
||||
legalFiles,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeSourceManifest(pack) {
|
||||
await writeFile(
|
||||
path.join(pack, "SOURCE-MANIFEST.json"),
|
||||
`${JSON.stringify(await expectedSourceManifest(pack), null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
|
||||
async function writeMetadata(pack, projectRoot) {
|
||||
const metadata = {
|
||||
schemaVersion: 1,
|
||||
engine: "cpp",
|
||||
engineName: "C++ std::wregex (Emscripten libc++)",
|
||||
engineVersion: CPP_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
"Emscripten 6.0.4 libc++ std::wregex; C++ modified ECMAScript grammar by default",
|
||||
status: "production-native-runtime",
|
||||
bridge: {
|
||||
abiVersion: CPP_LOCK.bridgeAbi,
|
||||
nativeOffsetUnit: "Unicode code points (32-bit wchar_t)",
|
||||
requestEncoding: "structured Embind values; no user source evaluation",
|
||||
dynamicExecutionDisabled: true,
|
||||
replacement: {
|
||||
semantics: "libc++ match_results::format default grammar",
|
||||
implementation:
|
||||
"bounded streaming expansion without materializing an unbounded native format result",
|
||||
paritySelfTest:
|
||||
"load-time fixtures compare the bounded expander with libc++ match_results::format",
|
||||
},
|
||||
supportedApplicationFlags: ["g", "i", "n", "o", "c"],
|
||||
grammars: ["ECMAScript", "basic", "extended", "awk", "grep", "egrep"],
|
||||
limits: {
|
||||
maximumPatternUtf16: 65_536,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementUtf16: 65_536,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
},
|
||||
sourceFiles: await sourceRecords(projectRoot),
|
||||
},
|
||||
source: {
|
||||
project: "Emscripten system runtime and LLVM libc++",
|
||||
repository: "https://github.com/emscripten-core/emscripten",
|
||||
tag: CPP_LOCK.emscriptenVersion,
|
||||
commitSha1: CPP_LOCK.emscriptenCommit,
|
||||
preferredSourceUrl: `https://github.com/emscripten-core/emscripten/tree/${CPP_LOCK.emscriptenCommit}`,
|
||||
sourceManifest: "SOURCE-MANIFEST.json",
|
||||
license:
|
||||
"MIT OR NCSA AND MIT AND Apache-2.0 WITH LLVM-exception and legacy component terms",
|
||||
},
|
||||
toolchain: {
|
||||
version: CPP_LOCK.emscriptenVersion,
|
||||
commitSha1: CPP_LOCK.emscriptenCommit,
|
||||
emsdkCommitSha1: CPP_LOCK.emsdkCommit,
|
||||
releaseCommitSha1: CPP_LOCK.releaseCommit,
|
||||
llvmVersion: CPP_LOCK.llvmVersion,
|
||||
llvmCommitSha1: CPP_LOCK.llvmCommit,
|
||||
target: "wasm32-unknown-emscripten",
|
||||
linkedArchives: [...CPP_LOCK.linkedArchives],
|
||||
command:
|
||||
"em++ -std=c++20 -O3 -fexceptions --bind -sMODULARIZE=1 -sEXPORT_ES6=1 -sMALLOC=emmalloc -sDYNAMIC_EXECUTION=0 -Wl,--trace",
|
||||
},
|
||||
licensing: {
|
||||
sourceManifest: "SOURCE-MANIFEST.json",
|
||||
linkedComponentCount: SOURCE_COMPONENTS.filter(
|
||||
(component) => component.linked,
|
||||
).length,
|
||||
auditedExclusions: ["libunwind"],
|
||||
independentlyPinnedLegalFiles: Object.keys(CPP_LOCK.legalFiles).length,
|
||||
},
|
||||
sourceDateEpoch: CPP_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(
|
||||
[
|
||||
...Object.keys(CPP_LOCK.legalFiles),
|
||||
"SOURCE-MANIFEST.json",
|
||||
"cpp-regex.mjs",
|
||||
"cpp-regex.wasm",
|
||||
].map((name) => fileRecord(pack, name)),
|
||||
),
|
||||
};
|
||||
await writeFile(
|
||||
path.join(pack, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeEngineChecksums(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export async function verifyCppPack(pack, projectRoot = root) {
|
||||
const metadata = await verifyChecksummedEnginePack(pack, "cpp");
|
||||
const files = (await readdir(pack)).sort();
|
||||
const expectedFiles = [...PACK_FILES].sort();
|
||||
let sourceManifest;
|
||||
try {
|
||||
sourceManifest = JSON.parse(
|
||||
await readFile(path.join(pack, "SOURCE-MANIFEST.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("C++ SOURCE-MANIFEST.json is missing or invalid.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (
|
||||
files.length !== expectedFiles.length ||
|
||||
files.some((name, index) => name !== expectedFiles[index]) ||
|
||||
metadata.engineVersion !== CPP_LOCK.engineVersion ||
|
||||
metadata.toolchain?.version !== CPP_LOCK.emscriptenVersion ||
|
||||
metadata.toolchain?.commitSha1 !== CPP_LOCK.emscriptenCommit ||
|
||||
metadata.toolchain?.emsdkCommitSha1 !== CPP_LOCK.emsdkCommit ||
|
||||
metadata.toolchain?.releaseCommitSha1 !== CPP_LOCK.releaseCommit ||
|
||||
metadata.toolchain?.llvmVersion !== CPP_LOCK.llvmVersion ||
|
||||
metadata.toolchain?.llvmCommitSha1 !== CPP_LOCK.llvmCommit ||
|
||||
JSON.stringify(metadata.toolchain?.linkedArchives) !==
|
||||
JSON.stringify(CPP_LOCK.linkedArchives) ||
|
||||
metadata.bridge?.abiVersion !== CPP_LOCK.bridgeAbi ||
|
||||
metadata.bridge?.dynamicExecutionDisabled !== true ||
|
||||
metadata.bridge?.replacement?.semantics !==
|
||||
"libc++ match_results::format default grammar" ||
|
||||
metadata.bridge?.replacement?.implementation !==
|
||||
"bounded streaming expansion without materializing an unbounded native format result" ||
|
||||
metadata.bridge?.replacement?.paritySelfTest !==
|
||||
"load-time fixtures compare the bounded expander with libc++ match_results::format" ||
|
||||
JSON.stringify(metadata.bridge?.sourceFiles) !==
|
||||
JSON.stringify(await sourceRecords(projectRoot)) ||
|
||||
JSON.stringify(sourceManifest) !==
|
||||
JSON.stringify(await expectedSourceManifest(pack)) ||
|
||||
metadata.source?.sourceManifest !== "SOURCE-MANIFEST.json" ||
|
||||
metadata.licensing?.linkedComponentCount !==
|
||||
SOURCE_COMPONENTS.filter((component) => component.linked).length ||
|
||||
JSON.stringify(metadata.licensing?.auditedExclusions) !==
|
||||
JSON.stringify(["libunwind"]) ||
|
||||
metadata.licensing?.independentlyPinnedLegalFiles !==
|
||||
Object.keys(CPP_LOCK.legalFiles).length ||
|
||||
Object.entries(CPP_LOCK.legalFiles).some(
|
||||
([name, digest]) =>
|
||||
metadata.files?.find((file) => file.path === name)?.sha256 !== digest,
|
||||
)
|
||||
) {
|
||||
throw new Error("C++ engine metadata drifted from its pinned sources.");
|
||||
}
|
||||
const module = await readFile(path.join(pack, "cpp-regex.mjs"), "utf8");
|
||||
if (module.includes("new Function") || /\beval\s*\(/u.test(module)) {
|
||||
throw new Error("C++ engine module contains dynamic JavaScript execution.");
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export async function buildCppPack(
|
||||
empp,
|
||||
output = path.join(root, ".engine-build", "cpp"),
|
||||
projectRoot = root,
|
||||
) {
|
||||
const toolchain = await exactEmpp(empp);
|
||||
const outputParent = path.dirname(output);
|
||||
await mkdir(outputParent, { recursive: true });
|
||||
const staging = await mkdtemp(path.join(outputParent, ".cpp-build-"));
|
||||
try {
|
||||
const build = await run(
|
||||
toolchain.executable,
|
||||
[
|
||||
path.join(projectRoot, "engines", "cpp", "cpp_regex_bridge.cpp"),
|
||||
"-std=c++20",
|
||||
"-O3",
|
||||
"-fexceptions",
|
||||
"--bind",
|
||||
"-sMODULARIZE=1",
|
||||
"-sEXPORT_ES6=1",
|
||||
"-sENVIRONMENT=web,worker,node",
|
||||
"-sFILESYSTEM=0",
|
||||
"-sALLOW_MEMORY_GROWTH=1",
|
||||
"-sINITIAL_MEMORY=16777216",
|
||||
"-sMAXIMUM_MEMORY=268435456",
|
||||
"-sMALLOC=emmalloc",
|
||||
"-sASSERTIONS=0",
|
||||
"-sDYNAMIC_EXECUTION=0",
|
||||
"-sINCOMING_MODULE_JS_API=locateFile,print,printErr,wasmBinary",
|
||||
"-sEXPORT_NAME=createCppRegexModule",
|
||||
"-Wl,--trace",
|
||||
"-o",
|
||||
path.join(staging, "cpp-regex.mjs"),
|
||||
],
|
||||
{ cwd: projectRoot, env: buildEnvironment() },
|
||||
);
|
||||
assertLinkedArchives(`${build.stdout}\n${build.stderr}`);
|
||||
for (const [name, source] of Object.entries(toolchain.legalFiles)) {
|
||||
await copyFile(source, path.join(staging, name));
|
||||
}
|
||||
await writeSourceManifest(staging);
|
||||
await writeMetadata(staging, projectRoot);
|
||||
await verifyCppPack(staging, projectRoot);
|
||||
await rm(output, { recursive: true, force: true });
|
||||
await rename(staging, output);
|
||||
return { output, metadata: await verifyCppPack(output, projectRoot) };
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function installCppPack(source, projectRoot = root) {
|
||||
await verifyCppPack(source, projectRoot);
|
||||
const destination = path.join(projectRoot, "public", "engines", "cpp");
|
||||
const parent = path.dirname(destination);
|
||||
const staging = await mkdtemp(path.join(parent, ".cpp-install-"));
|
||||
try {
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(path.join(source, name), path.join(staging, name));
|
||||
}
|
||||
await verifyCppPack(staging, projectRoot);
|
||||
await rm(destination, { recursive: true, force: true });
|
||||
await rename(staging, destination);
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
return { output: destination, metadata: await verifyCppPack(destination) };
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length !== 2 || arguments_[0] !== "--empp") {
|
||||
throw new Error(
|
||||
"Usage: node scripts/cpp-engine-pack.mjs --empp /absolute/path/to/em++",
|
||||
);
|
||||
}
|
||||
const result = await buildCppPack(arguments_[1]);
|
||||
console.log(`Built verified C++ engine pack at ${result.output}.`);
|
||||
}
|
||||
199
scripts/cpp-engine-pack.test.ts
Normal file
199
scripts/cpp-engine-pack.test.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { writeEngineChecksums } from "./checksummed-engine-pack.mjs";
|
||||
import { CPP_LOCK, verifyCppPack } from "./cpp-engine-pack.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("C++ engine-pack gate", () => {
|
||||
it("verifies the installed exact Emscripten/libc++ pack and bridge sources", async () => {
|
||||
await expect(
|
||||
verifyCppPack(path.resolve("public", "engines", "cpp")),
|
||||
).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: "Emscripten 6.0.4 libc++ std::wregex",
|
||||
bridge: expect.objectContaining({
|
||||
abiVersion: 1,
|
||||
dynamicExecutionDisabled: true,
|
||||
replacement: {
|
||||
semantics: "libc++ match_results::format default grammar",
|
||||
implementation:
|
||||
"bounded streaming expansion without materializing an unbounded native format result",
|
||||
paritySelfTest:
|
||||
"load-time fixtures compare the bounded expander with libc++ match_results::format",
|
||||
},
|
||||
}),
|
||||
toolchain: expect.objectContaining({
|
||||
version: "6.0.4",
|
||||
commitSha1: "fe5be6afdff43ad58860d821fcc8572a23f92d19",
|
||||
emsdkCommitSha1: "224ec5f9f2f72f09f9ce0e26d66bae7dbd8b692f",
|
||||
releaseCommitSha1: "b23272fac5d05617cd36dc451356dca0f79bf22d",
|
||||
llvmVersion: "24.0.0git",
|
||||
llvmCommitSha1: "c050c487e9edb97ef44f53cb29fe1d8bcddb8f76",
|
||||
}),
|
||||
licensing: {
|
||||
sourceManifest: "SOURCE-MANIFEST.json",
|
||||
linkedComponentCount: 6,
|
||||
auditedExclusions: ["libunwind"],
|
||||
independentlyPinnedLegalFiles: 6,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("pins exception-enabled compilation and its exact traced native component set", async () => {
|
||||
expect(CPP_LOCK.emscriptenVersion).toBe("6.0.4");
|
||||
expect(CPP_LOCK.linkedArchives).toEqual([
|
||||
"libc++.a",
|
||||
"libc++abi.a",
|
||||
"libc.a",
|
||||
"libclang_rt.builtins.a",
|
||||
"libembind-rtti.a",
|
||||
"libemmalloc.a",
|
||||
"libnoexit.a",
|
||||
"libstubs.a",
|
||||
]);
|
||||
const metadata = JSON.parse(
|
||||
await readFile(
|
||||
path.resolve("public", "engines", "cpp", "engine-metadata.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as { toolchain: { command: string } };
|
||||
expect(metadata.toolchain.command).toContain("-fexceptions");
|
||||
expect(metadata.toolchain.command).toContain("-sMALLOC=emmalloc");
|
||||
expect(metadata.toolchain.command).toContain("-Wl,--trace");
|
||||
|
||||
const sourceManifest = JSON.parse(
|
||||
await readFile(
|
||||
path.resolve("public", "engines", "cpp", "SOURCE-MANIFEST.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as {
|
||||
buildSelection: { exactLinkedArchives: string[] };
|
||||
sourceEvidence: { sourcePath: string; sourceSha256: string }[];
|
||||
linkedComponents: {
|
||||
id: string;
|
||||
linked: boolean;
|
||||
legalFiles: string[];
|
||||
}[];
|
||||
};
|
||||
expect(sourceManifest.buildSelection.exactLinkedArchives).toEqual(
|
||||
CPP_LOCK.linkedArchives,
|
||||
);
|
||||
expect(sourceManifest.sourceEvidence).toContainEqual({
|
||||
sourcePath: "system/lib/emmalloc.c",
|
||||
sourceSha256:
|
||||
"00eb577cbef51a3359c309f45a7c99a814b8c6151ab75c9a3c78a56f0169146b",
|
||||
});
|
||||
expect(
|
||||
sourceManifest.linkedComponents
|
||||
.filter(({ linked }) => linked)
|
||||
.map(({ id }) => id),
|
||||
).toEqual([
|
||||
"emscripten-runtime",
|
||||
"musl",
|
||||
"emmalloc",
|
||||
"compiler-rt",
|
||||
"libc++",
|
||||
"libc++abi",
|
||||
]);
|
||||
expect(
|
||||
sourceManifest.linkedComponents.find(({ id }) => id === "libunwind"),
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
linked: false,
|
||||
legalFiles: ["LICENSE-libunwind.txt"],
|
||||
}),
|
||||
);
|
||||
|
||||
const module = await readFile(
|
||||
path.resolve("public", "engines", "cpp", "cpp-regex.mjs"),
|
||||
"utf8",
|
||||
);
|
||||
expect(module).not.toContain("new Function");
|
||||
expect(module).not.toMatch(/\beval\s*\(/u);
|
||||
});
|
||||
|
||||
it("rejects substituted legal text even when self-reported checksums are regenerated", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-cpp-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "cpp");
|
||||
await cp(path.resolve("public", "engines", "cpp"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
const replacement = Buffer.from("substituted legal text\n");
|
||||
await writeFile(path.join(pack, "LICENSE-Emscripten.txt"), replacement);
|
||||
const metadataFile = path.join(pack, "engine-metadata.json");
|
||||
const metadata = JSON.parse(await readFile(metadataFile, "utf8")) as {
|
||||
files: { path: string; sha256: string; bytes: number }[];
|
||||
};
|
||||
const record = metadata.files.find(
|
||||
({ path: name }) => name === "LICENSE-Emscripten.txt",
|
||||
);
|
||||
expect(record).toBeDefined();
|
||||
if (!record) return;
|
||||
record.sha256 = createHash("sha256").update(replacement).digest("hex");
|
||||
record.bytes = replacement.byteLength;
|
||||
await writeFile(metadataFile, `${JSON.stringify(metadata, null, 2)}\n`);
|
||||
await writeEngineChecksums(pack);
|
||||
|
||||
await expect(verifyCppPack(pack)).rejects.toThrow(
|
||||
/metadata drifted from its pinned sources/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a forged source inventory even when self-reported checksums are regenerated", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-cpp-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "cpp");
|
||||
await cp(path.resolve("public", "engines", "cpp"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
const sourceManifestFile = path.join(pack, "SOURCE-MANIFEST.json");
|
||||
const sourceManifest = JSON.parse(
|
||||
await readFile(sourceManifestFile, "utf8"),
|
||||
) as {
|
||||
linkedComponents: { id: string; linked: boolean }[];
|
||||
};
|
||||
const libunwind = sourceManifest.linkedComponents.find(
|
||||
({ id }) => id === "libunwind",
|
||||
);
|
||||
expect(libunwind).toBeDefined();
|
||||
if (!libunwind) return;
|
||||
libunwind.linked = true;
|
||||
const replacement = Buffer.from(
|
||||
`${JSON.stringify(sourceManifest, null, 2)}\n`,
|
||||
);
|
||||
await writeFile(sourceManifestFile, replacement);
|
||||
const metadataFile = path.join(pack, "engine-metadata.json");
|
||||
const metadata = JSON.parse(await readFile(metadataFile, "utf8")) as {
|
||||
files: { path: string; sha256: string; bytes: number }[];
|
||||
};
|
||||
const record = metadata.files.find(
|
||||
({ path: name }) => name === "SOURCE-MANIFEST.json",
|
||||
);
|
||||
expect(record).toBeDefined();
|
||||
if (!record) return;
|
||||
record.sha256 = createHash("sha256").update(replacement).digest("hex");
|
||||
record.bytes = replacement.byteLength;
|
||||
await writeFile(metadataFile, `${JSON.stringify(metadata, null, 2)}\n`);
|
||||
await writeEngineChecksums(pack);
|
||||
|
||||
await expect(verifyCppPack(pack)).rejects.toThrow(
|
||||
/metadata drifted from its pinned sources/u,
|
||||
);
|
||||
});
|
||||
});
|
||||
465
scripts/dotnet-engine-pack.mjs
Normal file
465
scripts/dotnet-engine-pack.mjs
Normal file
@@ -0,0 +1,465 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFile,
|
||||
cp,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
verifyChecksummedEnginePack,
|
||||
writeEngineChecksums,
|
||||
} from "./checksummed-engine-pack.mjs";
|
||||
import { assertNoUnexpectedLocalPaths } from "./release-path-hygiene.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
export const DOTNET_LOCK = Object.freeze({
|
||||
sdkVersion: "10.0.302",
|
||||
sdkCommit: "35b593bebf",
|
||||
runtimeVersion: "10.0.10",
|
||||
runtimeCommit: "f7d90799ce",
|
||||
engineVersion: ".NET 10.0.10 System.Text.RegularExpressions",
|
||||
sourceDateEpoch: 1_785_110_400,
|
||||
bridgeAbi: 1,
|
||||
legalFiles: Object.freeze({
|
||||
"LICENSE-dotnet.txt":
|
||||
"cfc21f5e8bd655ae997eec916138b707b1d290b83272c02a95c9f821b8c87310",
|
||||
"THIRD-PARTY-NOTICES.txt":
|
||||
"2dc8f8c5a39401e928b5784ab564eb8b3ceb99ead3df8f260e0cab7e0bbecc7a",
|
||||
}),
|
||||
});
|
||||
|
||||
const SOURCE_FILES = Object.freeze([
|
||||
"engines/dotnet/Program.cs",
|
||||
"engines/dotnet/README.md",
|
||||
"engines/dotnet/RegexTools.DotNet.csproj",
|
||||
"engines/dotnet/wwwroot/main.js",
|
||||
]);
|
||||
|
||||
function compareText(left, right) {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function fileRecord(directory, name) {
|
||||
const value = await readFile(path.join(directory, name));
|
||||
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
||||
}
|
||||
|
||||
async function regularFiles(directory, relative = "") {
|
||||
const entries = (
|
||||
await readdir(path.join(directory, relative), {
|
||||
withFileTypes: true,
|
||||
})
|
||||
).sort((left, right) => compareText(left.name, right.name));
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
const candidate = path.join(directory, name);
|
||||
const details = await lstat(candidate);
|
||||
if (details.isSymbolicLink()) {
|
||||
throw new Error(`.NET engine assets may not be symlinks: ${candidate}`);
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await regularFiles(directory, name)));
|
||||
} else if (entry.isFile()) {
|
||||
files.push(name);
|
||||
} else {
|
||||
throw new Error(`Unsupported .NET engine asset: ${candidate}`);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function normalizeFrameworkPaths(
|
||||
framework,
|
||||
{ dotnetRoot, projectRoot, workspace },
|
||||
) {
|
||||
const replacements = [
|
||||
[workspace, "dotnet-build"],
|
||||
[dotnetRoot, `dotnet-sdk-${DOTNET_LOCK.sdkVersion}`],
|
||||
[path.resolve(projectRoot), "regex-tools-source"],
|
||||
].sort(
|
||||
([left], [right]) => Buffer.byteLength(right) - Buffer.byteLength(left),
|
||||
);
|
||||
const javascriptFiles = (await regularFiles(framework)).filter((name) =>
|
||||
name.endsWith(".js"),
|
||||
);
|
||||
for (const name of javascriptFiles) {
|
||||
const filename = path.join(framework, name);
|
||||
let source = await readFile(filename, "utf8");
|
||||
for (const [localPath, stableName] of replacements) {
|
||||
source = source
|
||||
.replaceAll(localPath, stableName)
|
||||
.replaceAll(localPath.replaceAll("\\", "\\\\"), stableName);
|
||||
}
|
||||
await writeFile(filename, source, "utf8");
|
||||
}
|
||||
|
||||
const nativeModules = javascriptFiles.filter((name) =>
|
||||
/^dotnet\.native\.[a-z0-9]+\.js$/u.test(name),
|
||||
);
|
||||
if (nativeModules.length !== 1) {
|
||||
throw new Error(
|
||||
"The .NET publish must produce exactly one fingerprinted native JavaScript module.",
|
||||
);
|
||||
}
|
||||
const originalName = nativeModules[0];
|
||||
const nativeSource = await readFile(path.join(framework, originalName));
|
||||
const stableName = `dotnet.native.${sha256(nativeSource).slice(0, 12)}.js`;
|
||||
if (stableName === originalName) return;
|
||||
|
||||
for (const name of javascriptFiles) {
|
||||
if (name === originalName) continue;
|
||||
const filename = path.join(framework, name);
|
||||
const source = await readFile(filename, "utf8");
|
||||
if (source.includes(originalName)) {
|
||||
await writeFile(
|
||||
filename,
|
||||
source.replaceAll(originalName, stableName),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
}
|
||||
await rename(
|
||||
path.join(framework, originalName),
|
||||
path.join(framework, stableName),
|
||||
);
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
throw new Error(
|
||||
`${command} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}:\n${[output, errorOutput].filter(Boolean).join("\n").trim()}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
async function exactDotNet(candidate) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`dotnet must be a regular file: ${candidate}`);
|
||||
}
|
||||
const executable = await realpath(candidate);
|
||||
const version = (await run(executable, ["--version"])).stdout.trim();
|
||||
const information = (await run(executable, ["--info"])).stdout;
|
||||
if (
|
||||
version !== DOTNET_LOCK.sdkVersion ||
|
||||
!information.includes(`Commit: ${DOTNET_LOCK.sdkCommit}`) ||
|
||||
!information.includes(`Version: ${DOTNET_LOCK.runtimeVersion}`) ||
|
||||
!information.includes(`Commit: ${DOTNET_LOCK.runtimeCommit}`) ||
|
||||
!information.includes("[wasm-tools]")
|
||||
) {
|
||||
throw new Error(
|
||||
`The .NET pack requires SDK ${DOTNET_LOCK.sdkVersion}, runtime ${DOTNET_LOCK.runtimeVersion}, and its wasm-tools workload.`,
|
||||
);
|
||||
}
|
||||
const sdkRoot = path.dirname(executable);
|
||||
const license = path.join(sdkRoot, "LICENSE.txt");
|
||||
const notices = path.join(sdkRoot, "ThirdPartyNotices.txt");
|
||||
for (const file of [license, notices]) {
|
||||
const legalDetails = await lstat(file).catch(() => null);
|
||||
if (!legalDetails?.isFile() || legalDetails.isSymbolicLink()) {
|
||||
throw new Error(`Required .NET legal file is missing: ${file}`);
|
||||
}
|
||||
}
|
||||
for (const [name, file] of [
|
||||
["LICENSE-dotnet.txt", license],
|
||||
["THIRD-PARTY-NOTICES.txt", notices],
|
||||
]) {
|
||||
if (sha256(await readFile(file)) !== DOTNET_LOCK.legalFiles[name]) {
|
||||
throw new Error(`The pinned .NET SDK legal file drifted: ${name}.`);
|
||||
}
|
||||
}
|
||||
return { executable, license, notices };
|
||||
}
|
||||
|
||||
function buildEnvironment(dotnetRoot) {
|
||||
const environment = { ...process.env };
|
||||
for (const variable of [
|
||||
"CFLAGS",
|
||||
"CPPFLAGS",
|
||||
"CXXFLAGS",
|
||||
"DOTNET_ROOT",
|
||||
"MSBuildSDKsPath",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.DOTNET_ROOT = dotnetRoot;
|
||||
environment.DOTNET_CLI_TELEMETRY_OPTOUT = "1";
|
||||
environment.DOTNET_NOLOGO = "1";
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.SOURCE_DATE_EPOCH = String(DOTNET_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceRecords(projectRoot) {
|
||||
return Promise.all(
|
||||
SOURCE_FILES.map(async (name) => {
|
||||
const value = await readFile(path.join(projectRoot, name));
|
||||
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function writeMetadata(pack, projectRoot) {
|
||||
const assetNames = (await regularFiles(pack))
|
||||
.filter((name) => name !== "SHA256SUMS" && name !== "engine-metadata.json")
|
||||
.sort(compareText);
|
||||
const metadata = {
|
||||
schemaVersion: 1,
|
||||
engine: "dotnet",
|
||||
engineName: "System.Text.RegularExpressions",
|
||||
engineVersion: DOTNET_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
".NET 10.0.10 System.Text.RegularExpressions; invariant culture; UTF-16 offsets",
|
||||
status: "production-native-runtime",
|
||||
bridge: {
|
||||
abiVersion: DOTNET_LOCK.bridgeAbi,
|
||||
nativeOffsetUnit: "UTF-16 code units",
|
||||
requestEncoding: "bounded JSON passed to fixed JSExport methods",
|
||||
userSourceEvaluation: false,
|
||||
replacement:
|
||||
"Fixed streaming parser for .NET replacement tokens ($$, $n, ${name}, $&, $`, $', $+, $_), verified against Match.Result at load",
|
||||
replacementOutputTransport:
|
||||
"bounded UTF-8 bytes encoded as canonical base64 in the JSON response",
|
||||
supportedApplicationFlags: ["g", "i", "m", "s", "n", "x", "r", "c", "b"],
|
||||
matchTimeoutMilliseconds: 9_000,
|
||||
limits: {
|
||||
maximumPatternUtf16: 65_536,
|
||||
maximumSubjectUtf16: 16_777_216,
|
||||
maximumReplacementUtf16: 65_536,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
},
|
||||
sourceFiles: await sourceRecords(projectRoot),
|
||||
},
|
||||
source: {
|
||||
project: ".NET",
|
||||
repository: "https://github.com/dotnet/runtime",
|
||||
license: "MIT",
|
||||
runtimeVersion: DOTNET_LOCK.runtimeVersion,
|
||||
runtimeCommit: DOTNET_LOCK.runtimeCommit,
|
||||
},
|
||||
toolchain: {
|
||||
sdkVersion: DOTNET_LOCK.sdkVersion,
|
||||
sdkCommit: DOTNET_LOCK.sdkCommit,
|
||||
workload: "wasm-tools",
|
||||
target: "browser-wasm",
|
||||
command:
|
||||
"dotnet publish engines/dotnet/RegexTools.DotNet.csproj -c Release --no-restore -p:PathMap=<source>=/regex-tools-source -p:EmccExtraCFlags=-ffile-prefix-map=<sdk>=/dotnet-sdk-10.0.302",
|
||||
},
|
||||
sourceDateEpoch: DOTNET_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(assetNames.map((name) => fileRecord(pack, name))),
|
||||
};
|
||||
await writeFile(
|
||||
path.join(pack, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeEngineChecksums(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export async function verifyDotNetPack(pack, projectRoot = root) {
|
||||
const metadata = await verifyChecksummedEnginePack(pack, "dotnet");
|
||||
const files = (await regularFiles(pack)).sort(compareText);
|
||||
const framework = files.filter((name) => name.startsWith("_framework/"));
|
||||
const requiredPatterns = [
|
||||
/^_framework\/RegexTools\.DotNet\.[a-z0-9]+\.wasm$/u,
|
||||
/^_framework\/System\.Text\.RegularExpressions\.[a-z0-9]+\.wasm$/u,
|
||||
/^_framework\/dotnet\.native\.[a-z0-9]+\.wasm$/u,
|
||||
/^_framework\/dotnet\.native\.[a-z0-9]+\.js$/u,
|
||||
/^_framework\/dotnet\.runtime\.[a-z0-9]+\.js$/u,
|
||||
];
|
||||
if (
|
||||
metadata.engineVersion !== DOTNET_LOCK.engineVersion ||
|
||||
metadata.source?.runtimeVersion !== DOTNET_LOCK.runtimeVersion ||
|
||||
metadata.source?.runtimeCommit !== DOTNET_LOCK.runtimeCommit ||
|
||||
metadata.toolchain?.sdkVersion !== DOTNET_LOCK.sdkVersion ||
|
||||
metadata.toolchain?.sdkCommit !== DOTNET_LOCK.sdkCommit ||
|
||||
metadata.bridge?.abiVersion !== DOTNET_LOCK.bridgeAbi ||
|
||||
metadata.bridge?.userSourceEvaluation !== false ||
|
||||
!String(metadata.bridge?.replacement).includes("Match.Result") ||
|
||||
!String(metadata.bridge?.replacementOutputTransport).includes("base64") ||
|
||||
JSON.stringify(metadata.bridge?.sourceFiles) !==
|
||||
JSON.stringify(await sourceRecords(projectRoot)) ||
|
||||
Object.entries(DOTNET_LOCK.legalFiles).some(
|
||||
([name, digest]) =>
|
||||
metadata.files?.find((file) => file.path === name)?.sha256 !== digest,
|
||||
) ||
|
||||
!files.includes("_framework/dotnet.js") ||
|
||||
!files.includes("LICENSE-dotnet.txt") ||
|
||||
!files.includes("THIRD-PARTY-NOTICES.txt") ||
|
||||
files.some((name) => /\.(?:br|gz|map|pdb)$/u.test(name)) ||
|
||||
requiredPatterns.some(
|
||||
(pattern) => framework.filter((name) => pattern.test(name)).length !== 1,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
".NET engine pack drifted from its pinned runtime contract.",
|
||||
);
|
||||
}
|
||||
const mainModule = await readFile(
|
||||
path.join(pack, "_framework/dotnet.js"),
|
||||
"utf8",
|
||||
);
|
||||
if (!mainModule.includes("dotnet") || !mainModule.includes("./")) {
|
||||
throw new Error(".NET engine entry module has an unexpected shape.");
|
||||
}
|
||||
for (const name of framework) {
|
||||
assertNoUnexpectedLocalPaths(
|
||||
await readFile(path.join(pack, name)),
|
||||
`.NET engine asset ${name}`,
|
||||
{
|
||||
allowedVirtualHomes:
|
||||
/^_framework\/dotnet\.native\.[a-z0-9]+\.js$/u.test(name)
|
||||
? ["/home/web_user"]
|
||||
: [],
|
||||
},
|
||||
);
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export async function buildDotNetPack(
|
||||
dotnet,
|
||||
output = path.join(root, ".engine-build", "dotnet"),
|
||||
projectRoot = root,
|
||||
) {
|
||||
const toolchain = await exactDotNet(dotnet);
|
||||
const outputParent = path.dirname(output);
|
||||
await mkdir(outputParent, { recursive: true });
|
||||
const workspace = await mkdtemp(path.join(outputParent, ".dotnet-build-"));
|
||||
const staging = path.join(workspace, "pack");
|
||||
const publish = path.join(workspace, "publish");
|
||||
await mkdir(staging);
|
||||
try {
|
||||
const dotnetRoot = path.dirname(toolchain.executable);
|
||||
const projectPath = path.join(
|
||||
projectRoot,
|
||||
"engines",
|
||||
"dotnet",
|
||||
"RegexTools.DotNet.csproj",
|
||||
);
|
||||
await run(
|
||||
toolchain.executable,
|
||||
[
|
||||
"publish",
|
||||
projectPath,
|
||||
"-c",
|
||||
"Release",
|
||||
"--no-restore",
|
||||
"--output",
|
||||
publish,
|
||||
`-p:PathMap=${path.resolve(projectRoot)}=/regex-tools-source`,
|
||||
`-p:EmccExtraCFlags=-ffile-prefix-map=${dotnetRoot}=/dotnet-sdk-${DOTNET_LOCK.sdkVersion}`,
|
||||
],
|
||||
{
|
||||
cwd: projectRoot,
|
||||
env: buildEnvironment(dotnetRoot),
|
||||
},
|
||||
);
|
||||
const publishedFramework = path.join(publish, "wwwroot", "_framework");
|
||||
const frameworkDetails = await lstat(publishedFramework).catch(() => null);
|
||||
if (!frameworkDetails?.isDirectory() || frameworkDetails.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
"The .NET publish did not produce a WebAssembly framework.",
|
||||
);
|
||||
}
|
||||
await cp(publishedFramework, path.join(staging, "_framework"), {
|
||||
recursive: true,
|
||||
filter: (source) => !/\.(?:br|gz)$/u.test(source),
|
||||
});
|
||||
for (const name of await regularFiles(path.join(staging, "_framework"))) {
|
||||
if (/\.(?:map|pdb)$/u.test(name)) {
|
||||
await rm(path.join(staging, "_framework", name), { force: true });
|
||||
}
|
||||
}
|
||||
await normalizeFrameworkPaths(path.join(staging, "_framework"), {
|
||||
dotnetRoot: path.dirname(toolchain.executable),
|
||||
projectRoot,
|
||||
workspace,
|
||||
});
|
||||
await copyFile(toolchain.license, path.join(staging, "LICENSE-dotnet.txt"));
|
||||
await copyFile(
|
||||
toolchain.notices,
|
||||
path.join(staging, "THIRD-PARTY-NOTICES.txt"),
|
||||
);
|
||||
await writeMetadata(staging, projectRoot);
|
||||
await verifyDotNetPack(staging, projectRoot);
|
||||
await rm(output, { recursive: true, force: true });
|
||||
await rename(staging, output);
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
return { output, metadata: await verifyDotNetPack(output, projectRoot) };
|
||||
} catch (error) {
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function installDotNetPack(source, projectRoot = root) {
|
||||
await verifyDotNetPack(source, projectRoot);
|
||||
const destination = path.join(projectRoot, "public", "engines", "dotnet");
|
||||
const parent = path.dirname(destination);
|
||||
const staging = await mkdtemp(path.join(parent, ".dotnet-install-"));
|
||||
try {
|
||||
await cp(source, staging, { recursive: true });
|
||||
await verifyDotNetPack(staging, projectRoot);
|
||||
await rm(destination, { recursive: true, force: true });
|
||||
await rename(staging, destination);
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
output: destination,
|
||||
metadata: await verifyDotNetPack(destination, projectRoot),
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length !== 2 || arguments_[0] !== "--dotnet") {
|
||||
throw new Error(
|
||||
"Usage: node scripts/dotnet-engine-pack.mjs --dotnet /absolute/path/to/dotnet",
|
||||
);
|
||||
}
|
||||
const result = await buildDotNetPack(arguments_[1]);
|
||||
console.log(`Built verified .NET engine pack at ${result.output}.`);
|
||||
}
|
||||
131
scripts/dotnet-engine-pack.test.ts
Normal file
131
scripts/dotnet-engine-pack.test.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { writeEngineChecksums } from "./checksummed-engine-pack.mjs";
|
||||
import { DOTNET_LOCK, verifyDotNetPack } from "./dotnet-engine-pack.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe(".NET engine-pack gate", () => {
|
||||
it("verifies the installed exact runtime, bridge sources and closed checksums", async () => {
|
||||
await expect(
|
||||
verifyDotNetPack(path.resolve("public", "engines", "dotnet")),
|
||||
).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: ".NET 10.0.10 System.Text.RegularExpressions",
|
||||
source: expect.objectContaining({
|
||||
runtimeVersion: "10.0.10",
|
||||
runtimeCommit: "f7d90799ce",
|
||||
}),
|
||||
toolchain: expect.objectContaining({
|
||||
sdkVersion: "10.0.302",
|
||||
sdkCommit: "35b593bebf",
|
||||
workload: "wasm-tools",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("pins the compiler/runtime identity used by the checked-in pack", () => {
|
||||
expect(DOTNET_LOCK).toEqual(
|
||||
expect.objectContaining({
|
||||
sdkVersion: "10.0.302",
|
||||
runtimeVersion: "10.0.10",
|
||||
bridgeAbi: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects modified framework assets", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-dotnet-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "dotnet");
|
||||
await cp(path.resolve("public", "engines", "dotnet"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
const metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
) as {
|
||||
files: readonly { path: string }[];
|
||||
};
|
||||
const runtime = metadata.files.find(({ path: name }) =>
|
||||
/^_framework\/System\.Text\.RegularExpressions\..+\.wasm$/u.test(name),
|
||||
);
|
||||
expect(runtime).toBeDefined();
|
||||
await writeFile(path.join(pack, runtime?.path ?? ""), "modified");
|
||||
|
||||
await expect(verifyDotNetPack(pack)).rejects.toThrow(/SHA-256 mismatch/u);
|
||||
});
|
||||
|
||||
it("rejects substituted notices even when self-reported checksums are regenerated", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-dotnet-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "dotnet");
|
||||
await cp(path.resolve("public", "engines", "dotnet"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
const replacement = Buffer.from("substituted notices\n");
|
||||
await writeFile(path.join(pack, "THIRD-PARTY-NOTICES.txt"), replacement);
|
||||
const metadataFile = path.join(pack, "engine-metadata.json");
|
||||
const metadata = JSON.parse(await readFile(metadataFile, "utf8")) as {
|
||||
files: { path: string; sha256: string; bytes: number }[];
|
||||
};
|
||||
const record = metadata.files.find(
|
||||
({ path: name }) => name === "THIRD-PARTY-NOTICES.txt",
|
||||
);
|
||||
expect(record).toBeDefined();
|
||||
if (!record) return;
|
||||
record.sha256 = createHash("sha256").update(replacement).digest("hex");
|
||||
record.bytes = replacement.byteLength;
|
||||
await writeFile(metadataFile, `${JSON.stringify(metadata, null, 2)}\n`);
|
||||
await writeEngineChecksums(pack);
|
||||
|
||||
await expect(verifyDotNetPack(pack)).rejects.toThrow(
|
||||
/pinned runtime contract/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a build-host path hidden in WebAssembly after checksums are regenerated", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-dotnet-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "dotnet");
|
||||
await cp(path.resolve("public", "engines", "dotnet"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
const metadataFile = path.join(pack, "engine-metadata.json");
|
||||
const metadata = JSON.parse(await readFile(metadataFile, "utf8")) as {
|
||||
files: { path: string; sha256: string; bytes: number }[];
|
||||
};
|
||||
const record = metadata.files.find(({ path: name }) =>
|
||||
/^_framework\/RegexTools\.DotNet\..+\.wasm$/u.test(name),
|
||||
);
|
||||
expect(record).toBeDefined();
|
||||
if (!record) return;
|
||||
const assetFile = path.join(pack, record.path);
|
||||
const replacement = Buffer.concat([
|
||||
await readFile(assetFile),
|
||||
Buffer.from("/mnt/private/regex-tools/obj/linked/RegexTools.DotNet.pdb"),
|
||||
]);
|
||||
await writeFile(assetFile, replacement);
|
||||
record.sha256 = createHash("sha256").update(replacement).digest("hex");
|
||||
record.bytes = replacement.byteLength;
|
||||
await writeFile(metadataFile, `${JSON.stringify(metadata, null, 2)}\n`);
|
||||
await writeEngineChecksums(pack);
|
||||
|
||||
await expect(verifyDotNetPack(pack)).rejects.toThrow(
|
||||
/Local absolute path/u,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { lstat, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { format } from "prettier";
|
||||
@@ -8,6 +8,7 @@ const sourcePath = join(root, "src", "toolbox", "manifest.source.json");
|
||||
const outputPath = join(root, "public", "toolbox-app.json");
|
||||
const packagePath = join(root, "package.json");
|
||||
const applicationVersionPath = join(root, "src", "version.ts");
|
||||
const publicPath = join(root, "public");
|
||||
const checkOnly = process.argv.includes("--check");
|
||||
|
||||
const source = JSON.parse(await readFile(sourcePath, "utf8"));
|
||||
@@ -34,6 +35,25 @@ if (
|
||||
throw new Error("Manifest source identity is incomplete or inconsistent");
|
||||
}
|
||||
|
||||
if (!Array.isArray(source.assets)) {
|
||||
throw new Error("Manifest assets must be an array");
|
||||
}
|
||||
for (const asset of source.assets) {
|
||||
if (
|
||||
typeof asset !== "string" ||
|
||||
!asset.startsWith("./") ||
|
||||
asset.includes("\\") ||
|
||||
asset.split("/").includes("..")
|
||||
) {
|
||||
throw new Error(`Unsafe manifest asset path: ${JSON.stringify(asset)}`);
|
||||
}
|
||||
const candidate = join(publicPath, asset.slice(2));
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`Manifest asset is missing or unsafe: ${asset}`);
|
||||
}
|
||||
}
|
||||
|
||||
const serialized = await format(JSON.stringify(source), {
|
||||
filepath: outputPath,
|
||||
});
|
||||
|
||||
478
scripts/go-engine-pack.mjs
Normal file
478
scripts/go-engine-pack.mjs
Normal file
@@ -0,0 +1,478 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export const GO_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
engineVersion: "1.26.5",
|
||||
runtimeVersion: "go1.26.5",
|
||||
sourceDateEpoch: 1_783_452_544,
|
||||
source: Object.freeze({
|
||||
repository: "https://go.googlesource.com/go",
|
||||
tag: "go1.26.5",
|
||||
commit: "c19862e5f8415b4f24b189d065ed739517c548ba",
|
||||
tree: "0bb2fb1cc06c334c36a2a92d2f0b07fea7236d74",
|
||||
license: "BSD-3-Clause",
|
||||
}),
|
||||
toolchain: Object.freeze({
|
||||
archive: "https://go.dev/dl/go1.26.5.linux-amd64.tar.gz",
|
||||
archiveSha256:
|
||||
"5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053",
|
||||
licenseSha256:
|
||||
"911f8f5782931320f5b8d1160a76365b83aea6447ee6c04fa6d5591467db9dad",
|
||||
wasmExecSha256:
|
||||
"0c949f4996f9a89698e4b5c586de32249c3b69b7baadb64d220073cc04acba14",
|
||||
}),
|
||||
limits: Object.freeze({
|
||||
maximumPatternBytes: 262_144,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementBytes: 262_144,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
maximumRequestJSONBytes: 103_874_560,
|
||||
}),
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze([
|
||||
"LICENSE-Go.txt",
|
||||
"SHA256SUMS",
|
||||
"engine-metadata.json",
|
||||
"go-regex.wasm",
|
||||
"wasm_exec.mjs",
|
||||
]);
|
||||
const CHECKSUM_FILES = Object.freeze([
|
||||
"LICENSE-Go.txt",
|
||||
"engine-metadata.json",
|
||||
"go-regex.wasm",
|
||||
"wasm_exec.mjs",
|
||||
]);
|
||||
const SOURCE_FILES = Object.freeze([
|
||||
"engines/go/README.md",
|
||||
"engines/go/go.mod",
|
||||
"engines/go/main.go",
|
||||
]);
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a regular file: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
function commandLabel(command, arguments_) {
|
||||
return [command, ...arguments_]
|
||||
.map((part) =>
|
||||
/^[A-Za-z0-9_./:=,+-]+$/u.test(part) ? part : JSON.stringify(part),
|
||||
)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
const detail = [output, errorOutput].filter(Boolean).join("\n").trim();
|
||||
throw new Error(
|
||||
`${commandLabel(command, arguments_)} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}${detail ? `:\n${detail}` : "."}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
async function verifyToolchain(goRootDirectory) {
|
||||
const goRoot = await assertRealDirectory(
|
||||
goRootDirectory,
|
||||
"Go 1.26.5 toolchain",
|
||||
);
|
||||
const go = await assertRegularFile(path.join(goRoot, "bin", "go"), "go");
|
||||
await access(go, fsConstants.X_OK);
|
||||
const version = await run(go, ["version"]);
|
||||
if (version.stdout.trim() !== "go version go1.26.5 linux/amd64") {
|
||||
throw new Error("The Go pack requires official go1.26.5 linux/amd64.");
|
||||
}
|
||||
const license = await assertRegularFile(
|
||||
path.join(goRoot, "LICENSE"),
|
||||
"Go licence",
|
||||
);
|
||||
const wasmExec = await assertRegularFile(
|
||||
path.join(goRoot, "lib", "wasm", "wasm_exec.js"),
|
||||
"Go wasm_exec.js",
|
||||
);
|
||||
if ((await sha256File(license)) !== GO_LOCK.toolchain.licenseSha256) {
|
||||
throw new Error("The Go toolchain licence SHA-256 is not pinned go1.26.5.");
|
||||
}
|
||||
if ((await sha256File(wasmExec)) !== GO_LOCK.toolchain.wasmExecSha256) {
|
||||
throw new Error("wasm_exec.js does not match official go1.26.5.");
|
||||
}
|
||||
return { goRoot, go, license, wasmExec };
|
||||
}
|
||||
|
||||
function buildEnvironment(goRoot) {
|
||||
const environment = { ...process.env };
|
||||
for (const variable of [
|
||||
"GOFLAGS",
|
||||
"GOENV",
|
||||
"GOEXPERIMENT",
|
||||
"GOMOD",
|
||||
"GOTOOLDIR",
|
||||
"GOVERSION",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.CGO_ENABLED = "0";
|
||||
environment.GOARCH = "wasm";
|
||||
environment.GOENV = "off";
|
||||
environment.GOOS = "js";
|
||||
environment.GOROOT = goRoot;
|
||||
environment.GOTOOLCHAIN = "local";
|
||||
environment.GOWORK = "off";
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.SOURCE_DATE_EPOCH = String(GO_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceMetadata(root) {
|
||||
return Promise.all(
|
||||
SOURCE_FILES.map(async (relativePath) => {
|
||||
const contents = await readFile(path.join(root, relativePath));
|
||||
return {
|
||||
path: relativePath,
|
||||
sha256: sha256(contents),
|
||||
bytes: contents.byteLength,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fileMetadata(pack, name) {
|
||||
const contents = await readFile(path.join(pack, name));
|
||||
return { path: name, sha256: sha256(contents), bytes: contents.byteLength };
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "go",
|
||||
engineName: "Go standard-library regexp",
|
||||
engineVersion: GO_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
"Go 1.26.5 standard-library regexp (RE2 syntax), official js/wasm runtime",
|
||||
status: "production-native-runtime",
|
||||
bridge: {
|
||||
abiVersion: GO_LOCK.bridgeAbi,
|
||||
offsetUnit: "utf8-byte",
|
||||
requestEncoding:
|
||||
"Exact UTF-8-bounded JSON values with worst-case escape allowance; fixed callbacks; base64 UTF-8 replacement output; no source evaluation",
|
||||
supportedApplicationFlags: ["g", "i", "m", "s", "U"],
|
||||
replacement:
|
||||
"Bounded streaming expansion with regexp.ExpandString grammar, parity-checked against the native API.",
|
||||
limits: { ...GO_LOCK.limits },
|
||||
sourceFiles: await sourceMetadata(root),
|
||||
},
|
||||
source: {
|
||||
repository: GO_LOCK.source.repository,
|
||||
tag: GO_LOCK.source.tag,
|
||||
commitSha1: GO_LOCK.source.commit,
|
||||
treeSha1: GO_LOCK.source.tree,
|
||||
license: GO_LOCK.source.license,
|
||||
},
|
||||
toolchain: {
|
||||
version: GO_LOCK.runtimeVersion,
|
||||
host: "linux/amd64",
|
||||
target: "js/wasm",
|
||||
archive: GO_LOCK.toolchain.archive,
|
||||
archiveSha256: GO_LOCK.toolchain.archiveSha256,
|
||||
wasmExecSha256: GO_LOCK.toolchain.wasmExecSha256,
|
||||
build:
|
||||
"CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -buildvcs=false -ldflags=-s,-w,-buildid=",
|
||||
},
|
||||
sourceDateEpoch: GO_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(
|
||||
["LICENSE-Go.txt", "go-regex.wasm", "wasm_exec.mjs"].map((name) =>
|
||||
fileMetadata(pack, name),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function checksumText(pack) {
|
||||
const lines = await Promise.all(
|
||||
CHECKSUM_FILES.map(
|
||||
async (name) => `${await sha256File(path.join(pack, name))} ${name}`,
|
||||
),
|
||||
);
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function equalJson(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
|
||||
async function smokePack(pack) {
|
||||
const runtimeUrl = new URL(`file://${path.join(pack, "wasm_exec.mjs")}`).href;
|
||||
const wasmPath = path.join(pack, "go-regex.wasm");
|
||||
const smoke = `
|
||||
import { readFile } from "node:fs/promises";
|
||||
await import(process.argv[1]);
|
||||
let install;
|
||||
const ready = new Promise((resolve) => { install = resolve; });
|
||||
globalThis.__regexToolsInstallGoBridge = (identity, run) =>
|
||||
install({ identity, run });
|
||||
const runtime = new globalThis.Go();
|
||||
const module = await WebAssembly.instantiate(
|
||||
await readFile(process.argv[2]),
|
||||
runtime.importObject,
|
||||
);
|
||||
void runtime.run(module.instance);
|
||||
const bridge = await ready;
|
||||
const identity = JSON.parse(bridge.identity());
|
||||
if (
|
||||
identity.abi !== 1 ||
|
||||
identity.ok !== true ||
|
||||
identity.identity.goVersion !== "go1.26.5" ||
|
||||
identity.identity.selfTest !== true
|
||||
) throw new Error("Go identity self-test failed.");
|
||||
const result = JSON.parse(bridge.run(JSON.stringify({
|
||||
abi: 1,
|
||||
operation: "replace",
|
||||
pattern: "(?P<word>😀+)",
|
||||
flags: "g",
|
||||
options: {},
|
||||
subject: "x😀😀",
|
||||
scanAll: false,
|
||||
maximumMatches: 10,
|
||||
maximumCaptureRows: 10,
|
||||
replacement: "\${word}-$$",
|
||||
maximumOutputBytes: 100,
|
||||
})));
|
||||
if (
|
||||
result.ok !== true ||
|
||||
result.matches[0].span[0] !== 1 ||
|
||||
result.matches[0].span[1] !== 9 ||
|
||||
Buffer.from(result.outputBase64, "base64").toString("utf8") !== "x😀😀-$"
|
||||
) throw new Error("Go execution self-test failed.");
|
||||
process.exit(0);
|
||||
`;
|
||||
await run(process.execPath, [
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
smoke,
|
||||
runtimeUrl,
|
||||
wasmPath,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function verifyGoPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const pack = await assertRealDirectory(packDirectory, "Go engine pack");
|
||||
const entries = (await readdir(pack, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== PACK_FILES.length ||
|
||||
entries.some(
|
||||
(entry, index) =>
|
||||
entry.name !== PACK_FILES[index] ||
|
||||
!entry.isFile() ||
|
||||
entry.isSymbolicLink(),
|
||||
)
|
||||
) {
|
||||
throw new Error(`The Go pack must contain only: ${PACK_FILES.join(", ")}.`);
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "LICENSE-Go.txt"))) !==
|
||||
GO_LOCK.toolchain.licenseSha256
|
||||
) {
|
||||
throw new Error("The staged Go licence does not match go1.26.5.");
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "wasm_exec.mjs"))) !==
|
||||
GO_LOCK.toolchain.wasmExecSha256
|
||||
) {
|
||||
throw new Error("The staged wasm_exec runtime does not match go1.26.5.");
|
||||
}
|
||||
const wasm = await readFile(path.join(pack, "go-regex.wasm"));
|
||||
if (
|
||||
wasm.byteLength < 100_000 ||
|
||||
wasm.subarray(0, 4).toString("hex") !== "0061736d"
|
||||
) {
|
||||
throw new Error("The staged Go WebAssembly module is invalid.");
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("Go engine-metadata.json is invalid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (!equalJson(metadata, await expectedMetadata(repositoryRoot, pack))) {
|
||||
throw new Error(
|
||||
"The Go metadata does not match the pinned build contract.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
|
||||
(await checksumText(pack))
|
||||
) {
|
||||
throw new Error("The Go SHA256SUMS file is incorrect.");
|
||||
}
|
||||
await smokePack(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function replaceDirectory(stage, target, label) {
|
||||
const details = await lstat(target).catch(() => null);
|
||||
if (details && (!details.isDirectory() || details.isSymbolicLink())) {
|
||||
throw new Error(`${label} must be a real directory.`);
|
||||
}
|
||||
if (!details) {
|
||||
await rename(stage, target);
|
||||
return;
|
||||
}
|
||||
const backup = `${target}.replaced-${process.pid}`;
|
||||
await rename(target, backup);
|
||||
try {
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rename(backup, target);
|
||||
throw error;
|
||||
}
|
||||
await rm(backup, { recursive: true });
|
||||
}
|
||||
|
||||
export async function buildGoPack(root, goRootDirectory, outputDirectory) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const engine = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "engines", "go"),
|
||||
"Go engine source",
|
||||
);
|
||||
const toolchain = await verifyToolchain(goRootDirectory);
|
||||
const environment = buildEnvironment(toolchain.goRoot);
|
||||
const buildParent = path.join(repositoryRoot, ".engine-build");
|
||||
await mkdir(buildParent, { recursive: true });
|
||||
const temporary = await mkdtemp(path.join(buildParent, "go-build-"));
|
||||
const first = path.join(temporary, "first.wasm");
|
||||
const second = path.join(temporary, "second.wasm");
|
||||
const buildArguments = [
|
||||
"build",
|
||||
"-trimpath",
|
||||
"-buildvcs=false",
|
||||
"-ldflags=-s -w -buildid=",
|
||||
];
|
||||
try {
|
||||
await run(toolchain.go, [...buildArguments, "-o", first, "."], {
|
||||
cwd: engine,
|
||||
env: environment,
|
||||
});
|
||||
await run(toolchain.go, [...buildArguments, "-o", second, "."], {
|
||||
cwd: engine,
|
||||
env: environment,
|
||||
});
|
||||
if ((await sha256File(first)) !== (await sha256File(second))) {
|
||||
throw new Error("Two pinned Go builds produced different module hashes.");
|
||||
}
|
||||
|
||||
const target = path.resolve(
|
||||
repositoryRoot,
|
||||
outputDirectory ?? path.join(".engine-build", "go"),
|
||||
);
|
||||
const stage = await mkdtemp(path.join(buildParent, "go-pack-"));
|
||||
await copyFile(toolchain.license, path.join(stage, "LICENSE-Go.txt"));
|
||||
await copyFile(first, path.join(stage, "go-regex.wasm"));
|
||||
await copyFile(toolchain.wasmExec, path.join(stage, "wasm_exec.mjs"));
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(stage, "SHA256SUMS"), await checksumText(stage));
|
||||
await verifyGoPack(stage, repositoryRoot);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
await replaceDirectory(stage, target, "Go staged pack");
|
||||
return { metadata, output: target };
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function installGoPack(sourceDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const source = await assertRealDirectory(sourceDirectory, "Go staged pack");
|
||||
const metadata = await verifyGoPack(source, repositoryRoot);
|
||||
const engineParent = path.join(repositoryRoot, "public", "engines");
|
||||
await mkdir(engineParent, { recursive: true });
|
||||
const stage = await mkdtemp(path.join(engineParent, "go-install-"));
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(path.join(source, name), path.join(stage, name));
|
||||
}
|
||||
await verifyGoPack(stage, repositoryRoot);
|
||||
const output = path.join(engineParent, "go");
|
||||
await replaceDirectory(stage, output, "Installed Go engine pack");
|
||||
return { metadata, output };
|
||||
}
|
||||
14
scripts/install-cpp-engine.mjs
Normal file
14
scripts/install-cpp-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installCppPack } from "./cpp-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-cpp-engine.mjs [.engine-build/cpp]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/cpp");
|
||||
const result = await installCppPack(source, root);
|
||||
console.log(`Installed verified C++ engine pack at ${result.output}.`);
|
||||
14
scripts/install-dotnet-engine.mjs
Normal file
14
scripts/install-dotnet-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installDotNetPack } from "./dotnet-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-dotnet-engine.mjs [.engine-build/dotnet]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/dotnet");
|
||||
const result = await installDotNetPack(source, root);
|
||||
console.log(`Installed verified .NET engine pack at ${result.output}.`);
|
||||
14
scripts/install-go-engine.mjs
Normal file
14
scripts/install-go-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installGoPack } from "./go-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-go-engine.mjs [.engine-build/go]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/go");
|
||||
const result = await installGoPack(source, root);
|
||||
console.log(`Installed verified Go engine assets at ${result.output}.`);
|
||||
14
scripts/install-perl-engine.mjs
Normal file
14
scripts/install-perl-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installPerlPack } from "./perl-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-perl-engine.mjs [.engine-build/perl]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/perl");
|
||||
const result = await installPerlPack(source, root);
|
||||
console.log(`Installed verified Perl engine pack at ${result.output}.`);
|
||||
16
scripts/install-php-engine.mjs
Normal file
16
scripts/install-php-engine.mjs
Normal file
@@ -0,0 +1,16 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installPhpPack } from "./php-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-php-engine.mjs [.engine-build/php]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/php");
|
||||
const result = await installPhpPack(source, root);
|
||||
console.log(
|
||||
`Installed verified ${result.metadata.engineVersion} assets at ${result.output}.`,
|
||||
);
|
||||
16
scripts/install-ruby-engine.mjs
Normal file
16
scripts/install-ruby-engine.mjs
Normal file
@@ -0,0 +1,16 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installRubyPack } from "./ruby-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-ruby-engine.mjs [.engine-build/ruby]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/ruby");
|
||||
const result = await installRubyPack(source, root);
|
||||
console.log(
|
||||
`Installed verified ${result.metadata.engineVersion} assets at ${result.output}.`,
|
||||
);
|
||||
14
scripts/install-rust-engine.mjs
Normal file
14
scripts/install-rust-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installRustPack } from "./rust-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-rust-engine.mjs [.engine-build/rust]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/rust");
|
||||
const result = await installRustPack(source, root);
|
||||
console.log(`Installed verified Rust engine assets at ${result.output}.`);
|
||||
@@ -41,6 +41,8 @@ export const JAVA_LOCK = Object.freeze({
|
||||
"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
|
||||
noticeSha256:
|
||||
"6c2b3372a6beee7fbaad482248ac0f5ad2b099d92ff2a528c04261cc9c0636f4",
|
||||
packagedNoticeSha256:
|
||||
"71bbe79ff78d0e5e64096acdd418589abfe886b8f827048dc655df438a57e477",
|
||||
}),
|
||||
toolchain: Object.freeze({
|
||||
mavenVersion: "3.9.15",
|
||||
@@ -282,6 +284,9 @@ async function expectedMetadata(root, pack) {
|
||||
license: JAVA_LOCK.source.license,
|
||||
upstreamLicenseSha256: JAVA_LOCK.source.licenseSha256,
|
||||
upstreamNoticeSha256: JAVA_LOCK.source.noticeSha256,
|
||||
packagedNoticeSha256: JAVA_LOCK.source.packagedNoticeSha256,
|
||||
noticeNormalization:
|
||||
"The packaged notice removes one redundant blank line and adds a final LF; its attribution text is unchanged.",
|
||||
},
|
||||
toolchain: {
|
||||
mavenVersion: JAVA_LOCK.toolchain.mavenVersion,
|
||||
@@ -418,6 +423,14 @@ export async function verifyJavaPack(packDirectory, root) {
|
||||
) {
|
||||
throw new Error("The staged TeaVM licence does not match tag 0.15.0.");
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "NOTICE.txt"))) !==
|
||||
JAVA_LOCK.source.packagedNoticeSha256
|
||||
) {
|
||||
throw new Error(
|
||||
"The staged TeaVM notice does not match the documented normalized notice.",
|
||||
);
|
||||
}
|
||||
const moduleText = await readFile(path.join(pack, "java-regex.mjs"), "utf8");
|
||||
if (
|
||||
moduleText.includes("sourceMappingURL") ||
|
||||
|
||||
@@ -13,12 +13,13 @@ import { createRequire } from "node:module";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { zipSync } from "fflate";
|
||||
import { checksumLine, sha256 } from "./checksum-release.mjs";
|
||||
import { assertNoUnexpectedLocalPaths } from "./release-path-hygiene.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const zipEpoch = new Date(1980, 0, 1, 0, 0, 0);
|
||||
const gplVersion3TextSha256 =
|
||||
"fb981668c18a279e285fc4d83fba1e836cc84dd4daa73c9697d3cfd2d8aca6e0";
|
||||
const expectedApplicationVersion = "0.3.0";
|
||||
const expectedApplicationVersion = "0.4.0";
|
||||
const requiredRootFiles = [
|
||||
"LICENSE",
|
||||
"README.md",
|
||||
@@ -268,7 +269,18 @@ function scanEntry(entry) {
|
||||
) {
|
||||
throw new Error(`Credential-like release path: ${entry.name}`);
|
||||
}
|
||||
if (!/\.(?:css|html|js|json|md|svg|txt)$/iu.test(entry.name)) return;
|
||||
const knownVirtualHomes =
|
||||
/^(?:assets\/php\.worker-[A-Za-z0-9_-]+\.js|engines\/cpp\/cpp-regex\.mjs|engines\/dotnet\/_framework\/dotnet\.native\.[a-z0-9]+\.js|engines\/perl\/emperl\.js|engines\/php\/engine-metadata\.json|engines\/php\/php-loader\.mjs|engines\/python\/pyodide(?:\.asm)?\.mjs)$/u.test(
|
||||
entry.name,
|
||||
)
|
||||
? ["/home/web_user", "/home/pyodide"]
|
||||
: [];
|
||||
assertNoUnexpectedLocalPaths(entry.data, entry.name, {
|
||||
allowedVirtualHomes: knownVirtualHomes,
|
||||
});
|
||||
if (!/\.(?:css|html|js|json|md|mjs|pl|rst|svg|txt)$/iu.test(entry.name)) {
|
||||
return;
|
||||
}
|
||||
const text = entry.data.toString("utf8");
|
||||
if (/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/u.test(text)) {
|
||||
throw new Error(`Private key material detected in ${entry.name}`);
|
||||
@@ -276,13 +288,6 @@ function scanEntry(entry) {
|
||||
if (/https?:\/\/[^/@\s]+:[^/@\s]+@/iu.test(text)) {
|
||||
throw new Error(`Credential-bearing URL detected in ${entry.name}`);
|
||||
}
|
||||
if (
|
||||
/(?:^|[\s"'`])(?:\/home\/|\/mnt\/|\/Users\/|[A-Za-z]:\\Users\\)/mu.test(
|
||||
text,
|
||||
)
|
||||
) {
|
||||
throw new Error(`Local absolute path detected in ${entry.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function createDeterministicZip(inputEntries) {
|
||||
@@ -392,20 +397,138 @@ async function collectReleaseEntries() {
|
||||
"engines/java/SHA256SUMS",
|
||||
"engines/java/engine-metadata.json",
|
||||
"engines/java/java-regex.mjs",
|
||||
"engines/cpp/LICENSE-Emscripten.txt",
|
||||
"engines/cpp/LICENSE-compiler-rt.txt",
|
||||
"engines/cpp/LICENSE-libcxx.txt",
|
||||
"engines/cpp/LICENSE-libcxxabi.txt",
|
||||
"engines/cpp/LICENSE-libunwind.txt",
|
||||
"engines/cpp/LICENSE-musl.txt",
|
||||
"engines/cpp/SHA256SUMS",
|
||||
"engines/cpp/SOURCE-MANIFEST.json",
|
||||
"engines/cpp/cpp-regex.mjs",
|
||||
"engines/cpp/cpp-regex.wasm",
|
||||
"engines/cpp/engine-metadata.json",
|
||||
"engines/dotnet/LICENSE-dotnet.txt",
|
||||
"engines/dotnet/SHA256SUMS",
|
||||
"engines/dotnet/THIRD-PARTY-NOTICES.txt",
|
||||
"engines/dotnet/_framework/dotnet.js",
|
||||
"engines/dotnet/engine-metadata.json",
|
||||
"engines/go/LICENSE-Go.txt",
|
||||
"engines/go/SHA256SUMS",
|
||||
"engines/go/engine-metadata.json",
|
||||
"engines/go/go-regex.wasm",
|
||||
"engines/go/wasm_exec.mjs",
|
||||
"engines/perl/LICENSE-WebPerl-Artistic.txt",
|
||||
"engines/perl/LICENSE-WebPerl-GPL.txt",
|
||||
"engines/perl/LICENSE-Devel-StackTrace-Artistic-2.0.txt",
|
||||
"engines/perl/LICENSE-Emscripten.txt",
|
||||
"engines/perl/NOTICE.txt",
|
||||
"engines/perl/SHA256SUMS",
|
||||
"engines/perl/emperl.data",
|
||||
"engines/perl/emperl.js",
|
||||
"engines/perl/emperl.wasm",
|
||||
"engines/perl/engine-metadata.json",
|
||||
"engines/perl/perl-runtime.worker.js",
|
||||
"engines/perl/regex_tools_bridge.pl",
|
||||
"engines/php/LICENSE.Emscripten-4.0.19.txt",
|
||||
"engines/php/LICENSE.Oniguruma-6.9.10.txt",
|
||||
"engines/php/LICENSE.OpenSSL-1.1.1t.txt",
|
||||
"engines/php/LICENSE.PCRE2-10.44.txt",
|
||||
"engines/php/LICENSE.PHP-4.0.txt",
|
||||
"engines/php/LICENSE.PHP-CLI-http-parser-MIT.txt",
|
||||
"engines/php/LICENSE.PHP-Lexbor-Apache-2.0.txt",
|
||||
"engines/php/LICENSE.PHP-Zend-2.0.txt",
|
||||
"engines/php/LICENSE.PHP-bcmath-LGPL-2.1.txt",
|
||||
"engines/php/LICENSE.PHP-libavifinfo-BSD-2-Clause.txt",
|
||||
"engines/php/LICENSE.PHP-libmagic-BSD.txt",
|
||||
"engines/php/LICENSE.PHP-libmbfl-LGPL-2.1.txt",
|
||||
"engines/php/LICENSE.PHP-timelib-MIT.txt",
|
||||
"engines/php/LICENSE.PHP-uriparser-BSD-3-Clause.txt",
|
||||
"engines/php/LICENSE.curl-7.69.1.txt",
|
||||
"engines/php/LICENSE.ini-ISC.txt",
|
||||
"engines/php/LICENSE.libaom-3.12.1.txt",
|
||||
"engines/php/LICENSE.libavif-1.3.0.txt",
|
||||
"engines/php/LICENSE.libcxx.txt",
|
||||
"engines/php/LICENSE.libcxxabi.txt",
|
||||
"engines/php/LICENSE.libgd-2.3.3.txt",
|
||||
"engines/php/LICENSE.libiconv-1.17-LGPL-2.1.txt",
|
||||
"engines/php/LICENSE.libjpeg-turbo-3.0.3.txt",
|
||||
"engines/php/LICENSE.libpng-1.6.39.txt",
|
||||
"engines/php/LICENSE.libwebp.txt",
|
||||
"engines/php/LICENSE.libxml2-2.9.10.txt",
|
||||
"engines/php/LICENSE.libyuv.txt",
|
||||
"engines/php/LICENSE.libzip-1.9.2.txt",
|
||||
"engines/php/LICENSE.llvm-compiler-rt.txt",
|
||||
"engines/php/LICENSE.musl.txt",
|
||||
"engines/php/LICENSE.php-wasm-GPL-2.0-or-later.txt",
|
||||
"engines/php/LICENSE.wasm-feature-detect-Apache-2.0.txt",
|
||||
"engines/php/LICENSE.zlib-1.2.13.txt",
|
||||
"engines/php/NOTICE.IANA-TZDATA-2026.1.txt",
|
||||
"engines/php/NOTICE.PHP-Lexbor.txt",
|
||||
"engines/php/NOTICE.PHP-REDIST-BINS.txt",
|
||||
"engines/php/NOTICE.PHP-public-domain-hash-code.txt",
|
||||
"engines/php/NOTICE.SQLite-3.51.0.txt",
|
||||
"engines/php/NOTICE.dlmalloc-2.8.6.txt",
|
||||
"engines/php/PATENTS.PHP-libavifinfo.txt",
|
||||
"engines/php/PATENTS.libaom-3.12.1.txt",
|
||||
"engines/php/PATENTS.libwebp.txt",
|
||||
"engines/php/SHA256SUMS",
|
||||
"engines/php/engine-metadata.json",
|
||||
"engines/php/native-components.json",
|
||||
"engines/php/php-loader.mjs",
|
||||
"engines/php/php.wasm",
|
||||
"engines/pcre2/LICENSE.txt",
|
||||
"engines/pcre2/SHA256SUMS",
|
||||
"engines/pcre2/engine-metadata.json",
|
||||
"engines/pcre2/pcre2.mjs",
|
||||
"engines/pcre2/pcre2.wasm",
|
||||
"engines/python/LICENSE.cpython.txt",
|
||||
"engines/python/LICENSE.bzip2.txt",
|
||||
"engines/python/LICENSE.emscripten-mini-lz4.txt",
|
||||
"engines/python/LICENSE.emscripten.txt",
|
||||
"engines/python/LICENSE.expat.txt",
|
||||
"engines/python/LICENSE.hacl-mit.txt",
|
||||
"engines/python/LICENSE.hiwire.txt",
|
||||
"engines/python/LICENSE.libffi.txt",
|
||||
"engines/python/LICENSE.libmpdec.txt",
|
||||
"engines/python/LICENSE.llvm-compiler-rt.txt",
|
||||
"engines/python/LICENSE.llvm-libcxx.txt",
|
||||
"engines/python/LICENSE.llvm-libcxxabi.txt",
|
||||
"engines/python/LICENSE.llvm-libunwind.txt",
|
||||
"engines/python/LICENSE.pyodide.txt",
|
||||
"engines/python/LICENSE.pyodide-stacktrace-vendors.txt",
|
||||
"engines/python/LICENSE.sqlite.txt",
|
||||
"engines/python/LICENSE.xz.txt",
|
||||
"engines/python/LICENSE.zlib.txt",
|
||||
"engines/python/LICENSE.zstd.txt",
|
||||
"engines/python/NOTICE.cpython-bundled.rst",
|
||||
"engines/python/NOTICE.emscripten-dlmalloc.txt",
|
||||
"engines/python/NOTICE.emscripten-musl.txt",
|
||||
"engines/python/SHA256SUMS",
|
||||
"engines/python/SOURCE-MANIFEST.json",
|
||||
"engines/python/engine-metadata.json",
|
||||
"engines/python/pyodide-lock.json",
|
||||
"engines/python/pyodide.asm.mjs",
|
||||
"engines/python/pyodide.asm.wasm",
|
||||
"engines/python/pyodide.mjs",
|
||||
"engines/python/python_stdlib.zip",
|
||||
"engines/ruby/LICENSE.browser-wasi-shim-Apache-2.0.txt",
|
||||
"engines/ruby/LICENSE.browser-wasi-shim-MIT.txt",
|
||||
"engines/ruby/LICENSE.ruby-wasm.txt",
|
||||
"engines/ruby/LICENSE.tslib.txt",
|
||||
"engines/ruby/NOTICE.txt",
|
||||
"engines/ruby/SHA256SUMS",
|
||||
"engines/ruby/engine-metadata.json",
|
||||
"engines/ruby/ruby.wasm",
|
||||
"engines/rust/LICENSE-Apache-2.0.txt",
|
||||
"engines/rust/LICENSE-MIT.txt",
|
||||
"engines/rust/LICENSE-Unicode-3.0.txt",
|
||||
"engines/rust/RUST-STDLIB-COPYRIGHT.html",
|
||||
"engines/rust/SHA256SUMS",
|
||||
"engines/rust/THIRD_PARTY_LICENSES.txt",
|
||||
"engines/rust/engine-metadata.json",
|
||||
"engines/rust/rust-regex.mjs",
|
||||
"engines/rust/rust-regex_bg.wasm",
|
||||
"LICENSES/Emscripten-MIT.txt",
|
||||
"LICENSES/PCRE2.txt",
|
||||
"LICENSES/build/rolldown-1.1.5/LICENSE",
|
||||
@@ -418,6 +541,18 @@ async function collectReleaseEntries() {
|
||||
if (![...names].some((name) => name.startsWith("assets/"))) {
|
||||
throw new Error("Release has no production assets.");
|
||||
}
|
||||
for (const requiredDotNetPattern of [
|
||||
/^engines\/dotnet\/_framework\/RegexTools\.DotNet\..+\.wasm$/u,
|
||||
/^engines\/dotnet\/_framework\/System\.Text\.RegularExpressions\..+\.wasm$/u,
|
||||
/^engines\/dotnet\/_framework\/dotnet\.native\..+\.wasm$/u,
|
||||
/^engines\/dotnet\/_framework\/dotnet\.runtime\..+\.js$/u,
|
||||
]) {
|
||||
if (![...names].some((name) => requiredDotNetPattern.test(name))) {
|
||||
throw new Error(
|
||||
`Required .NET runtime asset is missing: ${requiredDotNetPattern}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (![...names].some((name) => name.startsWith("LICENSES/"))) {
|
||||
throw new Error("Release has no third-party licence directory.");
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createDeterministicZip,
|
||||
@@ -41,6 +43,45 @@ describe("release packaging", () => {
|
||||
).toThrow("Credential-like");
|
||||
});
|
||||
|
||||
it("scans reStructuredText legal notices for local build paths", () => {
|
||||
expect(() =>
|
||||
createDeterministicZip([
|
||||
{
|
||||
name: "engines/python/NOTICE.cpython-bundled.rst",
|
||||
data: Buffer.from("Built from /home/example/private/source"),
|
||||
},
|
||||
]),
|
||||
).toThrow("Local absolute path");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["assets/leak.js", Buffer.from('const source = "/mnt/private/app.ts";')],
|
||||
[
|
||||
"engines/example/module.wasm",
|
||||
Buffer.from("/home/private/build/runtime.c"),
|
||||
],
|
||||
[
|
||||
"engines/example/module.pdb",
|
||||
Buffer.from("c:\\Users\\private\\source\\module.cs", "utf16le"),
|
||||
],
|
||||
])("scans JS, WebAssembly and PDB-like asset %s", (name, data) => {
|
||||
expect(() => createDeterministicZip([{ name, data }])).toThrow(
|
||||
"Local absolute path",
|
||||
);
|
||||
});
|
||||
|
||||
it("allows only the byte-exact upstream Perl and Ruby path-bearing assets", async () => {
|
||||
for (const name of ["engines/perl/emperl.data", "engines/ruby/ruby.wasm"]) {
|
||||
const data = await readFile(path.resolve("public", name));
|
||||
expect(() => createDeterministicZip([{ name, data }])).not.toThrow();
|
||||
expect(() =>
|
||||
createDeterministicZip([
|
||||
{ name, data: Buffer.concat([data, Buffer.from("changed")]) },
|
||||
]),
|
||||
).toThrow("Local absolute path");
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("requires an explicit force flag for exact-version replacement", () => {
|
||||
expect(parseReleaseArguments([])).toEqual({ force: false });
|
||||
expect(parseReleaseArguments(["--force"])).toEqual({ force: true });
|
||||
|
||||
1027
scripts/perl-engine-pack.mjs
Normal file
1027
scripts/perl-engine-pack.mjs
Normal file
File diff suppressed because it is too large
Load Diff
170
scripts/perl-engine-pack.test.ts
Normal file
170
scripts/perl-engine-pack.test.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFile,
|
||||
cp,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildPerlPack,
|
||||
installPerlPack,
|
||||
PERL_ENGINE_LOCK,
|
||||
verifyPerlPack,
|
||||
} from "./perl-engine-pack.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
const installedPack = path.resolve("public", "engines", "perl");
|
||||
const packFiles = [
|
||||
"LICENSE-Devel-StackTrace-Artistic-2.0.txt",
|
||||
"LICENSE-Emscripten.txt",
|
||||
"LICENSE-WebPerl-Artistic.txt",
|
||||
"LICENSE-WebPerl-GPL.txt",
|
||||
"NOTICE.txt",
|
||||
"SHA256SUMS",
|
||||
"emperl.data",
|
||||
"emperl.js",
|
||||
"emperl.wasm",
|
||||
"engine-metadata.json",
|
||||
"perl-runtime.worker.js",
|
||||
"regex_tools_bridge.pl",
|
||||
] as const;
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
async function digest(file: string): Promise<string> {
|
||||
return createHash("sha256")
|
||||
.update(await readFile(file))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
async function makeExtractedPrebuilt(parent: string): Promise<string> {
|
||||
const prebuilt = path.join(parent, "webperl_prebuilt_v0.09-beta");
|
||||
await mkdir(prebuilt, { recursive: true });
|
||||
for (const name of ["emperl.data", "emperl.js", "emperl.wasm"]) {
|
||||
await copyFile(path.join(installedPack, name), path.join(prebuilt, name));
|
||||
}
|
||||
await copyFile(
|
||||
path.join(installedPack, "LICENSE-WebPerl-Artistic.txt"),
|
||||
path.join(prebuilt, "LICENSE_artistic.txt"),
|
||||
);
|
||||
await copyFile(
|
||||
path.join(installedPack, "LICENSE-WebPerl-GPL.txt"),
|
||||
path.join(prebuilt, "LICENSE_gpl.txt"),
|
||||
);
|
||||
return prebuilt;
|
||||
}
|
||||
|
||||
describe("Perl engine-pack gate", () => {
|
||||
it("verifies the installed closed WebPerl pack in a strict-CSP Chromium worker", async () => {
|
||||
await expect(verifyPerlPack(installedPack)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: "5.28.1",
|
||||
status: "legacy-beta-compatibility-runtime",
|
||||
source: expect.objectContaining({
|
||||
tag: "v0.09-beta",
|
||||
commitSha1: "6f2173d29a2c2e3536e1de75ff5d291ae96ab348",
|
||||
}),
|
||||
bridge: expect.objectContaining({
|
||||
replacementOutputTransport: expect.stringContaining(
|
||||
"fixed MEMFS binary file",
|
||||
),
|
||||
limits: expect.objectContaining({
|
||||
maximumResponseJsonBytes: 8_388_608,
|
||||
outputChunkCharacters: 4_096,
|
||||
}),
|
||||
}),
|
||||
securityAudit: expect.objectContaining({
|
||||
generatedLoaderJavaScriptEvalSites: 1,
|
||||
generatedLoaderNewFunctionSites: 0,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}, 60_000);
|
||||
|
||||
it("pins the exact archive, runtime assets, and legacy identity", () => {
|
||||
expect(PERL_ENGINE_LOCK.engineVersion).toBe("5.28.1");
|
||||
expect(PERL_ENGINE_LOCK.webPerlVersion).toBe("0.09-beta");
|
||||
expect(PERL_ENGINE_LOCK.prebuilt.archiveSha256).toBe(
|
||||
"5f441249217e90ab378c666f473d4206ab4f44907f6bb0aa8d70834bc38c40dc",
|
||||
);
|
||||
expect(PERL_ENGINE_LOCK.prebuilt.files["emperl.wasm"]).toEqual(
|
||||
expect.objectContaining({
|
||||
bytes: 3_734_063,
|
||||
sha256:
|
||||
"f1d49c4514c7332a57992c4a2444fd6a56ae3b5e6651b4fd484852a641e5e4ec",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("builds byte-identical offline packs and installs one atomically", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-perl-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const prebuilt = await makeExtractedPrebuilt(directory);
|
||||
const first = path.join(directory, "first");
|
||||
const second = path.join(directory, "second");
|
||||
|
||||
await buildPerlPack({ prebuiltDirectory: prebuilt, output: first });
|
||||
await buildPerlPack({ prebuiltDirectory: prebuilt, output: second });
|
||||
|
||||
for (const file of packFiles) {
|
||||
expect(await digest(path.join(first, file)), file).toBe(
|
||||
await digest(path.join(second, file)),
|
||||
);
|
||||
}
|
||||
|
||||
const fakeRoot = path.join(directory, "repository");
|
||||
await mkdir(path.join(fakeRoot, "engines"), { recursive: true });
|
||||
await cp(
|
||||
path.resolve("engines", "perl"),
|
||||
path.join(fakeRoot, "engines", "perl"),
|
||||
{ recursive: true },
|
||||
);
|
||||
const oldTarget = path.join(fakeRoot, "public", "engines", "perl");
|
||||
await mkdir(oldTarget, { recursive: true });
|
||||
await writeFile(path.join(oldTarget, "old-marker"), "old\n", "utf8");
|
||||
|
||||
const installed = await installPerlPack(first, fakeRoot);
|
||||
|
||||
expect(installed.output).toBe(oldTarget);
|
||||
await expect(
|
||||
readFile(path.join(oldTarget, "old-marker")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
verifyPerlPack(oldTarget, fakeRoot, { smoke: false }),
|
||||
).resolves.toMatchObject({ engineVersion: "5.28.1" });
|
||||
}, 120_000);
|
||||
|
||||
it("rejects extra pack entries and drifted extracted inputs", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-perl-reject-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "pack");
|
||||
await cp(installedPack, pack, { recursive: true });
|
||||
await writeFile(path.join(pack, "unexpected"), "not allowed\n", "utf8");
|
||||
await expect(
|
||||
verifyPerlPack(pack, path.resolve("."), { smoke: false }),
|
||||
).rejects.toThrow("must contain only");
|
||||
|
||||
const prebuilt = await makeExtractedPrebuilt(directory);
|
||||
await writeFile(path.join(prebuilt, "emperl.wasm"), "not wasm\n");
|
||||
await expect(
|
||||
buildPerlPack({
|
||||
prebuiltDirectory: prebuilt,
|
||||
output: path.join(directory, "rejected"),
|
||||
}),
|
||||
).rejects.toThrow("pinned WebPerl release");
|
||||
});
|
||||
});
|
||||
1120
scripts/php-engine-pack.mjs
Normal file
1120
scripts/php-engine-pack.mjs
Normal file
File diff suppressed because it is too large
Load Diff
128
scripts/php-engine-pack.test.ts
Normal file
128
scripts/php-engine-pack.test.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
appendFile,
|
||||
cp,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildPhpPack,
|
||||
PHP_ENGINE_LOCK,
|
||||
verifyPhpPack,
|
||||
} from "./php-engine-pack.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
async function digest(file: string): Promise<string> {
|
||||
return createHash("sha256")
|
||||
.update(await readFile(file))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
describe("PHP engine-pack gate", () => {
|
||||
it("verifies the installed Asyncify PHP runtime and native smoke cases", async () => {
|
||||
await expect(
|
||||
verifyPhpPack(path.resolve("public", "engines", "php")),
|
||||
).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: "PHP 8.5.8 preg / PCRE2 10.44",
|
||||
platform: "wasm32-emscripten",
|
||||
source: expect.objectContaining({
|
||||
runtimeVariant: "PHP 8.5 Asyncify web build (without intl extension)",
|
||||
nativeComponents: expect.objectContaining({
|
||||
linkedComponentCount: 32,
|
||||
excludedDetectionCount: 7,
|
||||
}),
|
||||
}),
|
||||
licensing: expect.objectContaining({
|
||||
nativeInventory: expect.objectContaining({
|
||||
independentlyPinnedLegalAssets: 42,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("pins exact package, loader, and WebAssembly identities", () => {
|
||||
expect(PHP_ENGINE_LOCK.packageVersion).toBe("3.1.46");
|
||||
expect(PHP_ENGINE_LOCK.phpVersion).toBe("8.5.8");
|
||||
expect(PHP_ENGINE_LOCK.pcreVersion).toBe("10.44");
|
||||
expect(PHP_ENGINE_LOCK.asyncMode).toBe("asyncify");
|
||||
expect(PHP_ENGINE_LOCK.npm.web.integrity).toMatch(/^sha512-/u);
|
||||
expect(PHP_ENGINE_LOCK.npm.universal.integrity).toMatch(/^sha512-/u);
|
||||
expect(PHP_ENGINE_LOCK.sourceAssets.wasm).toHaveLength(64);
|
||||
expect(PHP_ENGINE_LOCK.sourceAssets.nativeInventory).toHaveLength(64);
|
||||
expect(Object.keys(PHP_ENGINE_LOCK.legalAssets)).toHaveLength(42);
|
||||
});
|
||||
|
||||
it("builds byte-identical closed packs from pinned npm inputs", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-php-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const first = path.join(directory, "first");
|
||||
const second = path.join(directory, "second");
|
||||
|
||||
await buildPhpPack({ output: first });
|
||||
await buildPhpPack({ output: second });
|
||||
|
||||
for (const file of [
|
||||
"php.wasm",
|
||||
"php-loader.mjs",
|
||||
"engine-metadata.json",
|
||||
"native-components.json",
|
||||
"SHA256SUMS",
|
||||
"LICENSE.Emscripten-4.0.19.txt",
|
||||
"LICENSE.llvm-compiler-rt.txt",
|
||||
"LICENSE.PHP-4.0.txt",
|
||||
"LICENSE.PHP-Zend-2.0.txt",
|
||||
"LICENSE.PCRE2-10.44.txt",
|
||||
"LICENSE.php-wasm-GPL-2.0-or-later.txt",
|
||||
"NOTICE.PHP-REDIST-BINS.txt",
|
||||
"PATENTS.libaom-3.12.1.txt",
|
||||
]) {
|
||||
expect(await digest(path.join(first, file))).toBe(
|
||||
await digest(path.join(second, file)),
|
||||
);
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("anchors legal assets independently from the pack checksum manifest", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-php-legal-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "pack");
|
||||
await cp(path.resolve("public", "engines", "php"), pack, {
|
||||
recursive: true,
|
||||
force: false,
|
||||
errorOnExist: true,
|
||||
});
|
||||
const legalName = "LICENSE.OpenSSL-1.1.1t.txt";
|
||||
const legalFile = path.join(pack, legalName);
|
||||
await appendFile(legalFile, "\n");
|
||||
const checksumsFile = path.join(pack, "SHA256SUMS");
|
||||
const checksums = await readFile(checksumsFile, "utf8");
|
||||
const updated = checksums.replace(
|
||||
new RegExp(`^[a-f0-9]{64} ${legalName}$`, "mu"),
|
||||
`${await digest(legalFile)} ${legalName}`,
|
||||
);
|
||||
expect(updated).not.toBe(checksums);
|
||||
await writeFile(checksumsFile, updated);
|
||||
|
||||
await expect(verifyPhpPack(pack)).rejects.toThrow(
|
||||
`Packed PHP legal asset ${legalName} SHA-256 mismatch`,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import {
|
||||
createPythonSourceManifest,
|
||||
PYTHON_LEGAL_RESOURCES,
|
||||
transformPythonLegalResource,
|
||||
} from "./python-engine-provenance.mjs";
|
||||
|
||||
export const PYTHON_ENGINE_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
@@ -77,9 +82,9 @@ export const PYTHON_ENGINE_LOCK = Object.freeze({
|
||||
|
||||
const PACK_FILES = Object.freeze(
|
||||
[
|
||||
"LICENSE.cpython.txt",
|
||||
"LICENSE.pyodide.txt",
|
||||
...PYTHON_LEGAL_RESOURCES.map(({ output }) => output),
|
||||
"SHA256SUMS",
|
||||
"SOURCE-MANIFEST.json",
|
||||
"engine-metadata.json",
|
||||
...Object.keys(PYTHON_ENGINE_LOCK.assets),
|
||||
].sort(),
|
||||
@@ -203,9 +208,18 @@ async function fileRecord(directory, name) {
|
||||
};
|
||||
}
|
||||
|
||||
async function expectedSourceManifest(pack) {
|
||||
const legalFiles = [];
|
||||
for (const { output } of PYTHON_LEGAL_RESOURCES) {
|
||||
legalFiles.push(await fileRecord(pack, output));
|
||||
}
|
||||
return createPythonSourceManifest(legalFiles);
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
const bridgePath = path.join(root, "engines", "python", "bridge.py");
|
||||
const bridge = await readFile(bridgePath);
|
||||
const sourceManifest = await fileRecord(pack, "SOURCE-MANIFEST.json");
|
||||
const files = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
if (name !== "engine-metadata.json") {
|
||||
@@ -229,6 +243,7 @@ async function expectedMetadata(root, pack) {
|
||||
regexModule: "re",
|
||||
supportModules: {
|
||||
serialization: "json",
|
||||
replacementOutputEncoding: "canonical RFC 4648 base64",
|
||||
runtimeIdentity: "sys",
|
||||
},
|
||||
maximumPatternUtf16: PYTHON_ENGINE_LOCK.maximumPatternUtf16,
|
||||
@@ -242,6 +257,7 @@ async function expectedMetadata(root, pack) {
|
||||
PYTHON_ENGINE_LOCK.maximumNativeExpansionCodePoints,
|
||||
},
|
||||
source: {
|
||||
manifest: sourceManifest,
|
||||
pyodide: {
|
||||
repository: PYTHON_ENGINE_LOCK.pyodide.repository,
|
||||
tag: PYTHON_ENGINE_LOCK.pyodide.tag,
|
||||
@@ -273,6 +289,8 @@ async function expectedMetadata(root, pack) {
|
||||
upstreamLoaderSha256: PYTHON_ENGINE_LOCK.assets["pyodide.mjs"],
|
||||
packedLoaderSha256: PYTHON_ENGINE_LOCK.strippedLoaderSha256,
|
||||
externalPackagesIncluded: false,
|
||||
legalClosure:
|
||||
"SOURCE-MANIFEST.json inventories every loader and native base-runtime component, exact preferred-form input and patch, plus independently anchored legal text.",
|
||||
lockPythonVersionNote:
|
||||
"The upstream lock's 3.14.0 field identifies its CPython 3.14 ABI baseline; the executable runtime self-identifies as 3.14.2.",
|
||||
},
|
||||
@@ -312,7 +330,7 @@ async function smokePythonPack(pack) {
|
||||
});
|
||||
const result = JSON.parse(
|
||||
runtime.runPython(`
|
||||
import json, re, sys
|
||||
import _zstd, decimal, json, pyexpat, re, sqlite3, sys, zlib
|
||||
expression = re.compile(r"(?P<emoji>😀)|(?P<word>[a-z]+)", re.ASCII)
|
||||
matches = [
|
||||
[match.span(0), match.lastgroup]
|
||||
@@ -322,6 +340,11 @@ replacement_match = re.compile(r"(?P<word>[a-z]+)").search("abc")
|
||||
json.dumps({
|
||||
"python": ".".join(str(part) for part in sys.version_info[:3]),
|
||||
"implementation": sys.implementation.name,
|
||||
"libmpdec": decimal.__libmpdec_version__,
|
||||
"expat": pyexpat.EXPAT_VERSION,
|
||||
"sqlite": sqlite3.sqlite_version,
|
||||
"zlib": zlib.ZLIB_RUNTIME_VERSION,
|
||||
"zstd": _zstd.zstd_version,
|
||||
"matches": matches,
|
||||
"replacement": replacement_match.expand(r"[\\g<word>]"),
|
||||
})
|
||||
@@ -330,6 +353,11 @@ json.dumps({
|
||||
if (
|
||||
result.python !== PYTHON_ENGINE_LOCK.pythonVersion ||
|
||||
result.implementation !== "cpython" ||
|
||||
result.libmpdec !== "2.5.1" ||
|
||||
result.expat !== "expat_2.7.3" ||
|
||||
result.sqlite !== "3.39.0" ||
|
||||
result.zlib !== "1.3.1" ||
|
||||
result.zstd !== "1.5.7" ||
|
||||
JSON.stringify(result.matches) !==
|
||||
JSON.stringify([
|
||||
[[0, 1], "emoji"],
|
||||
@@ -365,16 +393,13 @@ export async function verifyPythonPack(packDirectory, root) {
|
||||
);
|
||||
}
|
||||
|
||||
await assertHash(
|
||||
path.join(pack, "LICENSE.pyodide.txt"),
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
|
||||
"staged Pyodide licence",
|
||||
);
|
||||
await assertHash(
|
||||
path.join(pack, "LICENSE.cpython.txt"),
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
|
||||
"staged CPython licence",
|
||||
);
|
||||
for (const resource of PYTHON_LEGAL_RESOURCES) {
|
||||
await assertHash(
|
||||
path.join(pack, resource.output),
|
||||
resource.outputSha256,
|
||||
`staged ${resource.component} legal material`,
|
||||
);
|
||||
}
|
||||
await assertHash(
|
||||
path.join(pack, "pyodide.mjs"),
|
||||
PYTHON_ENGINE_LOCK.strippedLoaderSha256,
|
||||
@@ -406,6 +431,22 @@ export async function verifyPythonPack(packDirectory, root) {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
let sourceManifest;
|
||||
try {
|
||||
sourceManifest = JSON.parse(
|
||||
await readFile(path.join(pack, "SOURCE-MANIFEST.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("SOURCE-MANIFEST.json is not valid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const expectedManifest = await expectedSourceManifest(pack);
|
||||
if (!equalJson(sourceManifest, expectedManifest)) {
|
||||
throw new Error(
|
||||
"The staged Python source manifest does not match the pinned preferred-form and legal inventory.",
|
||||
);
|
||||
}
|
||||
const expected = await expectedMetadata(repositoryRoot, pack);
|
||||
if (!equalJson(metadata, expected)) {
|
||||
throw new Error(
|
||||
@@ -489,24 +530,19 @@ export async function buildPythonPack(options, root) {
|
||||
path.join(path.dirname(output), ".python-pack-build-"),
|
||||
);
|
||||
try {
|
||||
const [pyodideLicense, cpythonLicense] = await Promise.all([
|
||||
fetchPinned(
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseUrl,
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
|
||||
"Pyodide licence",
|
||||
),
|
||||
fetchPinned(
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseUrl,
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
|
||||
"CPython licence",
|
||||
),
|
||||
]);
|
||||
await writeFile(path.join(stage, "LICENSE.pyodide.txt"), pyodideLicense, {
|
||||
flag: "wx",
|
||||
});
|
||||
await writeFile(path.join(stage, "LICENSE.cpython.txt"), cpythonLicense, {
|
||||
flag: "wx",
|
||||
});
|
||||
const legalFiles = await Promise.all(
|
||||
PYTHON_LEGAL_RESOURCES.map(async (resource) => {
|
||||
const sourceValue = await fetchPinned(
|
||||
resource.url,
|
||||
resource.sourceSha256,
|
||||
`${resource.component} legal source`,
|
||||
);
|
||||
return [resource, transformPythonLegalResource(resource, sourceValue)];
|
||||
}),
|
||||
);
|
||||
for (const [resource, value] of legalFiles) {
|
||||
await writeFile(path.join(stage, resource.output), value, { flag: "wx" });
|
||||
}
|
||||
for (const name of Object.keys(PYTHON_ENGINE_LOCK.assets)) {
|
||||
if (name === "pyodide.mjs") {
|
||||
const upstream = await readFile(path.join(source, name), "utf8");
|
||||
@@ -523,6 +559,11 @@ export async function buildPythonPack(options, root) {
|
||||
);
|
||||
}
|
||||
}
|
||||
await writeFile(
|
||||
path.join(stage, "SOURCE-MANIFEST.json"),
|
||||
`${JSON.stringify(await expectedSourceManifest(stage), null, 2)}\n`,
|
||||
{ flag: "wx" },
|
||||
);
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
|
||||
134
scripts/python-engine-pack.test.ts
Normal file
134
scripts/python-engine-pack.test.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { verifyPythonPack } from "./python-engine-pack.mjs";
|
||||
import {
|
||||
PYTHON_LEGAL_RESOURCES,
|
||||
PYTHON_SOURCE_MANIFEST_BASE,
|
||||
} from "./python-engine-provenance.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("Python engine-pack provenance gate", () => {
|
||||
it("pins the complete base-runtime source, patch, and legal closure", () => {
|
||||
expect(PYTHON_LEGAL_RESOURCES.map(({ output }) => output)).toEqual([
|
||||
"LICENSE.pyodide.txt",
|
||||
"LICENSE.pyodide-stacktrace-vendors.txt",
|
||||
"LICENSE.cpython.txt",
|
||||
"NOTICE.cpython-bundled.rst",
|
||||
"LICENSE.expat.txt",
|
||||
"LICENSE.libmpdec.txt",
|
||||
"LICENSE.hacl-mit.txt",
|
||||
"LICENSE.libffi.txt",
|
||||
"LICENSE.hiwire.txt",
|
||||
"LICENSE.xz.txt",
|
||||
"LICENSE.zstd.txt",
|
||||
"LICENSE.sqlite.txt",
|
||||
"LICENSE.bzip2.txt",
|
||||
"LICENSE.zlib.txt",
|
||||
"LICENSE.emscripten.txt",
|
||||
"NOTICE.emscripten-dlmalloc.txt",
|
||||
"NOTICE.emscripten-musl.txt",
|
||||
"LICENSE.llvm-compiler-rt.txt",
|
||||
"LICENSE.llvm-libcxx.txt",
|
||||
"LICENSE.llvm-libcxxabi.txt",
|
||||
"LICENSE.llvm-libunwind.txt",
|
||||
"LICENSE.emscripten-mini-lz4.txt",
|
||||
]);
|
||||
expect(PYTHON_SOURCE_MANIFEST_BASE.patches.cpython).toHaveLength(9);
|
||||
expect(PYTHON_SOURCE_MANIFEST_BASE.patches.emscripten).toHaveLength(5);
|
||||
expect(
|
||||
PYTHON_SOURCE_MANIFEST_BASE.sourceInputs.map(({ id }) => id),
|
||||
).toEqual([
|
||||
"pyodide",
|
||||
"pyodide-build",
|
||||
"error-stack-parser",
|
||||
"stackframe",
|
||||
"cpython",
|
||||
"libffi",
|
||||
"hiwire",
|
||||
"xz-liblzma",
|
||||
"zstandard",
|
||||
"sqlite",
|
||||
"bzip2",
|
||||
"zlib",
|
||||
"emscripten",
|
||||
"hacl-star-preferred-source",
|
||||
]);
|
||||
});
|
||||
|
||||
it("verifies the installed runtime and independently anchored inventory", async () => {
|
||||
await expect(
|
||||
verifyPythonPack(
|
||||
path.resolve("public", "engines", "python"),
|
||||
path.resolve(),
|
||||
),
|
||||
).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: "CPython 3.14.2 re",
|
||||
runtime: "Pyodide 314.0.3",
|
||||
source: expect.objectContaining({
|
||||
manifest: expect.objectContaining({
|
||||
path: "SOURCE-MANIFEST.json",
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}, 30_000);
|
||||
|
||||
it("rejects substituted legal material before runtime startup", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-python-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const pack = path.join(directory, "python");
|
||||
await cp(path.resolve("public", "engines", "python"), pack, {
|
||||
recursive: true,
|
||||
});
|
||||
await writeFile(
|
||||
path.join(pack, "LICENSE.libffi.txt"),
|
||||
"substituted legal text\n",
|
||||
);
|
||||
|
||||
await expect(verifyPythonPack(pack, path.resolve())).rejects.toThrow(
|
||||
/libffi legal material SHA-256 mismatch/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("ships a manifest whose legal records match every pinned output", async () => {
|
||||
const manifest = JSON.parse(
|
||||
await readFile(
|
||||
path.resolve("public", "engines", "python", "SOURCE-MANIFEST.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as {
|
||||
legalFiles: {
|
||||
path: string;
|
||||
sha256: string;
|
||||
sourceSha256: string;
|
||||
}[];
|
||||
};
|
||||
expect(
|
||||
manifest.legalFiles.map(({ path: file, sha256, sourceSha256 }) => ({
|
||||
file,
|
||||
sha256,
|
||||
sourceSha256,
|
||||
})),
|
||||
).toEqual(
|
||||
PYTHON_LEGAL_RESOURCES.map(({ output, outputSha256, sourceSha256 }) => ({
|
||||
file: output,
|
||||
sha256: outputSha256,
|
||||
sourceSha256,
|
||||
})),
|
||||
);
|
||||
});
|
||||
});
|
||||
792
scripts/python-engine-provenance.mjs
Normal file
792
scripts/python-engine-provenance.mjs
Normal file
@@ -0,0 +1,792 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
export const PYTHON_LEGAL_RESOURCES = Object.freeze([
|
||||
{
|
||||
component: "Pyodide",
|
||||
output: "LICENSE.pyodide.txt",
|
||||
url: "https://raw.githubusercontent.com/pyodide/pyodide/ac57031be7564f864d061cb37c5c152e59f83ad4/LICENSE",
|
||||
sourceSha256:
|
||||
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
|
||||
outputSha256:
|
||||
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "Pyodide vendored Error Stack Parser and StackFrame",
|
||||
output: "LICENSE.pyodide-stacktrace-vendors.txt",
|
||||
url: "https://raw.githubusercontent.com/stacktracejs/error-stack-parser/9f33c224b5d7b607755eb277f9d51fcdb7287e24/LICENSE",
|
||||
sourceSha256:
|
||||
"899da9d991cb211a3642b84e82a9ae0b4b4e44546fd207e34d7d4ec2eb40f420",
|
||||
outputSha256:
|
||||
"899da9d991cb211a3642b84e82a9ae0b4b4e44546fd207e34d7d4ec2eb40f420",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "CPython",
|
||||
output: "LICENSE.cpython.txt",
|
||||
url: "https://raw.githubusercontent.com/python/cpython/df793163d5821791d4e7caf88885a2c11a107986/LICENSE",
|
||||
sourceSha256:
|
||||
"b0e25a78cffb43f4d92de8b61ccfa1f1f98ecbc22330b54b5251e7b6ba010231",
|
||||
outputSha256:
|
||||
"b0e25a78cffb43f4d92de8b61ccfa1f1f98ecbc22330b54b5251e7b6ba010231",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "CPython bundled components",
|
||||
output: "NOTICE.cpython-bundled.rst",
|
||||
url: "https://raw.githubusercontent.com/python/cpython/df793163d5821791d4e7caf88885a2c11a107986/Doc/license.rst",
|
||||
sourceSha256:
|
||||
"c695d550b135e53e38807e76496d1db17d22c40e461d1f3f354c86188d3305dd",
|
||||
outputSha256:
|
||||
"c695d550b135e53e38807e76496d1db17d22c40e461d1f3f354c86188d3305dd",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "Expat vendored by CPython",
|
||||
output: "LICENSE.expat.txt",
|
||||
url: "https://raw.githubusercontent.com/python/cpython/df793163d5821791d4e7caf88885a2c11a107986/Modules/expat/COPYING",
|
||||
sourceSha256:
|
||||
"122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534",
|
||||
outputSha256:
|
||||
"122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "libmpdec vendored by CPython",
|
||||
output: "LICENSE.libmpdec.txt",
|
||||
url: "https://raw.githubusercontent.com/python/cpython/df793163d5821791d4e7caf88885a2c11a107986/Modules/_decimal/libmpdec/mpdecimal.c",
|
||||
sourceSha256:
|
||||
"4f89b8095e408a18deff79cfb605299e615bae747898eb105d8936064f7fb626",
|
||||
outputSha256:
|
||||
"b07528d8b1dbf1e2d2741052996f0876e23342ce2d30d0effa39c5457716c25a",
|
||||
transformation: "leading-c-comment",
|
||||
},
|
||||
{
|
||||
component: "HACL* vendored by CPython",
|
||||
output: "LICENSE.hacl-mit.txt",
|
||||
url: "https://raw.githubusercontent.com/python/cpython/df793163d5821791d4e7caf88885a2c11a107986/Modules/_hacl/Hacl_HMAC.c",
|
||||
sourceSha256:
|
||||
"142adb769ff02b8a5327f0eb837e1f9a797bdf9a1684d21acd749dbb5b2e5be2",
|
||||
outputSha256:
|
||||
"998ce04fb8ad9dedb0bc1b44938f8c3dcf1089780fa105ce1c8c30fb5554d78c",
|
||||
transformation: "leading-c-comment",
|
||||
},
|
||||
{
|
||||
component: "libffi",
|
||||
output: "LICENSE.libffi.txt",
|
||||
url: "https://raw.githubusercontent.com/libffi/libffi/f08493d249d2067c8b3207ba46693dd858f95db3/LICENSE",
|
||||
sourceSha256:
|
||||
"2c9c2acb9743e6b007b91350475308aee44691d96aa20eacef8e199988c8c388",
|
||||
outputSha256:
|
||||
"2c9c2acb9743e6b007b91350475308aee44691d96aa20eacef8e199988c8c388",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "Hiwire",
|
||||
output: "LICENSE.hiwire.txt",
|
||||
url: "https://raw.githubusercontent.com/pyodide/hiwire/6a1e67280a15d929ebeceee54a6358c9c8d5f697/LICENSE",
|
||||
sourceSha256:
|
||||
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
|
||||
outputSha256:
|
||||
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "XZ Utils liblzma",
|
||||
output: "LICENSE.xz.txt",
|
||||
url: "https://raw.githubusercontent.com/xz-mirror/xz/9815cdf6987ef91a85493bfcfd1ce2aaf3b47a0a/COPYING",
|
||||
sourceSha256:
|
||||
"c4f8e14fafe458d84808a4cd8b69f94673ebe2bf8fc992291629a69ac12218f8",
|
||||
outputSha256:
|
||||
"c4f8e14fafe458d84808a4cd8b69f94673ebe2bf8fc992291629a69ac12218f8",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "Zstandard",
|
||||
output: "LICENSE.zstd.txt",
|
||||
url: "https://raw.githubusercontent.com/python/cpython-source-deps/eef946ae8cf1591c0e5cc5f43486210768647c2e/LICENSE",
|
||||
sourceSha256:
|
||||
"7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8",
|
||||
outputSha256:
|
||||
"7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "SQLite",
|
||||
output: "LICENSE.sqlite.txt",
|
||||
url: "https://raw.githubusercontent.com/sqlite/sqlite/47d8c40ca522babfadf3f774d9869cf5f6a5896e/LICENSE.md",
|
||||
sourceSha256:
|
||||
"9c6479123f32a1ed50d9a0af203dfe8972e65035b0cdeef436b5f8d0924517fe",
|
||||
outputSha256:
|
||||
"9c6479123f32a1ed50d9a0af203dfe8972e65035b0cdeef436b5f8d0924517fe",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "bzip2",
|
||||
output: "LICENSE.bzip2.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-ports/bzip2/60ce9dfe9a75f7ee92956aba8a83d1a37625bc1d/LICENSE",
|
||||
sourceSha256:
|
||||
"4919cfb14a73cd64fcef67b107613970cf1659a09aa675dba31314f373bc7204",
|
||||
outputSha256:
|
||||
"4919cfb14a73cd64fcef67b107613970cf1659a09aa675dba31314f373bc7204",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "zlib",
|
||||
output: "LICENSE.zlib.txt",
|
||||
url: "https://raw.githubusercontent.com/madler/zlib/51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf/LICENSE",
|
||||
sourceSha256:
|
||||
"845efc77857d485d91fb3e0b884aaa929368c717ae8186b66fe1ed2495753243",
|
||||
outputSha256:
|
||||
"845efc77857d485d91fb3e0b884aaa929368c717ae8186b66fe1ed2495753243",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "Emscripten",
|
||||
output: "LICENSE.emscripten.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/LICENSE",
|
||||
sourceSha256:
|
||||
"620a78084fc7ca97c0b5dea9abf891f3ffcadfdbf305276f099c9c4e12fc1d86",
|
||||
outputSha256:
|
||||
"620a78084fc7ca97c0b5dea9abf891f3ffcadfdbf305276f099c9c4e12fc1d86",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "dlmalloc in Emscripten",
|
||||
output: "NOTICE.emscripten-dlmalloc.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/dlmalloc.c",
|
||||
sourceSha256:
|
||||
"c008dc3c2b005ba822efb1bf6187a9d7a6aecd096826c6f28c6f5ee62885053d",
|
||||
outputSha256:
|
||||
"3a146a183961471088fac25b17b1e5e0b4b400439ae3abcf2fcd5a9b2a2f582c",
|
||||
transformation: "dlmalloc-public-domain-header",
|
||||
},
|
||||
{
|
||||
component: "musl libc in Emscripten",
|
||||
output: "NOTICE.emscripten-musl.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/libc/musl/COPYRIGHT",
|
||||
sourceSha256:
|
||||
"f9bc4423732350eb0b3f7ed7e91d530298476f8fec0c6c427a1c04ade22655af",
|
||||
outputSha256:
|
||||
"f9bc4423732350eb0b3f7ed7e91d530298476f8fec0c6c427a1c04ade22655af",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "LLVM compiler-rt in Emscripten",
|
||||
output: "LICENSE.llvm-compiler-rt.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/compiler-rt/LICENSE.TXT",
|
||||
sourceSha256:
|
||||
"1a8f1058753f1ba890de984e48f0242a3a5c29a6a8f2ed9fd813f36985387e8d",
|
||||
outputSha256:
|
||||
"1a8f1058753f1ba890de984e48f0242a3a5c29a6a8f2ed9fd813f36985387e8d",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "LLVM libc++ in Emscripten",
|
||||
output: "LICENSE.llvm-libcxx.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/libcxx/LICENSE.TXT",
|
||||
sourceSha256:
|
||||
"539dd7aed86e8a4f12cbdd0e6c50c189c7d74847e4fecc64ce2c6ee3a01da38b",
|
||||
outputSha256:
|
||||
"539dd7aed86e8a4f12cbdd0e6c50c189c7d74847e4fecc64ce2c6ee3a01da38b",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "LLVM libc++abi in Emscripten",
|
||||
output: "LICENSE.llvm-libcxxabi.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/libcxxabi/LICENSE.TXT",
|
||||
sourceSha256:
|
||||
"e2b35be49f7284a45b7baca8fc7b3ab7440e7902392b2528a457816b5bb2a15c",
|
||||
outputSha256:
|
||||
"e2b35be49f7284a45b7baca8fc7b3ab7440e7902392b2528a457816b5bb2a15c",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "LLVM libunwind in Emscripten",
|
||||
output: "LICENSE.llvm-libunwind.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/system/lib/libunwind/LICENSE.TXT",
|
||||
sourceSha256:
|
||||
"b5efebcaca80879234098e52d1725e6d9eb8fb96a19fce625d39184b705f7b6d",
|
||||
outputSha256:
|
||||
"b5efebcaca80879234098e52d1725e6d9eb8fb96a19fce625d39184b705f7b6d",
|
||||
transformation: "verbatim",
|
||||
},
|
||||
{
|
||||
component: "MiniLZ4 in Emscripten",
|
||||
output: "LICENSE.emscripten-mini-lz4.txt",
|
||||
url: "https://raw.githubusercontent.com/emscripten-core/emscripten/285c424dfa9e83b03cf8490c65ceadb7c45f28eb/third_party/mini-lz4.js",
|
||||
sourceSha256:
|
||||
"ac1eaa04b68d792d962d5946504ee5141ef58a9df86a0dfcf0ffa63d08634e46",
|
||||
outputSha256:
|
||||
"d27097e98c94f60e7f96e31e90296f32789738c1f8f271df785fe1dda9d52baa",
|
||||
transformation: "leading-c-comment",
|
||||
},
|
||||
]);
|
||||
|
||||
const CPYTHON_PATCHES = Object.freeze(
|
||||
[
|
||||
[
|
||||
"cpython/patches/0001-Public-pymain_run_python.patch",
|
||||
"ad29aa6d748176b873846bf14d601b65a694d002743792ae226d737c8bf67d5d",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0002-Fix-LONG_BIT-constant-to-be-always-32bit.patch",
|
||||
"e840dcb6c2f70ce7e2029ff51028fc980f0f98f6807c755a63ba1336c791ce08",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0003-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch",
|
||||
"a359c12516cc2f08720096d51f8cfb31d62ce57d096255beae0ca6848ede8cbe",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0004-Make-from-x-import-aware-of-jsproxy-modules.patch",
|
||||
"4297a7694d0f58e5c620f322b94bfdeef6ef5926bdf62d7dd9038c80d4f51720",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0005-Remove-JSPI-related-parts-in-emscripten_syscalls.c.patch",
|
||||
"5f7a57b74af22f3d2afd89d0cb99b8a44eed37d2b7a7cbbc28fe8f796c99d5cb",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0006-Teach-json-encoder.py-to-encode-jsnull.patch",
|
||||
"ba13244a5c1003f55c4ddfb4087c977c38b5fdeccf68f819deb0617bf04adc6a",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch",
|
||||
"bcf92273acce4b4b0ea8bed3fab3f375e944a2874a27b0f8e7b79ecb40f5a685",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0008-Export-_Py_emscripten_signal_clock-and-_Py_emscripte.patch",
|
||||
"7cf220d962a8a1f24d7c96a588d9c1f3d32849c3bc89329cbf76dff517ca737b",
|
||||
],
|
||||
[
|
||||
"cpython/patches/0009-Fix-Emscripten-trampoline-with-emcc-4.0.19.patch",
|
||||
"e4de0006020fa3118da139e566a6eea3d747ef11d39785d1d7cc5e74c8638371",
|
||||
],
|
||||
].map(([path, digest]) => ({ path, sha256: digest })),
|
||||
);
|
||||
|
||||
const EMSCRIPTEN_PATCHES = Object.freeze(
|
||||
[
|
||||
[
|
||||
"emsdk/patches/0001-Add-useful-error-when-symbol-resolution-fails.patch",
|
||||
"49fe15971dffd37da96ecaa55de036890c92d06a752d9bc481299ef844f22ab7",
|
||||
],
|
||||
[
|
||||
"emsdk/patches/0002-Fix-promise-order.patch",
|
||||
"7e6166923dc703fb571b449334ebcfbebfc5283c322d740297bf213a3e508089",
|
||||
],
|
||||
[
|
||||
"emsdk/patches/0003-dylink-Normalize-library-paths-to-prevent-duplicate-loading.patch",
|
||||
"6b04b74f6737f0e604a524e4217c7d5859a852e57308fce3793c7e07f64b33a0",
|
||||
],
|
||||
[
|
||||
"emsdk/patches/0004-Append-pic-directory-when-SIDE_MODULE.patch",
|
||||
"4e96b6bff08733dab7cad4f52e368df3669e518e0cf3aa8ba297e8097fb22147",
|
||||
],
|
||||
[
|
||||
"emsdk/patches/0005-Don-t-bundle-the-LLVM-profile-runtime-into-libcompil.patch",
|
||||
"cf23a6cb9a8b0c83230564f171afc559110dae4c4d6afa572c700264741244d8",
|
||||
],
|
||||
].map(([path, digest]) => ({ path, sha256: digest })),
|
||||
);
|
||||
|
||||
const SOURCE_INPUTS = Object.freeze([
|
||||
{
|
||||
id: "pyodide",
|
||||
version: "314.0.3",
|
||||
kind: "git",
|
||||
repository: "https://github.com/pyodide/pyodide.git",
|
||||
tag: "314.0.3",
|
||||
commitSha1: "ac57031be7564f864d061cb37c5c152e59f83ad4",
|
||||
treeSha1: "42da435a964851c39c7709bc06ef860fea80216a",
|
||||
license: "MPL-2.0",
|
||||
},
|
||||
{
|
||||
id: "pyodide-build",
|
||||
kind: "git-submodule",
|
||||
repository: "https://github.com/pyodide/pyodide-build.git",
|
||||
commitSha1: "03e7e6340d5b6eb5ef66430e391d8c2e821870df",
|
||||
treeSha1: "00d37c9eed9a479fd06bda67cedb367ec7249df9",
|
||||
pathInPyodide: "pyodide-build",
|
||||
license: "MPL-2.0",
|
||||
},
|
||||
{
|
||||
id: "error-stack-parser",
|
||||
version: "2.1.4",
|
||||
kind: "npm-source-archive-and-git",
|
||||
repository: "https://github.com/stacktracejs/error-stack-parser.git",
|
||||
tag: "v2.1.4",
|
||||
tagObjectSha1: "ce4d80da15c9124619cb086a7c26dbc3d71abf37",
|
||||
commitSha1: "9f33c224b5d7b607755eb277f9d51fcdb7287e24",
|
||||
treeSha1: "117482015c5aadf01fe2ed85e6ac78fabf6538e0",
|
||||
archive: {
|
||||
url: "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz",
|
||||
sha256:
|
||||
"e346f48f4fc606d4e52bf75806e439225508d914d147eb741df3076769ac2385",
|
||||
integrity:
|
||||
"sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==",
|
||||
},
|
||||
vendoredPort: {
|
||||
pyodideIntroductionCommitSha1: "b46e4a572d16dd96eb29b49f87fd8d34c2bc9c2e",
|
||||
path: "src/js/vendor/stackframe/error-stack-parser.ts",
|
||||
sha256:
|
||||
"06cb52d1e4ae6a0f9c3f349ba4751b1839dfe7a3515f74373f0a7fab62c52b8d",
|
||||
versionEvidence: {
|
||||
packageLockCommitSha1: "8b3e7c4cf25ed9fd0a983ba33cdf0afc36b77647",
|
||||
packageLockPath: "src/js/package-lock.json",
|
||||
packageLockSha256:
|
||||
"34cc94816d3cfdffe858e9c1f85c876f470fd886bd2de1063dd5d82199d84268",
|
||||
},
|
||||
},
|
||||
license: "MIT",
|
||||
},
|
||||
{
|
||||
id: "stackframe",
|
||||
version: "1.3.4",
|
||||
kind: "npm-source-archive-and-git",
|
||||
repository: "https://github.com/stacktracejs/stackframe.git",
|
||||
tag: "v1.3.4",
|
||||
tagObjectSha1: "6b68182ca814c4994c705fb561c9689dd5484998",
|
||||
commitSha1: "e07cfd43e89b0565f41856e9285d154aee6c0be3",
|
||||
treeSha1: "bc5fd9aa47c649bd8cf0bc53d69993e51c4f106c",
|
||||
archive: {
|
||||
url: "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz",
|
||||
sha256:
|
||||
"fb928460cb97df0ba99a1dfbeec6cb1dc9b12f9a9838a003fda25019ad9be929",
|
||||
integrity:
|
||||
"sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==",
|
||||
},
|
||||
vendoredPort: {
|
||||
pyodideIntroductionCommitSha1: "b46e4a572d16dd96eb29b49f87fd8d34c2bc9c2e",
|
||||
path: "src/js/vendor/stackframe/stackframe.ts",
|
||||
sha256:
|
||||
"f1ac1bce24f9a936957975f6205cf49337c46f6b32591e807f168ca377b392a4",
|
||||
versionEvidence: {
|
||||
packageLockCommitSha1: "8b3e7c4cf25ed9fd0a983ba33cdf0afc36b77647",
|
||||
packageLockPath: "src/js/package-lock.json",
|
||||
packageLockSha256:
|
||||
"34cc94816d3cfdffe858e9c1f85c876f470fd886bd2de1063dd5d82199d84268",
|
||||
},
|
||||
},
|
||||
license: "MIT",
|
||||
},
|
||||
{
|
||||
id: "cpython",
|
||||
version: "3.14.2",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/python/cpython.git",
|
||||
tag: "v3.14.2",
|
||||
tagObjectSha1: "a1d0069daf8e85b25a0c3f96abc43182be6d429e",
|
||||
commitSha1: "df793163d5821791d4e7caf88885a2c11a107986",
|
||||
treeSha1: "0110cee4f97a291ec6381838aac55e8183115ec5",
|
||||
archive: {
|
||||
url: "https://www.python.org/ftp/python/3.14.2/Python-3.14.2.tgz",
|
||||
sha256:
|
||||
"c609e078adab90e2c6bacb6afafacd5eaf60cd94cf670f1e159565725fcd448d",
|
||||
},
|
||||
license: "Python-2.0",
|
||||
},
|
||||
{
|
||||
id: "libffi",
|
||||
kind: "git",
|
||||
repository: "https://github.com/libffi/libffi.git",
|
||||
commitSha1: "f08493d249d2067c8b3207ba46693dd858f95db3",
|
||||
treeSha1: "fcacfbb6e4f0e3cb557f7dd733816c859c13588b",
|
||||
license: "MIT",
|
||||
},
|
||||
{
|
||||
id: "hiwire",
|
||||
version: "1.0.1",
|
||||
kind: "git",
|
||||
repository: "https://github.com/pyodide/hiwire.git",
|
||||
tag: "1.0.1",
|
||||
tagObjectSha1: "b0e3754248b05c22ab40360bcecc4c35425eac4c",
|
||||
commitSha1: "6a1e67280a15d929ebeceee54a6358c9c8d5f697",
|
||||
treeSha1: "7cd355c4e33f447004c13eeaed9a1bcbd30175f6",
|
||||
license: "MPL-2.0",
|
||||
},
|
||||
{
|
||||
id: "xz-liblzma",
|
||||
version: "5.2.2",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/xz-mirror/xz.git",
|
||||
tag: "v5.2.2",
|
||||
tagObjectSha1: "33e803345765854e065bfd58c72ea30ed7f75173",
|
||||
commitSha1: "9815cdf6987ef91a85493bfcfd1ce2aaf3b47a0a",
|
||||
treeSha1: "20acf559acea1c2fca2fd80e7edc85587ea712f9",
|
||||
archive: {
|
||||
url: "https://github.com/xz-mirror/xz/releases/download/v5.2.2/xz-5.2.2.tar.gz",
|
||||
sha256:
|
||||
"73df4d5d34f0468bd57d09f2d8af363e95ed6cc3a4a86129d2f2c366259902a2",
|
||||
},
|
||||
license: "LicenseRef-XZ-Public-Domain-With-Fallback-Grant",
|
||||
},
|
||||
{
|
||||
id: "zstandard",
|
||||
version: "1.5.7",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/python/cpython-source-deps.git",
|
||||
tag: "zstd-1.5.7",
|
||||
commitSha1: "eef946ae8cf1591c0e5cc5f43486210768647c2e",
|
||||
treeSha1: "65df13bfb803791fd5d2073f3ce86a8743dd05ff",
|
||||
archive: {
|
||||
url: "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz",
|
||||
sha256:
|
||||
"f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e",
|
||||
},
|
||||
license: "BSD-3-Clause",
|
||||
},
|
||||
{
|
||||
id: "sqlite",
|
||||
version: "3.39.0",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/sqlite/sqlite.git",
|
||||
tag: "version-3.39.0",
|
||||
commitSha1: "47d8c40ca522babfadf3f774d9869cf5f6a5896e",
|
||||
treeSha1: "23f10739e181800d49540f2836478e543862d971",
|
||||
archive: {
|
||||
url: "https://www.sqlite.org/2022/sqlite-autoconf-3390000.tar.gz",
|
||||
sha256:
|
||||
"e90bcaef6dd5813fcdee4e867f6b65f3c9bfd0aec0f1017f9f3bbce1e4ed09e2",
|
||||
},
|
||||
license: "LicenseRef-SQLite-Public-Domain-Blessing",
|
||||
},
|
||||
{
|
||||
id: "bzip2",
|
||||
version: "1.0.6",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/emscripten-ports/bzip2.git",
|
||||
tag: "1.0.6",
|
||||
commitSha1: "60ce9dfe9a75f7ee92956aba8a83d1a37625bc1d",
|
||||
treeSha1: "d5950f6fa509172a4195f8e7536ee404f0f09af4",
|
||||
archive: {
|
||||
url: "https://github.com/emscripten-ports/bzip2/archive/1.0.6.zip",
|
||||
sha256:
|
||||
"9628e7bc598719335ce1d1219c1a4fc241652c0a7b4d993fa82a9b98a6c5dec6",
|
||||
upstreamSha512:
|
||||
"512cbfde5144067f677496452f3335e9368fd5d7564899cb49e77847b9ae7dca598218276637cbf5ec524523be1e8ace4ad36a148ef7f4badf3f6d5a002a4bb2",
|
||||
},
|
||||
license: "bzip2-1.0.6",
|
||||
},
|
||||
{
|
||||
id: "zlib",
|
||||
version: "1.3.1",
|
||||
kind: "release-archive-and-git",
|
||||
repository: "https://github.com/madler/zlib.git",
|
||||
tag: "v1.3.1",
|
||||
tagObjectSha1: "925af44f3cde53c6b076611c297850091b5dc7bb",
|
||||
commitSha1: "51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf",
|
||||
treeSha1: "16b86ef85c591c244523c98c71508be7908d1189",
|
||||
archive: {
|
||||
url: "https://github.com/madler/zlib/archive/refs/tags/v1.3.1.tar.gz",
|
||||
sha256:
|
||||
"17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
|
||||
upstreamSha512:
|
||||
"8c9642495bafd6fad4ab9fb67f09b268c69ff9af0f4f20cf15dfc18852ff1f312bd8ca41de761b3f8d8e90e77d79f2ccacd3d4c5b19e475ecf09d021fdfe9088",
|
||||
},
|
||||
license: "Zlib",
|
||||
},
|
||||
{
|
||||
id: "emscripten",
|
||||
version: "5.0.3",
|
||||
kind: "git",
|
||||
repository: "https://github.com/emscripten-core/emscripten.git",
|
||||
tag: "5.0.3",
|
||||
commitSha1: "285c424dfa9e83b03cf8490c65ceadb7c45f28eb",
|
||||
treeSha1: "390d84c64eb93ae47aa9d71cf1cd2f74bc7c2b9e",
|
||||
license: "MIT OR NCSA",
|
||||
},
|
||||
{
|
||||
id: "hacl-star-preferred-source",
|
||||
kind: "git",
|
||||
repository: "https://github.com/hacl-star/hacl-star.git",
|
||||
commitSha1: "8ba599b2f6c9701b3dc961db895b0856a2210f76",
|
||||
treeSha1: "8efcb9246e0bf2593dca9fbfd94a34f7cee50fda",
|
||||
vendoredGeneratedSource:
|
||||
"CPython Modules/_hacl; CPython's refresh.sh pins this revision and records its deterministic copy/namespace transformations.",
|
||||
license: "MIT for the linked generated C sources",
|
||||
},
|
||||
]);
|
||||
|
||||
const LINKED_COMPONENTS = Object.freeze([
|
||||
{
|
||||
id: "pyodide",
|
||||
version: "314.0.3",
|
||||
license: "MPL-2.0",
|
||||
noticeFiles: ["LICENSE.pyodide.txt"],
|
||||
evidence: "npm package identity plus the Pyodide loader and core module",
|
||||
},
|
||||
{
|
||||
id: "error-stack-parser",
|
||||
version: "2.1.4 vendored ES-module port",
|
||||
license: "MIT",
|
||||
noticeFiles: ["LICENSE.pyodide-stacktrace-vendors.txt"],
|
||||
evidence:
|
||||
"Pyodide compat.ts imports the vendored port; its source hash, introduction commit and exact upstream tag/archive are recorded here",
|
||||
},
|
||||
{
|
||||
id: "stackframe",
|
||||
version: "1.3.4 vendored ES-module port",
|
||||
license: "MIT",
|
||||
noticeFiles: ["LICENSE.pyodide-stacktrace-vendors.txt"],
|
||||
evidence:
|
||||
"The vendored Error Stack Parser imports the StackFrame port; its source hash, introduction commit and exact upstream tag/archive are recorded here",
|
||||
},
|
||||
{
|
||||
id: "cpython",
|
||||
version: "3.14.2",
|
||||
license: "Python-2.0 and bundled component terms",
|
||||
noticeFiles: ["LICENSE.cpython.txt", "NOTICE.cpython-bundled.rst"],
|
||||
evidence: "runtime sys.version and pinned Pyodide CPython recipe",
|
||||
},
|
||||
{
|
||||
id: "expat",
|
||||
version: "2.7.3",
|
||||
license: "MIT",
|
||||
noticeFiles: ["LICENSE.expat.txt"],
|
||||
evidence:
|
||||
"CPython vendored source and Pyodide LIBEXPAT_OBJS; executable contains expat_2.7.3",
|
||||
},
|
||||
{
|
||||
id: "libmpdec",
|
||||
version: "2.5.1",
|
||||
license: "BSD-2-Clause",
|
||||
noticeFiles: ["LICENSE.libmpdec.txt"],
|
||||
evidence:
|
||||
"CPython vendored source and Pyodide LIBMPDEC_OBJS; executable contains 2.5.1",
|
||||
},
|
||||
{
|
||||
id: "hacl-star",
|
||||
version: "commit 8ba599b2f6c9701b3dc961db895b0856a2210f76",
|
||||
license: "MIT for the linked generated C sources",
|
||||
noticeFiles: ["LICENSE.hacl-mit.txt"],
|
||||
evidence:
|
||||
"CPython _hmac static module and Modules/_hacl/refresh.sh preferred-source pin",
|
||||
},
|
||||
{
|
||||
id: "libffi",
|
||||
version: "commit f08493d249d2067c8b3207ba46693dd858f95db3",
|
||||
license: "MIT",
|
||||
noticeFiles: ["LICENSE.libffi.txt"],
|
||||
evidence: "Pyodide CPython Makefile pin and linked _ctypes/libffi symbols",
|
||||
},
|
||||
{
|
||||
id: "hiwire",
|
||||
version: "1.0.1",
|
||||
license: "MPL-2.0",
|
||||
noticeFiles: ["LICENSE.hiwire.txt"],
|
||||
evidence: "Pyodide CPython Makefile pin and linked hiwire symbols",
|
||||
},
|
||||
{
|
||||
id: "xz-liblzma",
|
||||
version: "5.2.2",
|
||||
license: "public domain with the upstream fallback permission grant",
|
||||
noticeFiles: ["LICENSE.xz.txt"],
|
||||
evidence:
|
||||
"Pyodide CPython Makefile archive pin and executable 5.2.2 string",
|
||||
},
|
||||
{
|
||||
id: "zstandard",
|
||||
version: "1.5.7",
|
||||
license: "BSD-3-Clause",
|
||||
noticeFiles: ["LICENSE.zstd.txt"],
|
||||
evidence:
|
||||
"Pyodide CPython Makefile archive pin and executable 1.5.7 string",
|
||||
},
|
||||
{
|
||||
id: "sqlite",
|
||||
version: "3.39.0",
|
||||
license: "public-domain dedication/blessing",
|
||||
noticeFiles: ["LICENSE.sqlite.txt"],
|
||||
evidence:
|
||||
"Pyodide CPython Makefile archive pin and executable 3.39.0 string",
|
||||
},
|
||||
{
|
||||
id: "bzip2",
|
||||
version: "1.0.6",
|
||||
license: "bzip2-1.0.6",
|
||||
noticeFiles: ["LICENSE.bzip2.txt"],
|
||||
evidence:
|
||||
"Pyodide -sUSE_BZIP2 and Emscripten 5.0.3 port lock; executable version string",
|
||||
},
|
||||
{
|
||||
id: "zlib",
|
||||
version: "1.3.1",
|
||||
license: "Zlib",
|
||||
noticeFiles: ["LICENSE.zlib.txt"],
|
||||
evidence:
|
||||
"Pyodide -sUSE_ZLIB and Emscripten 5.0.3 port lock; executable version string",
|
||||
},
|
||||
{
|
||||
id: "emscripten",
|
||||
version: "5.0.3 with the five Pyodide patches recorded here",
|
||||
license: "MIT OR NCSA",
|
||||
noticeFiles: ["LICENSE.emscripten.txt"],
|
||||
evidence: "Pyodide lock platform and build recipe",
|
||||
},
|
||||
{
|
||||
id: "emscripten-musl",
|
||||
version: "snapshot in Emscripten 5.0.3",
|
||||
license: "MIT plus file-level permissive terms",
|
||||
noticeFiles: ["NOTICE.emscripten-musl.txt"],
|
||||
evidence: "Emscripten system libc selected by the main-module link",
|
||||
},
|
||||
{
|
||||
id: "emscripten-llvm-runtime",
|
||||
version: "snapshot in Emscripten 5.0.3",
|
||||
license: "Apache-2.0 WITH LLVM-exception",
|
||||
noticeFiles: [
|
||||
"LICENSE.llvm-compiler-rt.txt",
|
||||
"LICENSE.llvm-libcxx.txt",
|
||||
"LICENSE.llvm-libcxxabi.txt",
|
||||
"LICENSE.llvm-libunwind.txt",
|
||||
],
|
||||
evidence:
|
||||
"Conservative notice closure for compiler-rt, libc++, libc++abi and libunwind selected by the pinned Emscripten link; the stripped npm module retains no link map",
|
||||
},
|
||||
{
|
||||
id: "emscripten-dlmalloc",
|
||||
version: "2.8.6 snapshot in Emscripten 5.0.3",
|
||||
license: "public domain dedication",
|
||||
noticeFiles: ["NOTICE.emscripten-dlmalloc.txt"],
|
||||
evidence:
|
||||
"Emscripten 5.0.3 default MALLOC setting and independently hashed dlmalloc source dedication",
|
||||
},
|
||||
{
|
||||
id: "emscripten-mini-lz4",
|
||||
version: "snapshot in Emscripten 5.0.3",
|
||||
license: "MIT",
|
||||
noticeFiles: ["LICENSE.emscripten-mini-lz4.txt"],
|
||||
evidence:
|
||||
"Pyodide -sLZ4=1; executable loader contains MiniLZ4 after minification removed its source comment",
|
||||
},
|
||||
]);
|
||||
|
||||
export const PYTHON_SOURCE_MANIFEST_BASE = Object.freeze({
|
||||
schemaVersion: 1,
|
||||
pack: "python",
|
||||
distribution: {
|
||||
npmPackage: "pyodide@314.0.3",
|
||||
resolved: "https://registry.npmjs.org/pyodide/-/pyodide-314.0.3.tgz",
|
||||
integrity:
|
||||
"sha512-sK40My6m8tmBUYtYH9au9rXUeh9x0wfahtHdOlGmJxZDsKBGKtP6KznyFB2+u/klbQTdDionR0uaVd176zVQzQ==",
|
||||
role: "Exact binary distribution input; no wheel named by pyodide-lock.json is bundled unless it is one of the separately enumerated base-runtime files.",
|
||||
},
|
||||
sourceAvailability: {
|
||||
preferredFormRoot:
|
||||
"https://github.com/pyodide/pyodide/tree/ac57031be7564f864d061cb37c5c152e59f83ad4",
|
||||
instructions: [
|
||||
"Clone the Pyodide repository, checkout commit ac57031be7564f864d061cb37c5c152e59f83ad4, verify tree 42da435a964851c39c7709bc06ef860fea80216a, and initialise the recorded pyodide-build submodule.",
|
||||
"Acquire each release archive below from its primary upstream URL and verify SHA-256 before extraction; Git inputs must match both commit and tree identities.",
|
||||
"Apply the CPython and Emscripten patch series in the listed lexical order. The patches and recipe files live in the exact Pyodide preferred-form source tree and are independently hashed below.",
|
||||
"Follow the pinned Pyodide Makefiles. The npm runtime is the exact distributed executable reference; this source route closes preferred-form and patch availability but does not claim a byte-identical rebuild across host SDK packaging changes.",
|
||||
],
|
||||
regexToolsModification:
|
||||
"Regex Tools removes only the final //# sourceMappingURL=pyodide.mjs.map trailer from the copied loader; no executable statement in the upstream loader or WebAssembly module is changed.",
|
||||
},
|
||||
sourceInputs: SOURCE_INPUTS,
|
||||
buildRecipeFiles: [
|
||||
{
|
||||
path: "Makefile",
|
||||
sha256:
|
||||
"a00cd2fe502ee227a2fd53029cb51d833e043b260679544eabab5f96fc135719",
|
||||
},
|
||||
{
|
||||
path: "Makefile.envs",
|
||||
sha256:
|
||||
"091d32a55cef89e22d984eca596d23bcb711eefafa2aba5c7ae62079c1d634f8",
|
||||
},
|
||||
{
|
||||
path: "cpython/Makefile",
|
||||
sha256:
|
||||
"fbeeebe3363fa9da54b6908c903a16bbf736af65647a9fbfe277c8e40eb785bf",
|
||||
},
|
||||
{
|
||||
path: "cpython/Setup.local",
|
||||
sha256:
|
||||
"7fbbb0efe5497765122230c24ebc95861973bb985116f89705b6ff51cfcffa4c",
|
||||
},
|
||||
{
|
||||
path: "emsdk/Makefile",
|
||||
sha256:
|
||||
"8b2dfe4111c8ba746ecd475b6ee4774776ec090dc255c147ec56146b06a03fc8",
|
||||
},
|
||||
],
|
||||
patches: {
|
||||
cpython: CPYTHON_PATCHES,
|
||||
emscripten: EMSCRIPTEN_PATCHES,
|
||||
},
|
||||
linkedComponents: LINKED_COMPONENTS,
|
||||
});
|
||||
|
||||
export function transformPythonLegalResource(resource, source) {
|
||||
const actualSource = sha256(source);
|
||||
if (actualSource !== resource.sourceSha256) {
|
||||
throw new Error(
|
||||
`${resource.component} legal source SHA-256 mismatch: expected ${resource.sourceSha256}, got ${actualSource}.`,
|
||||
);
|
||||
}
|
||||
let output;
|
||||
if (resource.transformation === "verbatim") {
|
||||
output = source;
|
||||
} else if (resource.transformation === "leading-c-comment") {
|
||||
const match = /^\/\*[\s\S]*?\*\/\r?\n/u.exec(source.toString("utf8"));
|
||||
if (!match) {
|
||||
throw new Error(
|
||||
`${resource.component} legal source has no leading C comment.`,
|
||||
);
|
||||
}
|
||||
output = Buffer.from(match[0].replaceAll("\r\n", "\n"), "utf8");
|
||||
} else if (resource.transformation === "dlmalloc-public-domain-header") {
|
||||
const match =
|
||||
/ This is a version \(aka dlmalloc\)[\s\S]*? Check before installing!\r?\n/u.exec(
|
||||
source.toString("utf8"),
|
||||
);
|
||||
if (!match) {
|
||||
throw new Error(
|
||||
`${resource.component} source has no pinned public-domain/version header.`,
|
||||
);
|
||||
}
|
||||
output = Buffer.from(match[0].replaceAll("\r\n", "\n").trimStart(), "utf8");
|
||||
} else {
|
||||
throw new Error(
|
||||
`${resource.component} has an unsupported legal transformation.`,
|
||||
);
|
||||
}
|
||||
const actualOutput = sha256(output);
|
||||
if (actualOutput !== resource.outputSha256) {
|
||||
throw new Error(
|
||||
`${resource.component} legal output SHA-256 mismatch: expected ${resource.outputSha256}, got ${actualOutput}.`,
|
||||
);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function createPythonSourceManifest(legalFileRecords) {
|
||||
const expectedPaths = PYTHON_LEGAL_RESOURCES.map(({ output }) => output);
|
||||
if (
|
||||
legalFileRecords.length !== expectedPaths.length ||
|
||||
legalFileRecords.some(
|
||||
({ path, sha256: digest }, index) =>
|
||||
path !== expectedPaths[index] ||
|
||||
digest !== PYTHON_LEGAL_RESOURCES[index].outputSha256,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
"Python legal file records do not match the pinned provenance inventory.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
...PYTHON_SOURCE_MANIFEST_BASE,
|
||||
legalFiles: legalFileRecords.map((record, index) => ({
|
||||
...record,
|
||||
component: PYTHON_LEGAL_RESOURCES[index].component,
|
||||
sourceUrl: PYTHON_LEGAL_RESOURCES[index].url,
|
||||
sourceSha256: PYTHON_LEGAL_RESOURCES[index].sourceSha256,
|
||||
transformation: PYTHON_LEGAL_RESOURCES[index].transformation,
|
||||
})),
|
||||
};
|
||||
}
|
||||
132
scripts/release-path-hygiene.mjs
Normal file
132
scripts/release-path-hygiene.mjs
Normal file
@@ -0,0 +1,132 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
const DOCUMENTED_UPSTREAM_PATH_ARTIFACTS = Object.freeze({
|
||||
"engines/perl/emperl.data":
|
||||
"9529019418cf766a42cf2d25bd3fc97b47c9e689f5666cfc32dc11338d1b1e66",
|
||||
"engines/ruby/ruby.wasm":
|
||||
"fa1d5af1b4d601191739e6c4980d060d0d202181a7c83abf4930b3ca35b90f94",
|
||||
});
|
||||
|
||||
const LOCAL_PATH_PREFIXES = Object.freeze([
|
||||
"/mnt/",
|
||||
"/Users/",
|
||||
"/home/",
|
||||
...Array.from(
|
||||
{ length: 26 },
|
||||
(_, index) => `${String.fromCharCode(65 + index)}:\\Users\\`,
|
||||
),
|
||||
...Array.from(
|
||||
{ length: 26 },
|
||||
(_, index) => `${String.fromCharCode(65 + index)}:/Users/`,
|
||||
),
|
||||
...Array.from(
|
||||
{ length: 26 },
|
||||
(_, index) => `${String.fromCharCode(97 + index)}:\\Users\\`,
|
||||
),
|
||||
...Array.from(
|
||||
{ length: 26 },
|
||||
(_, index) => `${String.fromCharCode(97 + index)}:/Users/`,
|
||||
),
|
||||
]);
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function utf16BigEndian(value) {
|
||||
const encoded = Buffer.from(value, "utf16le");
|
||||
for (let index = 0; index < encoded.byteLength; index += 2) {
|
||||
const first = encoded[index];
|
||||
encoded[index] = encoded[index + 1];
|
||||
encoded[index + 1] = first;
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
function encodedForms(value) {
|
||||
return [
|
||||
{ encoding: "ASCII/UTF-8", value: Buffer.from(value, "utf8") },
|
||||
{ encoding: "UTF-16LE", value: Buffer.from(value, "utf16le") },
|
||||
{ encoding: "UTF-16BE", value: utf16BigEndian(value) },
|
||||
];
|
||||
}
|
||||
|
||||
function startsWithAt(data, value, offset) {
|
||||
return (
|
||||
offset + value.byteLength <= data.byteLength &&
|
||||
data.subarray(offset, offset + value.byteLength).equals(value)
|
||||
);
|
||||
}
|
||||
|
||||
function isAllowedVirtualHome(data, offset, encoding, allowedVirtualHomes) {
|
||||
return allowedVirtualHomes.some((home) => {
|
||||
const encodedHome = encodedForms(home).find(
|
||||
(candidate) => candidate.encoding === encoding,
|
||||
)?.value;
|
||||
if (!encodedHome || !startsWithAt(data, encodedHome, offset)) return false;
|
||||
const end = offset + encodedHome.byteLength;
|
||||
if (end === data.byteLength) return true;
|
||||
const character =
|
||||
encoding === "UTF-16LE" && end + 1 < data.byteLength
|
||||
? data.readUInt16LE(end)
|
||||
: encoding === "UTF-16BE" && end + 1 < data.byteLength
|
||||
? data.readUInt16BE(end)
|
||||
: data[end];
|
||||
return (
|
||||
character === 0 ||
|
||||
character === 0x2f ||
|
||||
!(
|
||||
(character >= 0x30 && character <= 0x39) ||
|
||||
(character >= 0x41 && character <= 0x5a) ||
|
||||
(character >= 0x61 && character <= 0x7a) ||
|
||||
character === 0x2e ||
|
||||
character === 0x5f ||
|
||||
character === 0x2d
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function localAbsolutePathMatch(
|
||||
input,
|
||||
artifactName,
|
||||
{ allowedVirtualHomes = [] } = {},
|
||||
) {
|
||||
const data = Buffer.from(input);
|
||||
const documentedDigest = DOCUMENTED_UPSTREAM_PATH_ARTIFACTS[artifactName];
|
||||
if (documentedDigest && sha256(data) === documentedDigest) return null;
|
||||
|
||||
for (const prefix of LOCAL_PATH_PREFIXES) {
|
||||
for (const encoded of encodedForms(prefix)) {
|
||||
let offset = data.indexOf(encoded.value);
|
||||
while (offset !== -1) {
|
||||
if (
|
||||
prefix !== "/home/" ||
|
||||
!isAllowedVirtualHome(
|
||||
data,
|
||||
offset,
|
||||
encoded.encoding,
|
||||
allowedVirtualHomes,
|
||||
)
|
||||
) {
|
||||
return { prefix, encoding: encoded.encoding };
|
||||
}
|
||||
offset = data.indexOf(encoded.value, offset + encoded.value.byteLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function assertNoUnexpectedLocalPaths(
|
||||
input,
|
||||
artifactName,
|
||||
options = {},
|
||||
) {
|
||||
const match = localAbsolutePathMatch(input, artifactName, options);
|
||||
if (match) {
|
||||
throw new Error(
|
||||
`Local absolute path detected in ${artifactName} (${match.encoding} ${match.prefix}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
537
scripts/ruby-engine-pack.mjs
Normal file
537
scripts/ruby-engine-pack.mjs
Normal file
@@ -0,0 +1,537 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { File, PreopenDirectory } from "@bjorn3/browser_wasi_shim";
|
||||
import { DefaultRubyVM } from "@ruby/wasm-wasi/dist/browser";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
export const RUBY_ENGINE_LOCK = Object.freeze({
|
||||
bridgeAbi: 2,
|
||||
rubyVersion: "4.0.0",
|
||||
platform: "wasm32-wasi",
|
||||
packageVersion: "2.9.3-2.9.4",
|
||||
maximumPatternUtf16: 65_536,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementUtf16: 65_536,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
npm: Object.freeze({
|
||||
runtime: Object.freeze({
|
||||
name: "@ruby/4.0-wasm-wasi",
|
||||
version: "2.9.3-2.9.4",
|
||||
resolved:
|
||||
"https://registry.npmjs.org/@ruby/4.0-wasm-wasi/-/4.0-wasm-wasi-2.9.3-2.9.4.tgz",
|
||||
integrity:
|
||||
"sha512-USLEWmCjM/DIota/9c7Mwzzl3v8mvmg4x24m+zMq8+vLx5+AdOS6dyXvsL/B8+XjQn5UgYhh3c8ekaUdWFkUMA==",
|
||||
packageJsonSha256:
|
||||
"a03a9475d54494a351c9c37ee9aca2db05f2652098907938628522d8a0f43807",
|
||||
}),
|
||||
host: Object.freeze({
|
||||
name: "@ruby/wasm-wasi",
|
||||
version: "2.9.3-2.9.4",
|
||||
resolved:
|
||||
"https://registry.npmjs.org/@ruby/wasm-wasi/-/wasm-wasi-2.9.3-2.9.4.tgz",
|
||||
integrity:
|
||||
"sha512-WxW9wON/TIf+8Ktng8qDJeV/6iH8kw+YwxOsOyXdAdLJgfYDPAXOqpIVd/96y2C9V8VJ2yqZm/IRv6nLeV6EKg==",
|
||||
packageJsonSha256:
|
||||
"4a6eacec227c5cf81f580435d0105933a456e9685cb3f1821db967307431aa20",
|
||||
}),
|
||||
wasiShim: Object.freeze({
|
||||
name: "@bjorn3/browser_wasi_shim",
|
||||
version: "0.4.2",
|
||||
integrity:
|
||||
"sha512-/iHkCVUG3VbcbmEHn5iIUpIrh7a7WPiwZ3sHy4HZKZzBdSadwdddYDZAII2zBvQYV0Lfi8naZngPCN7WPHI/hA==",
|
||||
packageJsonSha256:
|
||||
"4025e157abfadd44037158899001a4ea0407651c2ee9c8a9f51d5c9d0ecf613c",
|
||||
}),
|
||||
tslib: Object.freeze({
|
||||
name: "tslib",
|
||||
version: "2.8.1",
|
||||
integrity:
|
||||
"sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
packageJsonSha256:
|
||||
"f02b4851df316f29bdfcd404aa1eb41e4e216b5859015f81fcd3e91caf8be931",
|
||||
}),
|
||||
}),
|
||||
sourceAssets: Object.freeze({
|
||||
"ruby.wasm":
|
||||
"fa1d5af1b4d601191739e6c4980d060d0d202181a7c83abf4930b3ca35b90f94",
|
||||
"LICENSE.ruby-wasm.txt":
|
||||
"90357d3794c968704914d42a52354a83f2d8b10cb43df3b63ef1ca0e5bbc0bf2",
|
||||
"NOTICE.txt":
|
||||
"343c246a6e1f1234e29e51707a54799ea82b50d3a2a41c5221fa12058b2395b2",
|
||||
"LICENSE.browser-wasi-shim-MIT.txt":
|
||||
"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3",
|
||||
"LICENSE.browser-wasi-shim-Apache-2.0.txt":
|
||||
"c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
||||
"LICENSE.tslib.txt":
|
||||
"210b19e543130388c68654b7497e967119ce17145f66ab7d85688fbd70f08751",
|
||||
}),
|
||||
});
|
||||
|
||||
const SOURCE_FILES = Object.freeze({
|
||||
"ruby.wasm": ["@ruby", "4.0-wasm-wasi", "dist", "ruby.wasm"],
|
||||
"LICENSE.ruby-wasm.txt": ["@ruby", "4.0-wasm-wasi", "dist", "LICENSE"],
|
||||
"NOTICE.txt": ["@ruby", "4.0-wasm-wasi", "dist", "NOTICE"],
|
||||
"LICENSE.browser-wasi-shim-MIT.txt": [
|
||||
"@bjorn3",
|
||||
"browser_wasi_shim",
|
||||
"LICENSE-MIT",
|
||||
],
|
||||
"LICENSE.browser-wasi-shim-Apache-2.0.txt": [
|
||||
"@bjorn3",
|
||||
"browser_wasi_shim",
|
||||
"LICENSE-APACHE",
|
||||
],
|
||||
"LICENSE.tslib.txt": ["tslib", "LICENSE.txt"],
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze(
|
||||
[...Object.keys(SOURCE_FILES), "SHA256SUMS", "engine-metadata.json"].sort(),
|
||||
);
|
||||
const CHECKSUM_FILES = Object.freeze(
|
||||
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
|
||||
);
|
||||
|
||||
function compareText(left, right) {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`${label} must be a regular, non-symlink file: ${candidate}`,
|
||||
);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertHash(file, expected, label) {
|
||||
const actual = await sha256File(file);
|
||||
if (actual !== expected) {
|
||||
throw new Error(
|
||||
`${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyPackage(projectRoot, descriptor) {
|
||||
const directory = path.join(
|
||||
projectRoot,
|
||||
"node_modules",
|
||||
...descriptor.name.split("/"),
|
||||
);
|
||||
await assertRealDirectory(directory, `${descriptor.name} npm package`);
|
||||
const packageFile = path.join(directory, "package.json");
|
||||
await assertRegularFile(packageFile, `${descriptor.name} package.json`);
|
||||
await assertHash(
|
||||
packageFile,
|
||||
descriptor.packageJsonSha256,
|
||||
`${descriptor.name} package.json`,
|
||||
);
|
||||
const packageJson = JSON.parse(await readFile(packageFile, "utf8"));
|
||||
if (
|
||||
packageJson.name !== descriptor.name ||
|
||||
packageJson.version !== descriptor.version
|
||||
) {
|
||||
throw new Error(`${descriptor.name} npm package identity drifted.`);
|
||||
}
|
||||
return directory;
|
||||
}
|
||||
|
||||
async function verifyNpmLock(projectRoot) {
|
||||
const lock = JSON.parse(
|
||||
await readFile(path.join(projectRoot, "package-lock.json"), "utf8"),
|
||||
);
|
||||
for (const descriptor of Object.values(RUBY_ENGINE_LOCK.npm)) {
|
||||
const entry = lock.packages?.[`node_modules/${descriptor.name}`];
|
||||
if (
|
||||
entry?.version !== descriptor.version ||
|
||||
entry?.integrity !== descriptor.integrity ||
|
||||
(descriptor.resolved !== undefined &&
|
||||
entry?.resolved !== descriptor.resolved)
|
||||
) {
|
||||
throw new Error(
|
||||
`${descriptor.name}@${descriptor.version} is not exactly pinned in package-lock.json.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function verifySources(projectRoot) {
|
||||
await verifyNpmLock(projectRoot);
|
||||
const runtime = await verifyPackage(
|
||||
projectRoot,
|
||||
RUBY_ENGINE_LOCK.npm.runtime,
|
||||
);
|
||||
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.host);
|
||||
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.wasiShim);
|
||||
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.tslib);
|
||||
|
||||
const nodeModules = path.join(projectRoot, "node_modules");
|
||||
for (const [output, components] of Object.entries(SOURCE_FILES)) {
|
||||
const source = path.join(nodeModules, ...components);
|
||||
await assertRegularFile(source, `Ruby source ${output}`);
|
||||
await assertHash(
|
||||
source,
|
||||
RUBY_ENGINE_LOCK.sourceAssets[output],
|
||||
`Ruby source ${output}`,
|
||||
);
|
||||
}
|
||||
return { nodeModules, runtime };
|
||||
}
|
||||
|
||||
async function fileRecord(directory, name) {
|
||||
const value = await readFile(path.join(directory, name));
|
||||
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
||||
}
|
||||
|
||||
async function expectedMetadata(projectRoot, pack) {
|
||||
const bridgePath = path.join(projectRoot, "engines", "ruby", "bridge.rb");
|
||||
const bridge = await readFile(bridgePath);
|
||||
const files = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
if (name !== "engine-metadata.json") {
|
||||
files.push(await fileRecord(pack, name));
|
||||
}
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "ruby",
|
||||
engineVersion: `CRuby ${RUBY_ENGINE_LOCK.rubyVersion} Regexp`,
|
||||
runtime: `ruby.wasm ${RUBY_ENGINE_LOCK.packageVersion}`,
|
||||
platform: RUBY_ENGINE_LOCK.platform,
|
||||
status: "production-runtime",
|
||||
bridge: {
|
||||
abiVersion: RUBY_ENGINE_LOCK.bridgeAbi,
|
||||
sourceFile: {
|
||||
path: "engines/ruby/bridge.rb",
|
||||
sha256: sha256(bridge),
|
||||
bytes: bridge.byteLength,
|
||||
},
|
||||
requestEncoding:
|
||||
"bounded length-prefixed UTF-8 fields in a worker-local WASI memory file",
|
||||
responseEncoding:
|
||||
"bounded inert JSON metadata plus a worker-local bounded UTF-8 output file",
|
||||
userSourceEvaluation: false,
|
||||
replacement:
|
||||
"native Ruby String#gsub/String#sub match iteration with a bounded CRuby 4.0.0 rb_reg_regsub-compatible expander",
|
||||
nativeOffsetUnit: "Unicode code points",
|
||||
maximumPatternUtf16: RUBY_ENGINE_LOCK.maximumPatternUtf16,
|
||||
maximumSubjectBytes: RUBY_ENGINE_LOCK.maximumSubjectBytes,
|
||||
maximumReplacementUtf16: RUBY_ENGINE_LOCK.maximumReplacementUtf16,
|
||||
maximumOutputBytes: RUBY_ENGINE_LOCK.maximumOutputBytes,
|
||||
maximumMatches: RUBY_ENGINE_LOCK.maximumMatches,
|
||||
maximumCaptureRows: RUBY_ENGINE_LOCK.maximumCaptureRows,
|
||||
maximumCaptureGroups: RUBY_ENGINE_LOCK.maximumCaptureGroups,
|
||||
},
|
||||
source: {
|
||||
repository: "https://github.com/ruby/ruby.wasm",
|
||||
packages: Object.values(RUBY_ENGINE_LOCK.npm).map(
|
||||
({ name, version, integrity }) => ({ name, version, integrity }),
|
||||
),
|
||||
runtimeVariant: "minimal ruby.wasm (without bundled stdlib/debug)",
|
||||
cruby: {
|
||||
engine: "ruby",
|
||||
version: RUBY_ENGINE_LOCK.rubyVersion,
|
||||
platform: RUBY_ENGINE_LOCK.platform,
|
||||
},
|
||||
},
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
async function smokeRuby(pack, projectRoot) {
|
||||
const moduleBytes = await readFile(path.join(pack, "ruby.wasm"));
|
||||
if (!WebAssembly.validate(moduleBytes)) {
|
||||
throw new Error("Ruby runtime is not a valid WebAssembly module.");
|
||||
}
|
||||
const module = await WebAssembly.compile(moduleBytes);
|
||||
const { vm, wasi } = await DefaultRubyVM(module, { consolePrint: false });
|
||||
const requestFile = new File([]);
|
||||
const outputFile = new File([]);
|
||||
const rootDirectory = wasi.fds[3];
|
||||
if (!(rootDirectory instanceof PreopenDirectory)) {
|
||||
throw new Error("Ruby smoke runtime has no writable root filesystem.");
|
||||
}
|
||||
rootDirectory.dir.contents.set("regex-tools-request.bin", requestFile);
|
||||
rootDirectory.dir.contents.set("regex-tools-output.bin", outputFile);
|
||||
const bridgeSource = await readFile(
|
||||
path.join(projectRoot, "engines", "ruby", "bridge.rb"),
|
||||
"utf8",
|
||||
);
|
||||
const bridge = vm.eval(bridgeSource);
|
||||
const encoder = new TextEncoder();
|
||||
const call = (...values) => {
|
||||
const fields = values.map((value) => encoder.encode(String(value)));
|
||||
const payload = new Uint8Array(
|
||||
8 + fields.reduce((total, field) => total + 4 + field.byteLength, 0),
|
||||
);
|
||||
payload.set(encoder.encode("RGXRUBY1"));
|
||||
const view = new DataView(payload.buffer);
|
||||
let offset = 8;
|
||||
for (const field of fields) {
|
||||
view.setUint32(offset, field.byteLength, false);
|
||||
offset += 4;
|
||||
payload.set(field, offset);
|
||||
offset += field.byteLength;
|
||||
}
|
||||
requestFile.data = payload;
|
||||
const response = JSON.parse(bridge.call("run").toString());
|
||||
if (String(values[0]) === "replace" && response?.ok === true) {
|
||||
response.output = new TextDecoder("utf-8", { fatal: true }).decode(
|
||||
outputFile.data,
|
||||
);
|
||||
}
|
||||
return response;
|
||||
};
|
||||
const identity = call("identity", "", "", "", 1, 1, "", 1);
|
||||
if (
|
||||
identity?.ok !== true ||
|
||||
identity.identity?.implementation !== "ruby" ||
|
||||
identity.identity?.rubyVersion !== RUBY_ENGINE_LOCK.rubyVersion ||
|
||||
identity.identity?.platform !== RUBY_ENGINE_LOCK.platform
|
||||
) {
|
||||
throw new Error("Ruby runtime failed its identity smoke test.");
|
||||
}
|
||||
const execution = call(
|
||||
"execute",
|
||||
"(?<word>é+)",
|
||||
"g",
|
||||
"😀éé x",
|
||||
10,
|
||||
10,
|
||||
"",
|
||||
1,
|
||||
);
|
||||
if (
|
||||
execution?.ok !== true ||
|
||||
execution.groupCount !== 1 ||
|
||||
JSON.stringify(execution.groupNames) !== JSON.stringify([[1, "word"]]) ||
|
||||
JSON.stringify(execution.matches?.[0]?.span) !== JSON.stringify([1, 3])
|
||||
) {
|
||||
throw new Error("Ruby runtime failed its native match smoke test.");
|
||||
}
|
||||
const replacement = call(
|
||||
"replace",
|
||||
"(?<word>é+)",
|
||||
"g",
|
||||
"😀éé x",
|
||||
10,
|
||||
10,
|
||||
"[\\k<word>]",
|
||||
1_024,
|
||||
);
|
||||
if (
|
||||
replacement?.ok !== true ||
|
||||
replacement.output !== "😀[éé] x" ||
|
||||
replacement.outputTruncated !== false
|
||||
) {
|
||||
throw new Error("Ruby runtime failed its native replacement smoke test.");
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyRubyPack(directory, projectRoot = root) {
|
||||
const pack = await assertRealDirectory(directory, "Ruby engine pack");
|
||||
const names = (await readdir(pack, { withFileTypes: true }))
|
||||
.map((entry) => {
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`Ruby pack contains a non-regular entry: ${entry.name}`,
|
||||
);
|
||||
}
|
||||
return entry.name;
|
||||
})
|
||||
.sort(compareText);
|
||||
if (JSON.stringify(names) !== JSON.stringify(PACK_FILES)) {
|
||||
throw new Error(`Ruby pack must contain only: ${PACK_FILES.join(", ")}.`);
|
||||
}
|
||||
|
||||
const checksumLines = (await readFile(path.join(pack, "SHA256SUMS"), "utf8"))
|
||||
.trimEnd()
|
||||
.split("\n");
|
||||
const expectedLines = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
expectedLines.push(`${await sha256File(path.join(pack, name))} ${name}`);
|
||||
}
|
||||
if (JSON.stringify(checksumLines) !== JSON.stringify(expectedLines)) {
|
||||
throw new Error("Ruby pack checksum manifest is stale or malformed.");
|
||||
}
|
||||
|
||||
for (const [name, expected] of Object.entries(
|
||||
RUBY_ENGINE_LOCK.sourceAssets,
|
||||
)) {
|
||||
await assertHash(path.join(pack, name), expected, `Packed Ruby ${name}`);
|
||||
}
|
||||
const metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
const expected = await expectedMetadata(projectRoot, pack);
|
||||
if (JSON.stringify(metadata) !== JSON.stringify(expected)) {
|
||||
throw new Error("Ruby engine metadata does not match the pinned pack.");
|
||||
}
|
||||
await smokeRuby(pack, projectRoot);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export async function buildRubyPack({
|
||||
projectRoot = root,
|
||||
output = path.join(root, ".engine-build", "ruby"),
|
||||
force = false,
|
||||
} = {}) {
|
||||
const sources = await verifySources(projectRoot);
|
||||
const target = path.resolve(output);
|
||||
const existing = await lstat(target).catch(() => null);
|
||||
if (existing && !force) {
|
||||
throw new Error(`${target} exists; pass --force to replace it.`);
|
||||
}
|
||||
if (existing && (existing.isSymbolicLink() || !existing.isDirectory())) {
|
||||
throw new Error("Ruby pack output must be a real directory when replaced.");
|
||||
}
|
||||
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
const stage = await mkdtemp(path.join(path.dirname(target), ".ruby-pack-"));
|
||||
try {
|
||||
for (const [name, components] of Object.entries(SOURCE_FILES)) {
|
||||
await copyFile(
|
||||
path.join(sources.nodeModules, ...components),
|
||||
path.join(stage, name),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
}
|
||||
const metadata = await expectedMetadata(projectRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
{ flag: "wx", mode: 0o644 },
|
||||
);
|
||||
const checksumLines = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
checksumLines.push(
|
||||
`${await sha256File(path.join(stage, name))} ${name}`,
|
||||
);
|
||||
}
|
||||
await writeFile(
|
||||
path.join(stage, "SHA256SUMS"),
|
||||
`${checksumLines.join("\n")}\n`,
|
||||
{ flag: "wx", mode: 0o644 },
|
||||
);
|
||||
await verifyRubyPack(stage, projectRoot);
|
||||
if (existing) await rm(target, { recursive: true });
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
output: target,
|
||||
metadata: await verifyRubyPack(target, projectRoot),
|
||||
};
|
||||
}
|
||||
|
||||
export async function installRubyPack(source, projectRoot = root) {
|
||||
const sourcePath = await assertRealDirectory(source, "Ruby staged pack");
|
||||
const target = path.join(projectRoot, "public", "engines", "ruby");
|
||||
if (sourcePath === (await realpath(target).catch(() => ""))) {
|
||||
throw new Error("Ruby staged pack and installation target must differ.");
|
||||
}
|
||||
const metadata = await verifyRubyPack(sourcePath, projectRoot);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
const stage = await mkdtemp(
|
||||
path.join(path.dirname(target), ".ruby-install-"),
|
||||
);
|
||||
let backup;
|
||||
try {
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(
|
||||
path.join(sourcePath, name),
|
||||
path.join(stage, name),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
}
|
||||
await verifyRubyPack(stage, projectRoot);
|
||||
const existing = await lstat(target).catch(() => null);
|
||||
if (existing) {
|
||||
if (existing.isSymbolicLink() || !existing.isDirectory()) {
|
||||
throw new Error("Installed Ruby pack target must be a real directory.");
|
||||
}
|
||||
backup = `${target}.backup-${process.pid}-${Date.now()}`;
|
||||
await rename(target, backup);
|
||||
}
|
||||
await rename(stage, target);
|
||||
if (backup) await rm(backup, { recursive: true });
|
||||
} catch (error) {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
if (backup && !(await lstat(target).catch(() => null))) {
|
||||
await rename(backup, target);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { output: target, metadata };
|
||||
}
|
||||
|
||||
function parseArguments(arguments_) {
|
||||
let output = path.join(root, ".engine-build", "ruby");
|
||||
let force = false;
|
||||
for (let index = 0; index < arguments_.length; index += 1) {
|
||||
const argument = arguments_[index];
|
||||
if (argument === "--force") {
|
||||
if (force) throw new Error("--force was provided more than once.");
|
||||
force = true;
|
||||
} else if (argument === "--output") {
|
||||
const value = arguments_[index + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error("--output requires a directory.");
|
||||
}
|
||||
output = path.resolve(root, value);
|
||||
index += 1;
|
||||
} else {
|
||||
throw new Error(`Unknown Ruby engine-pack argument: ${argument}`);
|
||||
}
|
||||
}
|
||||
return { output, force };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const result = await buildRubyPack(parseArguments(process.argv.slice(2)));
|
||||
console.log(
|
||||
`Built verified ${result.metadata.engineVersion} pack at ${result.output}.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
await main();
|
||||
}
|
||||
74
scripts/ruby-engine-pack.test.ts
Normal file
74
scripts/ruby-engine-pack.test.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
// @vitest-environment node
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildRubyPack,
|
||||
RUBY_ENGINE_LOCK,
|
||||
verifyRubyPack,
|
||||
} from "./ruby-engine-pack.mjs";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
async function digest(file: string): Promise<string> {
|
||||
return createHash("sha256")
|
||||
.update(await readFile(file))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
describe("Ruby engine-pack gate", () => {
|
||||
it("verifies the installed minimal CRuby runtime and native smoke cases", async () => {
|
||||
await expect(
|
||||
verifyRubyPack(path.resolve("public", "engines", "ruby")),
|
||||
).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
engineVersion: "CRuby 4.0.0 Regexp",
|
||||
platform: "wasm32-wasi",
|
||||
source: expect.objectContaining({
|
||||
runtimeVariant: "minimal ruby.wasm (without bundled stdlib/debug)",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("pins exact runtime/host package and WebAssembly identities", () => {
|
||||
expect(RUBY_ENGINE_LOCK.packageVersion).toBe("2.9.3-2.9.4");
|
||||
expect(RUBY_ENGINE_LOCK.rubyVersion).toBe("4.0.0");
|
||||
expect(RUBY_ENGINE_LOCK.platform).toBe("wasm32-wasi");
|
||||
expect(RUBY_ENGINE_LOCK.npm.runtime.integrity).toMatch(/^sha512-/u);
|
||||
expect(RUBY_ENGINE_LOCK.npm.host.integrity).toMatch(/^sha512-/u);
|
||||
expect(RUBY_ENGINE_LOCK.sourceAssets["ruby.wasm"]).toHaveLength(64);
|
||||
});
|
||||
|
||||
it("builds byte-identical closed packs from the pinned npm inputs", async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "regex-ruby-pack-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const first = path.join(directory, "first");
|
||||
const second = path.join(directory, "second");
|
||||
|
||||
await buildRubyPack({ output: first });
|
||||
await buildRubyPack({ output: second });
|
||||
|
||||
for (const file of [
|
||||
"ruby.wasm",
|
||||
"engine-metadata.json",
|
||||
"SHA256SUMS",
|
||||
"LICENSE.ruby-wasm.txt",
|
||||
"NOTICE.txt",
|
||||
]) {
|
||||
expect(await digest(path.join(first, file))).toBe(
|
||||
await digest(path.join(second, file)),
|
||||
);
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
806
scripts/rust-engine-pack.mjs
Normal file
806
scripts/rust-engine-pack.mjs
Normal file
@@ -0,0 +1,806 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export const RUST_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
engineVersion: "1.13.1",
|
||||
rustcVersion: "1.97.1",
|
||||
rustcIdentity: "rustc 1.97.1 (8bab26f4f 2026-07-14)",
|
||||
rustcCommit: "8bab26f4f68e0e26f0bb7960be334d5b520ea452",
|
||||
cargoIdentity: "cargo 1.97.1 (c980f4866 2026-06-30)",
|
||||
llvmVersion: "22.1.6",
|
||||
wasmBindgenVersion: "0.2.126",
|
||||
wasmBindgenCrateChecksum:
|
||||
"4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4",
|
||||
wasmBindgenCliChecksum:
|
||||
"8e2ebf6eef87c34e662347f1dddfbea541304bb7294711ed2c2acd8746cc5b50",
|
||||
sourceDateEpoch: 1_784_146_753,
|
||||
source: Object.freeze({
|
||||
repository: "https://github.com/rust-lang/regex",
|
||||
tag: "1.13.1",
|
||||
tagObject: "7c28ee4dc4aef53571c96757f4825c51ac0c3d44",
|
||||
commit: "2b527599eb9eea0dcc288c704584f242f26a5c61",
|
||||
tree: "25bea9f6f55a4cf528afefaa00f5335c595c182c",
|
||||
crateChecksum:
|
||||
"f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d",
|
||||
license: "MIT OR Apache-2.0",
|
||||
}),
|
||||
distribution: Object.freeze({
|
||||
manifest: "https://static.rust-lang.org/dist/channel-rust-1.97.1.toml",
|
||||
manifestSha256:
|
||||
"03569b1886ceb5c05276b50c8431ab111de944cd6140fe1fa7d821dd8e0f29cf",
|
||||
rustcArchiveSha256:
|
||||
"1c441e430c1cca49dff54a8d59c41038bf6f79f7b8756596cb2f36511a015eba",
|
||||
wasmStdArchiveSha256:
|
||||
"13902d5573eeea50701d75acc774b6df2dfb4942ec88cdbe40bb07e448c307ea",
|
||||
licenseHashes: Object.freeze({
|
||||
apache:
|
||||
"074e6e32c86a4c0ef8b3ed25b721ca23aca83df277cd88106ef7177c354615ff",
|
||||
mit: "b85dcd3e453d05982552c52b5fc9e0bdd6d23c6f8e844b984a88af32570b0cc0",
|
||||
unicode:
|
||||
"f5062c9a188d81dfe66b56db4182dcf9e4b17c0d9b0d311a8e20b3a1b075c443",
|
||||
stdlibCopyright:
|
||||
"0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed",
|
||||
}),
|
||||
}),
|
||||
limits: Object.freeze({
|
||||
maximumPatternBytes: 262_144,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementBytes: 262_144,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
maximumRequestJSONBytes: 103_874_560,
|
||||
compiledSizeLimitBytes: 10_485_760,
|
||||
dfaSizeLimitBytes: 2_097_152,
|
||||
}),
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze([
|
||||
"LICENSE-Apache-2.0.txt",
|
||||
"LICENSE-MIT.txt",
|
||||
"LICENSE-Unicode-3.0.txt",
|
||||
"RUST-STDLIB-COPYRIGHT.html",
|
||||
"SHA256SUMS",
|
||||
"THIRD_PARTY_LICENSES.txt",
|
||||
"engine-metadata.json",
|
||||
"rust-regex.mjs",
|
||||
"rust-regex_bg.wasm",
|
||||
]);
|
||||
const CHECKSUM_FILES = Object.freeze(
|
||||
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
|
||||
);
|
||||
const SOURCE_FILES = Object.freeze([
|
||||
"engines/rust/Cargo.lock",
|
||||
"engines/rust/Cargo.toml",
|
||||
"engines/rust/README.md",
|
||||
"engines/rust/rust-toolchain.toml",
|
||||
"engines/rust/src/lib.rs",
|
||||
]);
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a regular file: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
function commandLabel(command, arguments_) {
|
||||
return [command, ...arguments_]
|
||||
.map((part) =>
|
||||
/^[A-Za-z0-9_./:=,+-]+$/u.test(part) ? part : JSON.stringify(part),
|
||||
)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
const detail = [output, errorOutput].filter(Boolean).join("\n").trim();
|
||||
throw new Error(
|
||||
`${commandLabel(command, arguments_)} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}${detail ? `:\n${detail}` : "."}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
function parseCargoLock(contents) {
|
||||
const packages = [];
|
||||
for (const block of contents.split("[[package]]").slice(1)) {
|
||||
const field = (name) => {
|
||||
const match = block.match(new RegExp(`^${name} = "([^"]+)"`, "mu"));
|
||||
return match?.[1];
|
||||
};
|
||||
const name = field("name");
|
||||
const version = field("version");
|
||||
if (!name || !version) {
|
||||
throw new Error("Cargo.lock contains a malformed package block.");
|
||||
}
|
||||
const source = field("source");
|
||||
const checksum = field("checksum");
|
||||
packages.push({
|
||||
name,
|
||||
version,
|
||||
...(source ? { source } : {}),
|
||||
...(checksum ? { checksum } : {}),
|
||||
});
|
||||
}
|
||||
return packages;
|
||||
}
|
||||
|
||||
async function findRegistryCrate(cargoHome, package_) {
|
||||
const registrySource = path.join(cargoHome, "registry", "src");
|
||||
const indexes = await readdir(registrySource, { withFileTypes: true });
|
||||
const matches = [];
|
||||
for (const index of indexes) {
|
||||
if (!index.isDirectory() || index.isSymbolicLink()) continue;
|
||||
const candidate = path.join(
|
||||
registrySource,
|
||||
index.name,
|
||||
`${package_.name}-${package_.version}`,
|
||||
);
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (details?.isDirectory() && !details.isSymbolicLink()) {
|
||||
matches.push(await realpath(candidate));
|
||||
}
|
||||
}
|
||||
if (matches.length !== 1) {
|
||||
throw new Error(
|
||||
`Expected one cached source directory for ${package_.name} ${package_.version}.`,
|
||||
);
|
||||
}
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
async function thirdPartyLicenses(cargoHome, packages) {
|
||||
const sections = [
|
||||
"Regex Tools Rust engine third-party source notices",
|
||||
"Generated from the checked-in Cargo.lock; package order is deterministic.",
|
||||
"",
|
||||
];
|
||||
for (const package_ of packages.filter(({ source }) => source)) {
|
||||
const crate = await findRegistryCrate(cargoHome, package_);
|
||||
const entries = (await readdir(crate, { withFileTypes: true }))
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.isFile() &&
|
||||
!entry.isSymbolicLink() &&
|
||||
/^(?:COPYING|COPYRIGHT|LICENSE|UNLICENSE)/iu.test(entry.name),
|
||||
)
|
||||
.sort((left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
sections.push(
|
||||
`===============================================================================`,
|
||||
`${package_.name} ${package_.version}`,
|
||||
`Cargo source: ${package_.source}`,
|
||||
`Cargo checksum: ${package_.checksum ?? "not recorded"}`,
|
||||
);
|
||||
if (entries.length === 0) {
|
||||
sections.push(
|
||||
"No root licence file was present in the packaged crate source.",
|
||||
"",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
sections.push(
|
||||
`-------------------------------------------------------------------------------`,
|
||||
entry.name,
|
||||
"",
|
||||
(await readFile(path.join(crate, entry.name), "utf8")).trimEnd(),
|
||||
"",
|
||||
);
|
||||
}
|
||||
}
|
||||
return `${sections.join("\n")}\n`;
|
||||
}
|
||||
|
||||
async function verifyToolchain(toolchainRootDirectory) {
|
||||
const root = await assertRealDirectory(
|
||||
toolchainRootDirectory,
|
||||
"Rust 1.97.1 toolchain",
|
||||
);
|
||||
const cargoHome = await assertRealDirectory(
|
||||
path.join(root, "cargo"),
|
||||
"Rust Cargo home",
|
||||
);
|
||||
const rustupHome = await assertRealDirectory(
|
||||
path.join(root, "rustup"),
|
||||
"Rustup home",
|
||||
);
|
||||
const toolchain = await assertRealDirectory(
|
||||
path.join(rustupHome, "toolchains", "1.97.1-x86_64-unknown-linux-gnu"),
|
||||
"Rust distribution toolchain",
|
||||
);
|
||||
const cargo = await assertRegularFile(
|
||||
path.join(toolchain, "bin", "cargo"),
|
||||
"cargo",
|
||||
);
|
||||
const rustc = await assertRegularFile(
|
||||
path.join(toolchain, "bin", "rustc"),
|
||||
"rustc",
|
||||
);
|
||||
const wasmBindgen = await assertRegularFile(
|
||||
path.join(cargoHome, "bin", "wasm-bindgen"),
|
||||
"wasm-bindgen",
|
||||
);
|
||||
for (const executable of [cargo, rustc, wasmBindgen]) {
|
||||
await access(executable, fsConstants.X_OK);
|
||||
}
|
||||
const baseEnvironment = {
|
||||
...process.env,
|
||||
CARGO_HOME: cargoHome,
|
||||
RUSTUP_HOME: rustupHome,
|
||||
PATH: [
|
||||
path.join(toolchain, "bin"),
|
||||
path.join(cargoHome, "bin"),
|
||||
process.env.PATH ?? "",
|
||||
].join(path.delimiter),
|
||||
};
|
||||
const rustcVersion = await run(rustc, ["--version", "--verbose"], {
|
||||
env: baseEnvironment,
|
||||
});
|
||||
if (
|
||||
!rustcVersion.stdout.includes(`release: ${RUST_LOCK.rustcVersion}`) ||
|
||||
!rustcVersion.stdout.includes(`commit-hash: ${RUST_LOCK.rustcCommit}`) ||
|
||||
!rustcVersion.stdout.includes(`LLVM version: ${RUST_LOCK.llvmVersion}`)
|
||||
) {
|
||||
throw new Error("The Rust engine requires pinned rustc 1.97.1.");
|
||||
}
|
||||
const cargoVersion = await run(cargo, ["--version"], {
|
||||
env: baseEnvironment,
|
||||
});
|
||||
if (cargoVersion.stdout.trim() !== RUST_LOCK.cargoIdentity) {
|
||||
throw new Error("The Rust engine requires pinned Cargo 1.97.1.");
|
||||
}
|
||||
const bindgenVersion = await run(wasmBindgen, ["--version"], {
|
||||
env: baseEnvironment,
|
||||
});
|
||||
if (
|
||||
bindgenVersion.stdout.trim() !==
|
||||
`wasm-bindgen ${RUST_LOCK.wasmBindgenVersion}`
|
||||
) {
|
||||
throw new Error("The Rust engine requires pinned wasm-bindgen 0.2.126.");
|
||||
}
|
||||
|
||||
const docs = path.join(toolchain, "share", "doc", "rust");
|
||||
const files = {
|
||||
apache: await assertRegularFile(
|
||||
path.join(docs, "licenses", "Apache-2.0.txt"),
|
||||
"Rust Apache licence",
|
||||
),
|
||||
mit: await assertRegularFile(
|
||||
path.join(docs, "licenses", "MIT.txt"),
|
||||
"Rust MIT licence",
|
||||
),
|
||||
unicode: await assertRegularFile(
|
||||
path.join(docs, "licenses", "Unicode-3.0.txt"),
|
||||
"Rust Unicode licence",
|
||||
),
|
||||
stdlibCopyright: await assertRegularFile(
|
||||
path.join(docs, "COPYRIGHT-library.html"),
|
||||
"Rust standard-library copyright notice",
|
||||
),
|
||||
};
|
||||
for (const [name, file] of Object.entries(files)) {
|
||||
if (
|
||||
(await sha256File(file)) !== RUST_LOCK.distribution.licenseHashes[name]
|
||||
) {
|
||||
throw new Error(`The pinned Rust ${name} notice does not match 1.97.1.`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
root,
|
||||
cargoHome,
|
||||
rustupHome,
|
||||
cargo,
|
||||
rustc,
|
||||
wasmBindgen,
|
||||
baseEnvironment,
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
function buildEnvironment(toolchain, repositoryRoot, targetDirectory) {
|
||||
const environment = { ...toolchain.baseEnvironment };
|
||||
for (const variable of [
|
||||
"CARGO_BUILD_RUSTC",
|
||||
"CARGO_ENCODED_RUSTFLAGS",
|
||||
"CARGO_PROFILE_RELEASE_DEBUG",
|
||||
"RUSTC",
|
||||
"RUSTDOC",
|
||||
"RUSTDOCFLAGS",
|
||||
"RUSTFLAGS",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.CARGO_INCREMENTAL = "0";
|
||||
environment.CARGO_NET_OFFLINE = "true";
|
||||
environment.CARGO_TARGET_DIR = targetDirectory;
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.REGEX_TOOLS_RUSTC_VERSION = RUST_LOCK.rustcIdentity;
|
||||
environment.RUST_BACKTRACE = "0";
|
||||
environment.RUSTUP_TOOLCHAIN = "1.97.1-x86_64-unknown-linux-gnu";
|
||||
environment.RUSTFLAGS = [
|
||||
`--remap-path-prefix=${repositoryRoot}=.`,
|
||||
`--remap-path-prefix=${toolchain.cargoHome}=./cargo-home`,
|
||||
].join(" ");
|
||||
environment.SOURCE_DATE_EPOCH = String(RUST_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceMetadata(root) {
|
||||
return Promise.all(
|
||||
SOURCE_FILES.map(async (relativePath) => {
|
||||
const contents = await readFile(path.join(root, relativePath));
|
||||
return {
|
||||
path: relativePath,
|
||||
sha256: sha256(contents),
|
||||
bytes: contents.byteLength,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fileMetadata(pack, name) {
|
||||
const contents = await readFile(path.join(pack, name));
|
||||
return { path: name, sha256: sha256(contents), bytes: contents.byteLength };
|
||||
}
|
||||
|
||||
async function lockedPackages(root) {
|
||||
const contents = await readFile(
|
||||
path.join(root, "engines", "rust", "Cargo.lock"),
|
||||
"utf8",
|
||||
);
|
||||
return parseCargoLock(contents)
|
||||
.filter(({ source }) => source)
|
||||
.map(({ name, version, checksum }) => ({ name, version, checksum }));
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "rust",
|
||||
engineName: "Rust regex crate",
|
||||
engineVersion: RUST_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
"Rust regex crate 1.13.1 compiled to WebAssembly with rustc 1.97.1",
|
||||
status: "production-native-runtime",
|
||||
bridge: {
|
||||
abiVersion: RUST_LOCK.bridgeAbi,
|
||||
offsetUnit: "utf8-byte",
|
||||
requestEncoding:
|
||||
"Exact UTF-8-bounded JSON values with worst-case escape allowance; fixed exports; base64 UTF-8 replacement output; no source evaluation",
|
||||
supportedApplicationFlags: ["g", "i", "m", "s", "U", "u", "x", "R"],
|
||||
unicode:
|
||||
"Enabled by default, matching the Rust regex crate; u explicitly affirms that default.",
|
||||
replacement:
|
||||
"Bounded streaming expansion with regex::Captures::expand grammar, parity-checked against the native API.",
|
||||
limits: { ...RUST_LOCK.limits },
|
||||
sourceFiles: await sourceMetadata(root),
|
||||
},
|
||||
source: {
|
||||
repository: RUST_LOCK.source.repository,
|
||||
tag: RUST_LOCK.source.tag,
|
||||
tagObjectSha1: RUST_LOCK.source.tagObject,
|
||||
commitSha1: RUST_LOCK.source.commit,
|
||||
treeSha1: RUST_LOCK.source.tree,
|
||||
crateChecksumSha256: RUST_LOCK.source.crateChecksum,
|
||||
license: RUST_LOCK.source.license,
|
||||
},
|
||||
toolchain: {
|
||||
rustcVersion: RUST_LOCK.rustcVersion,
|
||||
rustcIdentity: RUST_LOCK.rustcIdentity,
|
||||
rustcCommitSha1: RUST_LOCK.rustcCommit,
|
||||
llvmVersion: RUST_LOCK.llvmVersion,
|
||||
cargoIdentity: RUST_LOCK.cargoIdentity,
|
||||
target: "wasm32-unknown-unknown",
|
||||
wasmBindgenVersion: RUST_LOCK.wasmBindgenVersion,
|
||||
wasmBindgenCrateChecksumSha256: RUST_LOCK.wasmBindgenCrateChecksum,
|
||||
wasmBindgenCliChecksumSha256: RUST_LOCK.wasmBindgenCliChecksum,
|
||||
distributionManifest: RUST_LOCK.distribution.manifest,
|
||||
distributionManifestSha256: RUST_LOCK.distribution.manifestSha256,
|
||||
rustcArchiveSha256: RUST_LOCK.distribution.rustcArchiveSha256,
|
||||
wasmStdArchiveSha256: RUST_LOCK.distribution.wasmStdArchiveSha256,
|
||||
cargoLockPackages: await lockedPackages(root),
|
||||
},
|
||||
sourceDateEpoch: RUST_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(
|
||||
CHECKSUM_FILES.filter((name) => name !== "engine-metadata.json").map(
|
||||
(name) => fileMetadata(pack, name),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function checksumText(pack) {
|
||||
const lines = await Promise.all(
|
||||
CHECKSUM_FILES.map(
|
||||
async (name) => `${await sha256File(path.join(pack, name))} ${name}`,
|
||||
),
|
||||
);
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function equalJson(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
|
||||
async function smokePack(pack) {
|
||||
const moduleUrl = new URL(`file://${path.join(pack, "rust-regex.mjs")}`).href;
|
||||
const wasmPath = path.join(pack, "rust-regex_bg.wasm");
|
||||
const smoke = `
|
||||
import { readFile } from "node:fs/promises";
|
||||
const engine = await import(process.argv[1]);
|
||||
engine.initSync({
|
||||
module: await WebAssembly.compile(await readFile(process.argv[2])),
|
||||
});
|
||||
const identity = JSON.parse(engine.identity());
|
||||
if (
|
||||
identity.abi !== 1 ||
|
||||
identity.ok !== true ||
|
||||
identity.identity.engineVersion !== "1.13.1" ||
|
||||
identity.identity.rustcVersion !==
|
||||
"rustc 1.97.1 (8bab26f4f 2026-07-14)" ||
|
||||
identity.identity.selfTest !== true
|
||||
) throw new Error("Rust identity self-test failed.");
|
||||
const result = JSON.parse(engine.run(JSON.stringify({
|
||||
abi: 1,
|
||||
operation: "replace",
|
||||
pattern: "(?P<word>😀+)",
|
||||
flags: "g",
|
||||
options: {},
|
||||
subject: "x😀😀",
|
||||
scanAll: false,
|
||||
maximumMatches: 10,
|
||||
maximumCaptureRows: 10,
|
||||
replacement: "\${word}-$$",
|
||||
maximumOutputBytes: 100,
|
||||
})));
|
||||
if (
|
||||
result.ok !== true ||
|
||||
result.matches[0].span[0] !== 1 ||
|
||||
result.matches[0].span[1] !== 9 ||
|
||||
Buffer.from(result.outputBase64, "base64").toString("utf8") !== "x😀😀-$"
|
||||
) throw new Error("Rust execution self-test failed.");
|
||||
`;
|
||||
await run(process.execPath, [
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
smoke,
|
||||
moduleUrl,
|
||||
wasmPath,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function verifyRustPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const pack = await assertRealDirectory(packDirectory, "Rust engine pack");
|
||||
const entries = (await readdir(pack, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== PACK_FILES.length ||
|
||||
entries.some(
|
||||
(entry, index) =>
|
||||
entry.name !== PACK_FILES[index] ||
|
||||
!entry.isFile() ||
|
||||
entry.isSymbolicLink(),
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`The Rust pack must contain only: ${PACK_FILES.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
const licenseFiles = {
|
||||
"LICENSE-Apache-2.0.txt": RUST_LOCK.distribution.licenseHashes.apache,
|
||||
"LICENSE-MIT.txt": RUST_LOCK.distribution.licenseHashes.mit,
|
||||
"LICENSE-Unicode-3.0.txt": RUST_LOCK.distribution.licenseHashes.unicode,
|
||||
"RUST-STDLIB-COPYRIGHT.html":
|
||||
RUST_LOCK.distribution.licenseHashes.stdlibCopyright,
|
||||
};
|
||||
for (const [name, digest] of Object.entries(licenseFiles)) {
|
||||
if ((await sha256File(path.join(pack, name))) !== digest) {
|
||||
throw new Error(`${name} does not match the Rust 1.97.1 distribution.`);
|
||||
}
|
||||
}
|
||||
const wasm = await readFile(path.join(pack, "rust-regex_bg.wasm"));
|
||||
if (
|
||||
wasm.byteLength < 100_000 ||
|
||||
wasm.subarray(0, 4).toString("hex") !== "0061736d"
|
||||
) {
|
||||
throw new Error("The staged Rust WebAssembly module is invalid.");
|
||||
}
|
||||
const moduleText = await readFile(path.join(pack, "rust-regex.mjs"), "utf8");
|
||||
if (
|
||||
moduleText.includes("sourceMappingURL") ||
|
||||
moduleText.includes("/mnt/") ||
|
||||
moduleText.includes("/home/") ||
|
||||
!moduleText.includes("export function identity") ||
|
||||
!moduleText.includes("export function run") ||
|
||||
!moduleText.includes("export { initSync")
|
||||
) {
|
||||
throw new Error(
|
||||
"rust-regex.mjs leaks a build path, has a source map, or lacks bridge exports.",
|
||||
);
|
||||
}
|
||||
const packages = await lockedPackages(repositoryRoot);
|
||||
const notices = await readFile(
|
||||
path.join(pack, "THIRD_PARTY_LICENSES.txt"),
|
||||
"utf8",
|
||||
);
|
||||
for (const package_ of packages) {
|
||||
if (
|
||||
!notices.includes(`${package_.name} ${package_.version}\n`) ||
|
||||
!notices.includes(`Cargo checksum: ${package_.checksum}`)
|
||||
) {
|
||||
throw new Error(
|
||||
`Third-party notices omit ${package_.name} ${package_.version}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("Rust engine-metadata.json is invalid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (!equalJson(metadata, await expectedMetadata(repositoryRoot, pack))) {
|
||||
throw new Error(
|
||||
"The Rust metadata does not match the pinned build contract.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
|
||||
(await checksumText(pack))
|
||||
) {
|
||||
throw new Error("The Rust SHA256SUMS file is incorrect.");
|
||||
}
|
||||
await smokePack(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function replaceDirectory(stage, target, label) {
|
||||
const details = await lstat(target).catch(() => null);
|
||||
if (details && (!details.isDirectory() || details.isSymbolicLink())) {
|
||||
throw new Error(`${label} must be a real directory.`);
|
||||
}
|
||||
if (!details) {
|
||||
await rename(stage, target);
|
||||
return;
|
||||
}
|
||||
const backup = `${target}.replaced-${process.pid}`;
|
||||
await rename(target, backup);
|
||||
try {
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rename(backup, target);
|
||||
throw error;
|
||||
}
|
||||
await rm(backup, { recursive: true });
|
||||
}
|
||||
|
||||
async function bindgen(toolchain, rawWasm, output) {
|
||||
await mkdir(output, { recursive: true });
|
||||
await run(
|
||||
toolchain.wasmBindgen,
|
||||
[
|
||||
"--target",
|
||||
"web",
|
||||
"--no-typescript",
|
||||
"--out-name",
|
||||
"rust-regex",
|
||||
"--out-dir",
|
||||
output,
|
||||
rawWasm,
|
||||
],
|
||||
{ env: toolchain.baseEnvironment },
|
||||
);
|
||||
await rename(
|
||||
path.join(output, "rust-regex.js"),
|
||||
path.join(output, "rust-regex.mjs"),
|
||||
);
|
||||
}
|
||||
|
||||
export async function buildRustPack(
|
||||
root,
|
||||
toolchainRootDirectory,
|
||||
outputDirectory,
|
||||
) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const engine = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "engines", "rust"),
|
||||
"Rust engine source",
|
||||
);
|
||||
const toolchain = await verifyToolchain(toolchainRootDirectory);
|
||||
const lockPackages = parseCargoLock(
|
||||
await readFile(path.join(engine, "Cargo.lock"), "utf8"),
|
||||
);
|
||||
const regexPackage = lockPackages.find(
|
||||
({ name, version }) => name === "regex" && version === "1.13.1",
|
||||
);
|
||||
const bindgenPackage = lockPackages.find(
|
||||
({ name, version }) =>
|
||||
name === "wasm-bindgen" && version === RUST_LOCK.wasmBindgenVersion,
|
||||
);
|
||||
if (
|
||||
regexPackage?.checksum !== RUST_LOCK.source.crateChecksum ||
|
||||
bindgenPackage?.checksum !== RUST_LOCK.wasmBindgenCrateChecksum
|
||||
) {
|
||||
throw new Error("Cargo.lock does not contain the pinned engine crates.");
|
||||
}
|
||||
|
||||
const buildParent = path.join(repositoryRoot, ".engine-build");
|
||||
await mkdir(buildParent, { recursive: true });
|
||||
const temporary = await mkdtemp(path.join(buildParent, "rust-build-"));
|
||||
try {
|
||||
const generated = [];
|
||||
for (const label of ["first", "second"]) {
|
||||
const target = path.join(temporary, `${label}-target`);
|
||||
const environment = buildEnvironment(toolchain, repositoryRoot, target);
|
||||
await run(
|
||||
toolchain.cargo,
|
||||
[
|
||||
"build",
|
||||
"--locked",
|
||||
"--offline",
|
||||
"--release",
|
||||
"--target",
|
||||
"wasm32-unknown-unknown",
|
||||
],
|
||||
{ cwd: engine, env: environment },
|
||||
);
|
||||
const rawWasm = await assertRegularFile(
|
||||
path.join(
|
||||
target,
|
||||
"wasm32-unknown-unknown",
|
||||
"release",
|
||||
"regex_tools_rust_engine.wasm",
|
||||
),
|
||||
"compiled Rust WebAssembly",
|
||||
);
|
||||
const output = path.join(temporary, `${label}-bindgen`);
|
||||
await bindgen(toolchain, rawWasm, output);
|
||||
generated.push(output);
|
||||
}
|
||||
for (const name of ["rust-regex.mjs", "rust-regex_bg.wasm"]) {
|
||||
if (
|
||||
(await sha256File(path.join(generated[0], name))) !==
|
||||
(await sha256File(path.join(generated[1], name)))
|
||||
) {
|
||||
throw new Error(
|
||||
`Two pinned Rust builds produced different ${name} hashes.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const target = path.resolve(
|
||||
repositoryRoot,
|
||||
outputDirectory ?? path.join(".engine-build", "rust"),
|
||||
);
|
||||
const stage = await mkdtemp(path.join(buildParent, "rust-pack-"));
|
||||
await copyFile(
|
||||
toolchain.files.apache,
|
||||
path.join(stage, "LICENSE-Apache-2.0.txt"),
|
||||
);
|
||||
await copyFile(toolchain.files.mit, path.join(stage, "LICENSE-MIT.txt"));
|
||||
await copyFile(
|
||||
toolchain.files.unicode,
|
||||
path.join(stage, "LICENSE-Unicode-3.0.txt"),
|
||||
);
|
||||
await copyFile(
|
||||
toolchain.files.stdlibCopyright,
|
||||
path.join(stage, "RUST-STDLIB-COPYRIGHT.html"),
|
||||
);
|
||||
await copyFile(
|
||||
path.join(generated[0], "rust-regex.mjs"),
|
||||
path.join(stage, "rust-regex.mjs"),
|
||||
);
|
||||
await copyFile(
|
||||
path.join(generated[0], "rust-regex_bg.wasm"),
|
||||
path.join(stage, "rust-regex_bg.wasm"),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(stage, "THIRD_PARTY_LICENSES.txt"),
|
||||
await thirdPartyLicenses(toolchain.cargoHome, lockPackages),
|
||||
"utf8",
|
||||
);
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(stage, "SHA256SUMS"), await checksumText(stage));
|
||||
await verifyRustPack(stage, repositoryRoot);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
await replaceDirectory(stage, target, "Rust staged pack");
|
||||
return { metadata, output: target };
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function installRustPack(sourceDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const source = await assertRealDirectory(sourceDirectory, "Rust staged pack");
|
||||
const metadata = await verifyRustPack(source, repositoryRoot);
|
||||
const engineParent = path.join(repositoryRoot, "public", "engines");
|
||||
await mkdir(engineParent, { recursive: true });
|
||||
const stage = await mkdtemp(path.join(engineParent, "rust-install-"));
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(path.join(source, name), path.join(stage, name));
|
||||
}
|
||||
await verifyRustPack(stage, repositoryRoot);
|
||||
const output = path.join(engineParent, "rust");
|
||||
await replaceDirectory(stage, output, "Installed Rust engine pack");
|
||||
return { metadata, output };
|
||||
}
|
||||
@@ -1,28 +1,89 @@
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { verifyCppPack } from "./cpp-engine-pack.mjs";
|
||||
import { verifyDotNetPack } from "./dotnet-engine-pack.mjs";
|
||||
import { verifyGoPack } from "./go-engine-pack.mjs";
|
||||
import { verifyJavaPack } from "./java-engine-pack.mjs";
|
||||
import { verifyPerlPack } from "./perl-engine-pack.mjs";
|
||||
import { verifyPcre2Pack } from "./pcre2-engine-pack.mjs";
|
||||
import { verifyPhpPack } from "./php-engine-pack.mjs";
|
||||
import { verifyPythonPack } from "./python-engine-pack.mjs";
|
||||
import { verifyRubyPack } from "./ruby-engine-pack.mjs";
|
||||
import { verifyRustPack } from "./rust-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
|
||||
const enginePacks = [
|
||||
{
|
||||
id: "cpp",
|
||||
argument: "--cpp-pack",
|
||||
verify: (pack) => verifyCppPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "dotnet",
|
||||
argument: "--dotnet-pack",
|
||||
verify: (pack) => verifyDotNetPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "go",
|
||||
argument: "--go-pack",
|
||||
verify: (pack) => verifyGoPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "java",
|
||||
argument: "--java-pack",
|
||||
verify: (pack) => verifyJavaPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "pcre2",
|
||||
argument: "--pcre2-pack",
|
||||
verify: (pack) => verifyPcre2Pack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "perl",
|
||||
argument: "--perl-pack",
|
||||
verify: (pack) => verifyPerlPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "php",
|
||||
argument: "--php-pack",
|
||||
verify: (pack) => verifyPhpPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "python",
|
||||
argument: "--python-pack",
|
||||
verify: (pack) => verifyPythonPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "ruby",
|
||||
argument: "--ruby-pack",
|
||||
verify: (pack) => verifyRubyPack(pack, root),
|
||||
},
|
||||
{
|
||||
id: "rust",
|
||||
argument: "--rust-pack",
|
||||
verify: (pack) => verifyRustPack(pack, root),
|
||||
},
|
||||
];
|
||||
|
||||
if (arguments_.length > 0) {
|
||||
const verifiers = new Map([
|
||||
["--java-pack", verifyJavaPack],
|
||||
["--pcre2-pack", verifyPcre2Pack],
|
||||
["--python-pack", verifyPythonPack],
|
||||
]);
|
||||
const verifier = verifiers.get(arguments_[0]);
|
||||
if (arguments_.length !== 2 || !verifier) {
|
||||
const selected = enginePacks.find(
|
||||
({ argument }) => argument === arguments_[0],
|
||||
);
|
||||
if (arguments_.length !== 2 || !selected) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/verify-engine-assets.mjs [--java-pack|--pcre2-pack|--python-pack PATH]",
|
||||
`Usage: node scripts/verify-engine-assets.mjs [${enginePacks
|
||||
.map(({ argument }) => argument)
|
||||
.join("|")} PATH]`,
|
||||
);
|
||||
}
|
||||
const pack = path.resolve(root, arguments_[1]);
|
||||
const metadata = await verifier(pack, root);
|
||||
console.log(`Verified staged ${metadata.engineVersion} engine pack.`);
|
||||
const metadata = await selected.verify(pack);
|
||||
console.log(
|
||||
`Verified staged ${metadata.engineVersion ?? metadata.semanticIdentity} ${selected.id} engine pack.`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -36,35 +97,58 @@ const entries = (await readdir(engineDirectory, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
const expectedEntries = [
|
||||
{ name: "README.md", directory: false },
|
||||
...enginePacks.map(({ id }) => ({ name: id, directory: true })),
|
||||
].sort((left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== 4 ||
|
||||
!entries[0]?.isFile() ||
|
||||
entries[0].name !== "README.md" ||
|
||||
!entries[1]?.isDirectory() ||
|
||||
entries[1].name !== "java" ||
|
||||
!entries[2]?.isDirectory() ||
|
||||
entries[2].name !== "pcre2" ||
|
||||
!entries[3]?.isDirectory() ||
|
||||
entries[3].name !== "python"
|
||||
entries.length !== expectedEntries.length ||
|
||||
entries.some((entry, index) => {
|
||||
const expected = expectedEntries[index];
|
||||
return (
|
||||
!expected ||
|
||||
entry.name !== expected.name ||
|
||||
(expected.directory ? !entry.isDirectory() : !entry.isFile())
|
||||
);
|
||||
})
|
||||
) {
|
||||
throw new Error(
|
||||
"The production build must contain only README.md and the declared Java, PCRE2 and Python packs.",
|
||||
`The production build must contain only README.md and the declared engine packs: ${enginePacks
|
||||
.map(({ id }) => id)
|
||||
.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const notice = await readFile(path.join(engineDirectory, "README.md"), "utf8");
|
||||
if (
|
||||
!notice.includes("native `RegExp`") ||
|
||||
!notice.includes("PCRE2 10.47") ||
|
||||
!notice.includes("Pyodide 314.0.3") ||
|
||||
!notice.includes("TeaVM 0.15.0")
|
||||
) {
|
||||
throw new Error("The engine asset notice does not describe this release.");
|
||||
for (const identity of [
|
||||
"native `RegExp`",
|
||||
"PCRE2 10.47",
|
||||
"PHP 8.5.8",
|
||||
"Perl 5.28.1",
|
||||
"Pyodide 314.0.3",
|
||||
"CRuby 4.0.0",
|
||||
"TeaVM 0.15.0",
|
||||
"Emscripten 6.0.4",
|
||||
"Go 1.26.5",
|
||||
"Rust `regex` crate 1.13.1",
|
||||
".NET 10.0.10",
|
||||
"Scala",
|
||||
]) {
|
||||
if (!notice.includes(identity)) {
|
||||
throw new Error(
|
||||
`The engine asset notice does not describe ${identity} in this release.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const engine of enginePacks) {
|
||||
await engine.verify(path.join(engineDirectory, engine.id));
|
||||
}
|
||||
await verifyPcre2Pack(path.join(engineDirectory, "pcre2"), root);
|
||||
await verifyPythonPack(path.join(engineDirectory, "python"), root);
|
||||
await verifyJavaPack(path.join(engineDirectory, "java"), root);
|
||||
|
||||
console.log(
|
||||
"Verified native ECMAScript and bundled PCRE2, CPython/Pyodide and TeaVM Java engine assets.",
|
||||
`Verified native ECMAScript plus ${enginePacks
|
||||
.map(({ id }) => id)
|
||||
.join(", ")} checked-in engine packs.`,
|
||||
);
|
||||
|
||||
14
scripts/verify-go-engine.mjs
Normal file
14
scripts/verify-go-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { verifyGoPack } from "./go-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/verify-go-engine.mjs [public/engines/go]",
|
||||
);
|
||||
}
|
||||
const pack = path.resolve(root, arguments_[0] ?? "public/engines/go");
|
||||
const metadata = await verifyGoPack(pack, root);
|
||||
console.log(`Verified ${metadata.semanticIdentity}.`);
|
||||
14
scripts/verify-rust-engine.mjs
Normal file
14
scripts/verify-rust-engine.mjs
Normal file
@@ -0,0 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { verifyRustPack } from "./rust-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/verify-rust-engine.mjs [public/engines/rust]",
|
||||
);
|
||||
}
|
||||
const pack = path.resolve(root, arguments_[0] ?? "public/engines/rust");
|
||||
const metadata = await verifyRustPack(pack, root);
|
||||
console.log(`Verified ${metadata.semanticIdentity}.`);
|
||||
Reference in New Issue
Block a user