import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile, } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { verifyChecksummedEnginePack, writeEngineChecksums, } from "./checksummed-engine-pack.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); export const CPP_LOCK = Object.freeze({ engineVersion: "Emscripten 6.0.4 libc++ std::wregex", emscriptenVersion: "6.0.4", emscriptenCommit: "fe5be6afdff43ad58860d821fcc8572a23f92d19", emsdkCommit: "224ec5f9f2f72f09f9ce0e26d66bae7dbd8b692f", releaseCommit: "b23272fac5d05617cd36dc451356dca0f79bf22d", llvmVersion: "24.0.0git", llvmCommit: "c050c487e9edb97ef44f53cb29fe1d8bcddb8f76", sourceDateEpoch: 1_785_110_400, bridgeAbi: 1, legalFiles: Object.freeze({ "LICENSE-Emscripten.txt": "620a78084fc7ca97c0b5dea9abf891f3ffcadfdbf305276f099c9c4e12fc1d86", "LICENSE-compiler-rt.txt": "1a8f1058753f1ba890de984e48f0242a3a5c29a6a8f2ed9fd813f36985387e8d", "LICENSE-libcxx.txt": "539dd7aed86e8a4f12cbdd0e6c50c189c7d74847e4fecc64ce2c6ee3a01da38b", "LICENSE-libcxxabi.txt": "e2b35be49f7284a45b7baca8fc7b3ab7440e7902392b2528a457816b5bb2a15c", "LICENSE-libunwind.txt": "b5efebcaca80879234098e52d1725e6d9eb8fb96a19fce625d39184b705f7b6d", "LICENSE-musl.txt": "b870108ec5e7790e9f9919064f1b9421d62d5f9b0e6c230c6adf7ea2da62e97b", }), sourceEvidence: Object.freeze({ "emscripten-revision.txt": "33d347ba6aa1de448bc1727d46912c1abcbddac9a32dcf4794d7098dea16e1ad", "emscripten-version.txt": "1ae2390b7da62b8488a9f596a39ed978b22356de0f267db9ae9dd43d0d3faa6e", "system/lib/compiler-rt/README.md": "50648d4da5c5c88468da064bd9ba29d68d47597607852330423da0a1cf987946", "system/lib/libc/README.md": "69aeb7c1991af58c547f8da100c57ff38f53c4015f4df579731c5abab84654b9", "system/lib/libcxx/README.md": "da90136512d04c06e2bd27ef18d71be34bfc64c06bc29b12ddcf66bb76ae193b", "system/lib/libcxxabi/README.md": "dcfec7b2c2987ce00e513cac89933f4d30329070570c4efdb4d1d00297182736", "system/lib/libunwind/README.md": "26b3b06b2640cee75b2cf4e01b0956a0fd6cc5ca2054998e18d66ff060762410", "system/lib/emmalloc.c": "00eb577cbef51a3359c309f45a7c99a814b8c6151ab75c9a3c78a56f0169146b", }), linkedArchives: Object.freeze([ "libc++.a", "libc++abi.a", "libc.a", "libclang_rt.builtins.a", "libembind-rtti.a", "libemmalloc.a", "libnoexit.a", "libstubs.a", ]), }); const SOURCE_FILES = Object.freeze([ "engines/cpp/README.md", "engines/cpp/cpp_regex_bridge.cpp", "scripts/cpp-engine-pack.mjs", ]); const PACK_FILES = Object.freeze( [ ...Object.keys(CPP_LOCK.legalFiles), "SHA256SUMS", "SOURCE-MANIFEST.json", "cpp-regex.mjs", "cpp-regex.wasm", "engine-metadata.json", ].sort(), ); const SOURCE_COMPONENTS = Object.freeze([ Object.freeze({ id: "emscripten-runtime", version: CPP_LOCK.emscriptenVersion, linked: true, linkedArchives: ["libembind-rtti.a", "libnoexit.a", "libstubs.a"], sourcePath: "src, system/lib and tools", sourceEvidencePath: "emscripten-revision.txt", license: "MIT OR NCSA", legalFiles: ["LICENSE-Emscripten.txt"], evidence: "Emscripten-generated JavaScript/runtime imports plus exact linker-trace archive members", }), Object.freeze({ id: "musl", version: "1.2.6 with Emscripten changes", linked: true, linkedArchives: ["libc.a"], sourcePath: "system/lib/libc", sourceEvidencePath: "system/lib/libc/README.md", license: "MIT and file-level compatible terms", legalFiles: ["LICENSE-musl.txt"], evidence: "exact linker trace selects members of libc.a", }), Object.freeze({ id: "emmalloc", version: `Emscripten ${CPP_LOCK.emscriptenVersion} snapshot`, linked: true, linkedArchives: ["libemmalloc.a"], sourcePath: "system/lib/emmalloc.c", sourceEvidencePath: "system/lib/emmalloc.c", license: "MIT OR NCSA", legalFiles: ["LICENSE-Emscripten.txt"], evidence: "the build fixes -sMALLOC=emmalloc and the exact linker trace selects emmalloc.o and sbrk.o", }), Object.freeze({ id: "compiler-rt", version: "22.1.8 with Emscripten changes", linked: true, linkedArchives: ["libclang_rt.builtins.a"], sourcePath: "system/lib/compiler-rt", sourceEvidencePath: "system/lib/compiler-rt/README.md", license: "Apache-2.0 WITH LLVM-exception and legacy component terms", legalFiles: ["LICENSE-compiler-rt.txt"], evidence: "exact linker trace selects compiler-rt builtins, including Emscripten exception support", }), Object.freeze({ id: "libc++", version: "21.1.8 with Emscripten changes", linked: true, linkedArchives: ["libc++.a"], sourcePath: "system/lib/libcxx", sourceEvidencePath: "system/lib/libcxx/README.md", license: "Apache-2.0 WITH LLVM-exception and legacy component terms", legalFiles: ["LICENSE-libcxx.txt"], evidence: "exact linker trace selects libc++ regex, locale, string, exception and support objects", }), Object.freeze({ id: "libc++abi", version: "21.1.8 with Emscripten changes", linked: true, linkedArchives: ["libc++abi.a"], sourcePath: "system/lib/libcxxabi", sourceEvidencePath: "system/lib/libcxxabi/README.md", license: "Apache-2.0 WITH LLVM-exception and legacy component terms", legalFiles: ["LICENSE-libcxxabi.txt"], evidence: "exact linker trace selects the Emscripten C++ exception ABI and type-information objects", }), Object.freeze({ id: "libunwind", version: "22.1.8 with Emscripten changes", linked: false, linkedArchives: [], consideredArchive: "libunwind.a", sourcePath: "system/lib/libunwind", sourceEvidencePath: "system/lib/libunwind/README.md", license: "Apache-2.0 WITH LLVM-exception and legacy component terms", legalFiles: ["LICENSE-libunwind.txt"], evidence: "audited exclusion: the exact JavaScript-exception linker trace selects no libunwind.a member; its legal text is retained to make that boundary reviewable", }), ]); function sha256(value) { return createHash("sha256").update(value).digest("hex"); } async function fileRecord(directory, name) { const value = await readFile(path.join(directory, name)); return { path: name, sha256: sha256(value), bytes: value.byteLength }; } async function run(command, arguments_, options = {}) { const child = spawn(command, arguments_, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"], }); const stdout = []; const stderr = []; child.stdout.on("data", (chunk) => stdout.push(chunk)); child.stderr.on("data", (chunk) => stderr.push(chunk)); const result = await new Promise((resolve, reject) => { child.once("error", reject); child.once("close", (code, signal) => resolve({ code, signal })); }); const output = Buffer.concat(stdout).toString("utf8"); const errorOutput = Buffer.concat(stderr).toString("utf8"); if (result.code !== 0) { throw new Error( `${command} failed with ${ result.signal ? `signal ${result.signal}` : `exit code ${result.code}` }:\n${[output, errorOutput].filter(Boolean).join("\n").trim()}`, ); } return { stdout: output, stderr: errorOutput }; } async function exactEmpp(candidate) { const details = await lstat(candidate).catch(() => null); if (!details?.isFile() || details.isSymbolicLink()) { throw new Error(`em++ must be a regular file: ${candidate}`); } const executable = await realpath(candidate); const version = await run(executable, ["--version"]); if ( !version.stdout.startsWith( `emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) ${CPP_LOCK.emscriptenVersion} (${CPP_LOCK.emscriptenCommit})`, ) ) { throw new Error( `The C++ pack requires Emscripten ${CPP_LOCK.emscriptenVersion} at commit ${CPP_LOCK.emscriptenCommit}.`, ); } const upstreamRoot = path.dirname(path.dirname(executable)); const sourceRoot = path.join(upstreamRoot, "emscripten"); const legalFiles = { "LICENSE-Emscripten.txt": path.join(sourceRoot, "LICENSE"), "LICENSE-compiler-rt.txt": path.join( sourceRoot, "system", "lib", "compiler-rt", "LICENSE.TXT", ), "LICENSE-libcxx.txt": path.join( sourceRoot, "system", "lib", "libcxx", "LICENSE.TXT", ), "LICENSE-libcxxabi.txt": path.join( sourceRoot, "system", "lib", "libcxxabi", "LICENSE.TXT", ), "LICENSE-libunwind.txt": path.join( sourceRoot, "system", "lib", "libunwind", "LICENSE.TXT", ), "LICENSE-musl.txt": path.join( sourceRoot, "system", "lib", "libc", "musl", "COPYRIGHT", ), }; for (const [name, file] of Object.entries(legalFiles)) { const licenseDetails = await lstat(file).catch(() => null); if (!licenseDetails?.isFile() || licenseDetails.isSymbolicLink()) { throw new Error(`Required toolchain licence is missing: ${file}`); } if (sha256(await readFile(file)) !== CPP_LOCK.legalFiles[name]) { throw new Error( `The pinned Emscripten toolchain legal file drifted: ${name}.`, ); } } for (const [name, digest] of Object.entries(CPP_LOCK.sourceEvidence)) { const file = path.join(sourceRoot, name); const sourceDetails = await lstat(file).catch(() => null); if (!sourceDetails?.isFile() || sourceDetails.isSymbolicLink()) { throw new Error(`Required toolchain source evidence is missing: ${file}`); } if (sha256(await readFile(file)) !== digest) { throw new Error( `The pinned Emscripten source evidence drifted: ${name}.`, ); } } const clang = await realpath(path.join(upstreamRoot, "bin", "clang++")); const clangDetails = await lstat(clang).catch(() => null); if (!clangDetails?.isFile()) { throw new Error(`Required pinned clang++ is missing: ${clang}`); } const clangVersion = await run(clang, ["--version"]); if ( !clangVersion.stdout.startsWith(`clang version ${CPP_LOCK.llvmVersion} `) || !clangVersion.stdout.includes(CPP_LOCK.llvmCommit) ) { throw new Error( `The C++ pack requires LLVM ${CPP_LOCK.llvmVersion} at commit ${CPP_LOCK.llvmCommit}.`, ); } return { executable, legalFiles }; } function buildEnvironment() { const environment = { ...process.env }; for (const variable of [ "CFLAGS", "CPPFLAGS", "CXXFLAGS", "EMCC_CFLAGS", "EM_CONFIG", ]) { delete environment[variable]; } environment.LANG = "C"; environment.LC_ALL = "C"; environment.SOURCE_DATE_EPOCH = String(CPP_LOCK.sourceDateEpoch); environment.TZ = "UTC"; return environment; } async function sourceRecords(projectRoot) { return Promise.all( SOURCE_FILES.map(async (name) => { const value = await readFile(path.join(projectRoot, name)); return { path: name, sha256: sha256(value), bytes: value.byteLength }; }), ); } function linkedArchives(buildOutput) { const archives = new Set(); for (const match of buildOutput.matchAll( /(?:^|\n)[^\n]*[\\/](lib[^/\\()\s]+\.a)\([^)\n]+\)/gu, )) { archives.add(match[1]); } return [...archives].sort(); } function assertLinkedArchives(buildOutput) { const actual = linkedArchives(buildOutput); if ( actual.length !== CPP_LOCK.linkedArchives.length || actual.some((name, index) => name !== CPP_LOCK.linkedArchives[index]) || /[\\/]libunwind\.a\(/u.test(buildOutput) ) { throw new Error( `C++ exact linker inventory drifted: ${actual.join(", ") || "(none)"}.`, ); } return actual; } async function expectedSourceManifest(pack) { const wasm = await fileRecord(pack, "cpp-regex.wasm"); const legalFiles = await Promise.all( Object.keys(CPP_LOCK.legalFiles).map((name) => fileRecord(pack, name)), ); return { schemaVersion: 1, engine: "cpp", scope: { asset: wasm.path, assetSha256: wasm.sha256, assetBytes: wasm.bytes, purpose: "Complete linked-component, preferred-source and legal inventory for the exact C++ WebAssembly module", }, sourceSnapshot: { project: "Emscripten", version: CPP_LOCK.emscriptenVersion, tag: CPP_LOCK.emscriptenVersion, commitSha1: CPP_LOCK.emscriptenCommit, repository: "https://github.com/emscripten-core/emscripten", preferredSourceUrl: `https://github.com/emscripten-core/emscripten/tree/${CPP_LOCK.emscriptenCommit}`, archiveUrl: `https://github.com/emscripten-core/emscripten/archive/${CPP_LOCK.emscriptenCommit}.tar.gz`, emsdk: { tag: CPP_LOCK.emscriptenVersion, commitSha1: CPP_LOCK.emsdkCommit, releaseCommitSha1: CPP_LOCK.releaseCommit, repository: "https://github.com/emscripten-core/emsdk", }, llvmToolchain: { version: CPP_LOCK.llvmVersion, commitSha1: CPP_LOCK.llvmCommit, repository: "https://github.com/llvm/llvm-project", }, }, buildSelection: { exceptionMode: "Emscripten JavaScript exceptions (-fexceptions)", allocator: "emmalloc (-sMALLOC=emmalloc)", exactLinkedArchives: [...CPP_LOCK.linkedArchives], inventoryMethod: "the pack builder adds wasm-ld --trace, accepts this exact archive set and fails on any addition, removal or libunwind selection", }, sourceEvidence: Object.entries(CPP_LOCK.sourceEvidence).map( ([sourcePath, sourceSha256]) => ({ sourcePath, sourceSha256 }), ), linkedComponents: SOURCE_COMPONENTS, legalFiles, }; } async function writeSourceManifest(pack) { await writeFile( path.join(pack, "SOURCE-MANIFEST.json"), `${JSON.stringify(await expectedSourceManifest(pack), null, 2)}\n`, "utf8", ); } async function writeMetadata(pack, projectRoot) { const metadata = { schemaVersion: 1, engine: "cpp", engineName: "C++ std::wregex (Emscripten libc++)", engineVersion: CPP_LOCK.engineVersion, semanticIdentity: "Emscripten 6.0.4 libc++ std::wregex; C++ modified ECMAScript grammar by default", status: "production-native-runtime", bridge: { abiVersion: CPP_LOCK.bridgeAbi, nativeOffsetUnit: "Unicode code points (32-bit wchar_t)", requestEncoding: "structured Embind values; no user source evaluation", dynamicExecutionDisabled: true, replacement: { semantics: "libc++ match_results::format default grammar", implementation: "bounded streaming expansion without materializing an unbounded native format result", paritySelfTest: "load-time fixtures compare the bounded expander with libc++ match_results::format", }, supportedApplicationFlags: ["g", "i", "n", "o", "c"], grammars: ["ECMAScript", "basic", "extended", "awk", "grep", "egrep"], limits: { maximumPatternUtf16: 65_536, maximumSubjectBytes: 16_777_216, maximumReplacementUtf16: 65_536, maximumMatches: 10_000, maximumCaptureRows: 100_000, maximumCaptureGroups: 1_000, maximumOutputBytes: 67_108_864, }, sourceFiles: await sourceRecords(projectRoot), }, source: { project: "Emscripten system runtime and LLVM libc++", repository: "https://github.com/emscripten-core/emscripten", tag: CPP_LOCK.emscriptenVersion, commitSha1: CPP_LOCK.emscriptenCommit, preferredSourceUrl: `https://github.com/emscripten-core/emscripten/tree/${CPP_LOCK.emscriptenCommit}`, sourceManifest: "SOURCE-MANIFEST.json", license: "MIT OR NCSA AND MIT AND Apache-2.0 WITH LLVM-exception and legacy component terms", }, toolchain: { version: CPP_LOCK.emscriptenVersion, commitSha1: CPP_LOCK.emscriptenCommit, emsdkCommitSha1: CPP_LOCK.emsdkCommit, releaseCommitSha1: CPP_LOCK.releaseCommit, llvmVersion: CPP_LOCK.llvmVersion, llvmCommitSha1: CPP_LOCK.llvmCommit, target: "wasm32-unknown-emscripten", linkedArchives: [...CPP_LOCK.linkedArchives], command: "em++ -std=c++20 -O3 -fexceptions --bind -sMODULARIZE=1 -sEXPORT_ES6=1 -sMALLOC=emmalloc -sDYNAMIC_EXECUTION=0 -Wl,--trace", }, licensing: { sourceManifest: "SOURCE-MANIFEST.json", linkedComponentCount: SOURCE_COMPONENTS.filter( (component) => component.linked, ).length, auditedExclusions: ["libunwind"], independentlyPinnedLegalFiles: Object.keys(CPP_LOCK.legalFiles).length, }, sourceDateEpoch: CPP_LOCK.sourceDateEpoch, files: await Promise.all( [ ...Object.keys(CPP_LOCK.legalFiles), "SOURCE-MANIFEST.json", "cpp-regex.mjs", "cpp-regex.wasm", ].map((name) => fileRecord(pack, name)), ), }; await writeFile( path.join(pack, "engine-metadata.json"), `${JSON.stringify(metadata, null, 2)}\n`, "utf8", ); await writeEngineChecksums(pack); return metadata; } export async function verifyCppPack(pack, projectRoot = root) { const metadata = await verifyChecksummedEnginePack(pack, "cpp"); const files = (await readdir(pack)).sort(); const expectedFiles = [...PACK_FILES].sort(); let sourceManifest; try { sourceManifest = JSON.parse( await readFile(path.join(pack, "SOURCE-MANIFEST.json"), "utf8"), ); } catch (error) { throw new Error("C++ SOURCE-MANIFEST.json is missing or invalid.", { cause: error, }); } if ( files.length !== expectedFiles.length || files.some((name, index) => name !== expectedFiles[index]) || metadata.engineVersion !== CPP_LOCK.engineVersion || metadata.toolchain?.version !== CPP_LOCK.emscriptenVersion || metadata.toolchain?.commitSha1 !== CPP_LOCK.emscriptenCommit || metadata.toolchain?.emsdkCommitSha1 !== CPP_LOCK.emsdkCommit || metadata.toolchain?.releaseCommitSha1 !== CPP_LOCK.releaseCommit || metadata.toolchain?.llvmVersion !== CPP_LOCK.llvmVersion || metadata.toolchain?.llvmCommitSha1 !== CPP_LOCK.llvmCommit || JSON.stringify(metadata.toolchain?.linkedArchives) !== JSON.stringify(CPP_LOCK.linkedArchives) || metadata.bridge?.abiVersion !== CPP_LOCK.bridgeAbi || metadata.bridge?.dynamicExecutionDisabled !== true || metadata.bridge?.replacement?.semantics !== "libc++ match_results::format default grammar" || metadata.bridge?.replacement?.implementation !== "bounded streaming expansion without materializing an unbounded native format result" || metadata.bridge?.replacement?.paritySelfTest !== "load-time fixtures compare the bounded expander with libc++ match_results::format" || JSON.stringify(metadata.bridge?.sourceFiles) !== JSON.stringify(await sourceRecords(projectRoot)) || JSON.stringify(sourceManifest) !== JSON.stringify(await expectedSourceManifest(pack)) || metadata.source?.sourceManifest !== "SOURCE-MANIFEST.json" || metadata.licensing?.linkedComponentCount !== SOURCE_COMPONENTS.filter((component) => component.linked).length || JSON.stringify(metadata.licensing?.auditedExclusions) !== JSON.stringify(["libunwind"]) || metadata.licensing?.independentlyPinnedLegalFiles !== Object.keys(CPP_LOCK.legalFiles).length || Object.entries(CPP_LOCK.legalFiles).some( ([name, digest]) => metadata.files?.find((file) => file.path === name)?.sha256 !== digest, ) ) { throw new Error("C++ engine metadata drifted from its pinned sources."); } const module = await readFile(path.join(pack, "cpp-regex.mjs"), "utf8"); if (module.includes("new Function") || /\beval\s*\(/u.test(module)) { throw new Error("C++ engine module contains dynamic JavaScript execution."); } return metadata; } export async function buildCppPack( empp, output = path.join(root, ".engine-build", "cpp"), projectRoot = root, ) { const toolchain = await exactEmpp(empp); const outputParent = path.dirname(output); await mkdir(outputParent, { recursive: true }); const staging = await mkdtemp(path.join(outputParent, ".cpp-build-")); try { const build = await run( toolchain.executable, [ path.join(projectRoot, "engines", "cpp", "cpp_regex_bridge.cpp"), "-std=c++20", "-O3", "-fexceptions", "--bind", "-sMODULARIZE=1", "-sEXPORT_ES6=1", "-sENVIRONMENT=web,worker,node", "-sFILESYSTEM=0", "-sALLOW_MEMORY_GROWTH=1", "-sINITIAL_MEMORY=16777216", "-sMAXIMUM_MEMORY=268435456", "-sMALLOC=emmalloc", "-sASSERTIONS=0", "-sDYNAMIC_EXECUTION=0", "-sINCOMING_MODULE_JS_API=locateFile,print,printErr,wasmBinary", "-sEXPORT_NAME=createCppRegexModule", "-Wl,--trace", "-o", path.join(staging, "cpp-regex.mjs"), ], { cwd: projectRoot, env: buildEnvironment() }, ); assertLinkedArchives(`${build.stdout}\n${build.stderr}`); for (const [name, source] of Object.entries(toolchain.legalFiles)) { await copyFile(source, path.join(staging, name)); } await writeSourceManifest(staging); await writeMetadata(staging, projectRoot); await verifyCppPack(staging, projectRoot); await rm(output, { recursive: true, force: true }); await rename(staging, output); return { output, metadata: await verifyCppPack(output, projectRoot) }; } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; } } export async function installCppPack(source, projectRoot = root) { await verifyCppPack(source, projectRoot); const destination = path.join(projectRoot, "public", "engines", "cpp"); const parent = path.dirname(destination); const staging = await mkdtemp(path.join(parent, ".cpp-install-")); try { for (const name of PACK_FILES) { await copyFile(path.join(source, name), path.join(staging, name)); } await verifyCppPack(staging, projectRoot); await rm(destination, { recursive: true, force: true }); await rename(staging, destination); } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; } return { output: destination, metadata: await verifyCppPack(destination) }; } if (process.argv[1] === fileURLToPath(import.meta.url)) { const arguments_ = process.argv.slice(2); if (arguments_.length !== 2 || arguments_[0] !== "--empp") { throw new Error( "Usage: node scripts/cpp-engine-pack.mjs --empp /absolute/path/to/em++", ); } const result = await buildCppPack(arguments_[1]); console.log(`Built verified C++ engine pack at ${result.output}.`); }