import { createHash } from "node:crypto"; import { constants as fsConstants, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile, } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { File, PreopenDirectory } from "@bjorn3/browser_wasi_shim"; import { DefaultRubyVM } from "@ruby/wasm-wasi/dist/browser"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); export const RUBY_ENGINE_LOCK = Object.freeze({ bridgeAbi: 2, rubyVersion: "4.0.0", platform: "wasm32-wasi", packageVersion: "2.9.3-2.9.4", maximumPatternUtf16: 65_536, maximumSubjectBytes: 16_777_216, maximumReplacementUtf16: 65_536, maximumOutputBytes: 67_108_864, maximumMatches: 10_000, maximumCaptureRows: 100_000, maximumCaptureGroups: 1_000, npm: Object.freeze({ runtime: Object.freeze({ name: "@ruby/4.0-wasm-wasi", version: "2.9.3-2.9.4", resolved: "https://registry.npmjs.org/@ruby/4.0-wasm-wasi/-/4.0-wasm-wasi-2.9.3-2.9.4.tgz", integrity: "sha512-USLEWmCjM/DIota/9c7Mwzzl3v8mvmg4x24m+zMq8+vLx5+AdOS6dyXvsL/B8+XjQn5UgYhh3c8ekaUdWFkUMA==", packageJsonSha256: "a03a9475d54494a351c9c37ee9aca2db05f2652098907938628522d8a0f43807", }), host: Object.freeze({ name: "@ruby/wasm-wasi", version: "2.9.3-2.9.4", resolved: "https://registry.npmjs.org/@ruby/wasm-wasi/-/wasm-wasi-2.9.3-2.9.4.tgz", integrity: "sha512-WxW9wON/TIf+8Ktng8qDJeV/6iH8kw+YwxOsOyXdAdLJgfYDPAXOqpIVd/96y2C9V8VJ2yqZm/IRv6nLeV6EKg==", packageJsonSha256: "4a6eacec227c5cf81f580435d0105933a456e9685cb3f1821db967307431aa20", }), wasiShim: Object.freeze({ name: "@bjorn3/browser_wasi_shim", version: "0.4.2", integrity: "sha512-/iHkCVUG3VbcbmEHn5iIUpIrh7a7WPiwZ3sHy4HZKZzBdSadwdddYDZAII2zBvQYV0Lfi8naZngPCN7WPHI/hA==", packageJsonSha256: "4025e157abfadd44037158899001a4ea0407651c2ee9c8a9f51d5c9d0ecf613c", }), tslib: Object.freeze({ name: "tslib", version: "2.8.1", integrity: "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", packageJsonSha256: "f02b4851df316f29bdfcd404aa1eb41e4e216b5859015f81fcd3e91caf8be931", }), }), sourceAssets: Object.freeze({ "ruby.wasm": "fa1d5af1b4d601191739e6c4980d060d0d202181a7c83abf4930b3ca35b90f94", "LICENSE.ruby-wasm.txt": "90357d3794c968704914d42a52354a83f2d8b10cb43df3b63ef1ca0e5bbc0bf2", "NOTICE.txt": "343c246a6e1f1234e29e51707a54799ea82b50d3a2a41c5221fa12058b2395b2", "LICENSE.browser-wasi-shim-MIT.txt": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3", "LICENSE.browser-wasi-shim-Apache-2.0.txt": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", "LICENSE.tslib.txt": "210b19e543130388c68654b7497e967119ce17145f66ab7d85688fbd70f08751", }), }); const SOURCE_FILES = Object.freeze({ "ruby.wasm": ["@ruby", "4.0-wasm-wasi", "dist", "ruby.wasm"], "LICENSE.ruby-wasm.txt": ["@ruby", "4.0-wasm-wasi", "dist", "LICENSE"], "NOTICE.txt": ["@ruby", "4.0-wasm-wasi", "dist", "NOTICE"], "LICENSE.browser-wasi-shim-MIT.txt": [ "@bjorn3", "browser_wasi_shim", "LICENSE-MIT", ], "LICENSE.browser-wasi-shim-Apache-2.0.txt": [ "@bjorn3", "browser_wasi_shim", "LICENSE-APACHE", ], "LICENSE.tslib.txt": ["tslib", "LICENSE.txt"], }); const PACK_FILES = Object.freeze( [...Object.keys(SOURCE_FILES), "SHA256SUMS", "engine-metadata.json"].sort(), ); const CHECKSUM_FILES = Object.freeze( PACK_FILES.filter((name) => name !== "SHA256SUMS"), ); function compareText(left, right) { return left < right ? -1 : left > right ? 1 : 0; } function sha256(value) { return createHash("sha256").update(value).digest("hex"); } async function sha256File(file) { return sha256(await readFile(file)); } async function assertRealDirectory(candidate, label) { const details = await lstat(candidate).catch(() => null); if (!details?.isDirectory() || details.isSymbolicLink()) { throw new Error(`${label} must be a real directory: ${candidate}`); } return realpath(candidate); } async function assertRegularFile(candidate, label) { const details = await lstat(candidate).catch(() => null); if (!details?.isFile() || details.isSymbolicLink()) { throw new Error( `${label} must be a regular, non-symlink file: ${candidate}`, ); } return realpath(candidate); } async function assertHash(file, expected, label) { const actual = await sha256File(file); if (actual !== expected) { throw new Error( `${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`, ); } } async function verifyPackage(projectRoot, descriptor) { const directory = path.join( projectRoot, "node_modules", ...descriptor.name.split("/"), ); await assertRealDirectory(directory, `${descriptor.name} npm package`); const packageFile = path.join(directory, "package.json"); await assertRegularFile(packageFile, `${descriptor.name} package.json`); await assertHash( packageFile, descriptor.packageJsonSha256, `${descriptor.name} package.json`, ); const packageJson = JSON.parse(await readFile(packageFile, "utf8")); if ( packageJson.name !== descriptor.name || packageJson.version !== descriptor.version ) { throw new Error(`${descriptor.name} npm package identity drifted.`); } return directory; } async function verifyNpmLock(projectRoot) { const lock = JSON.parse( await readFile(path.join(projectRoot, "package-lock.json"), "utf8"), ); for (const descriptor of Object.values(RUBY_ENGINE_LOCK.npm)) { const entry = lock.packages?.[`node_modules/${descriptor.name}`]; if ( entry?.version !== descriptor.version || entry?.integrity !== descriptor.integrity || (descriptor.resolved !== undefined && entry?.resolved !== descriptor.resolved) ) { throw new Error( `${descriptor.name}@${descriptor.version} is not exactly pinned in package-lock.json.`, ); } } } async function verifySources(projectRoot) { await verifyNpmLock(projectRoot); const runtime = await verifyPackage( projectRoot, RUBY_ENGINE_LOCK.npm.runtime, ); await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.host); await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.wasiShim); await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.tslib); const nodeModules = path.join(projectRoot, "node_modules"); for (const [output, components] of Object.entries(SOURCE_FILES)) { const source = path.join(nodeModules, ...components); await assertRegularFile(source, `Ruby source ${output}`); await assertHash( source, RUBY_ENGINE_LOCK.sourceAssets[output], `Ruby source ${output}`, ); } return { nodeModules, runtime }; } async function fileRecord(directory, name) { const value = await readFile(path.join(directory, name)); return { path: name, sha256: sha256(value), bytes: value.byteLength }; } async function expectedMetadata(projectRoot, pack) { const bridgePath = path.join(projectRoot, "engines", "ruby", "bridge.rb"); const bridge = await readFile(bridgePath); const files = []; for (const name of CHECKSUM_FILES) { if (name !== "engine-metadata.json") { files.push(await fileRecord(pack, name)); } } return { schemaVersion: 1, engine: "ruby", engineVersion: `CRuby ${RUBY_ENGINE_LOCK.rubyVersion} Regexp`, runtime: `ruby.wasm ${RUBY_ENGINE_LOCK.packageVersion}`, platform: RUBY_ENGINE_LOCK.platform, status: "production-runtime", bridge: { abiVersion: RUBY_ENGINE_LOCK.bridgeAbi, sourceFile: { path: "engines/ruby/bridge.rb", sha256: sha256(bridge), bytes: bridge.byteLength, }, requestEncoding: "bounded length-prefixed UTF-8 fields in a worker-local WASI memory file", responseEncoding: "bounded inert JSON metadata plus a worker-local bounded UTF-8 output file", userSourceEvaluation: false, replacement: "native Ruby String#gsub/String#sub match iteration with a bounded CRuby 4.0.0 rb_reg_regsub-compatible expander", nativeOffsetUnit: "Unicode code points", maximumPatternUtf16: RUBY_ENGINE_LOCK.maximumPatternUtf16, maximumSubjectBytes: RUBY_ENGINE_LOCK.maximumSubjectBytes, maximumReplacementUtf16: RUBY_ENGINE_LOCK.maximumReplacementUtf16, maximumOutputBytes: RUBY_ENGINE_LOCK.maximumOutputBytes, maximumMatches: RUBY_ENGINE_LOCK.maximumMatches, maximumCaptureRows: RUBY_ENGINE_LOCK.maximumCaptureRows, maximumCaptureGroups: RUBY_ENGINE_LOCK.maximumCaptureGroups, }, source: { repository: "https://github.com/ruby/ruby.wasm", packages: Object.values(RUBY_ENGINE_LOCK.npm).map( ({ name, version, integrity }) => ({ name, version, integrity }), ), runtimeVariant: "minimal ruby.wasm (without bundled stdlib/debug)", cruby: { engine: "ruby", version: RUBY_ENGINE_LOCK.rubyVersion, platform: RUBY_ENGINE_LOCK.platform, }, }, files, }; } async function smokeRuby(pack, projectRoot) { const moduleBytes = await readFile(path.join(pack, "ruby.wasm")); if (!WebAssembly.validate(moduleBytes)) { throw new Error("Ruby runtime is not a valid WebAssembly module."); } const module = await WebAssembly.compile(moduleBytes); const { vm, wasi } = await DefaultRubyVM(module, { consolePrint: false }); const requestFile = new File([]); const outputFile = new File([]); const rootDirectory = wasi.fds[3]; if (!(rootDirectory instanceof PreopenDirectory)) { throw new Error("Ruby smoke runtime has no writable root filesystem."); } rootDirectory.dir.contents.set("regex-tools-request.bin", requestFile); rootDirectory.dir.contents.set("regex-tools-output.bin", outputFile); const bridgeSource = await readFile( path.join(projectRoot, "engines", "ruby", "bridge.rb"), "utf8", ); const bridge = vm.eval(bridgeSource); const encoder = new TextEncoder(); const call = (...values) => { const fields = values.map((value) => encoder.encode(String(value))); const payload = new Uint8Array( 8 + fields.reduce((total, field) => total + 4 + field.byteLength, 0), ); payload.set(encoder.encode("RGXRUBY1")); const view = new DataView(payload.buffer); let offset = 8; for (const field of fields) { view.setUint32(offset, field.byteLength, false); offset += 4; payload.set(field, offset); offset += field.byteLength; } requestFile.data = payload; const response = JSON.parse(bridge.call("run").toString()); if (String(values[0]) === "replace" && response?.ok === true) { response.output = new TextDecoder("utf-8", { fatal: true }).decode( outputFile.data, ); } return response; }; const identity = call("identity", "", "", "", 1, 1, "", 1); if ( identity?.ok !== true || identity.identity?.implementation !== "ruby" || identity.identity?.rubyVersion !== RUBY_ENGINE_LOCK.rubyVersion || identity.identity?.platform !== RUBY_ENGINE_LOCK.platform ) { throw new Error("Ruby runtime failed its identity smoke test."); } const execution = call( "execute", "(?é+)", "g", "😀éé x", 10, 10, "", 1, ); if ( execution?.ok !== true || execution.groupCount !== 1 || JSON.stringify(execution.groupNames) !== JSON.stringify([[1, "word"]]) || JSON.stringify(execution.matches?.[0]?.span) !== JSON.stringify([1, 3]) ) { throw new Error("Ruby runtime failed its native match smoke test."); } const replacement = call( "replace", "(?é+)", "g", "😀éé x", 10, 10, "[\\k]", 1_024, ); if ( replacement?.ok !== true || replacement.output !== "😀[éé] x" || replacement.outputTruncated !== false ) { throw new Error("Ruby runtime failed its native replacement smoke test."); } } export async function verifyRubyPack(directory, projectRoot = root) { const pack = await assertRealDirectory(directory, "Ruby engine pack"); const names = (await readdir(pack, { withFileTypes: true })) .map((entry) => { if (!entry.isFile() || entry.isSymbolicLink()) { throw new Error( `Ruby pack contains a non-regular entry: ${entry.name}`, ); } return entry.name; }) .sort(compareText); if (JSON.stringify(names) !== JSON.stringify(PACK_FILES)) { throw new Error(`Ruby pack must contain only: ${PACK_FILES.join(", ")}.`); } const checksumLines = (await readFile(path.join(pack, "SHA256SUMS"), "utf8")) .trimEnd() .split("\n"); const expectedLines = []; for (const name of CHECKSUM_FILES) { expectedLines.push(`${await sha256File(path.join(pack, name))} ${name}`); } if (JSON.stringify(checksumLines) !== JSON.stringify(expectedLines)) { throw new Error("Ruby pack checksum manifest is stale or malformed."); } for (const [name, expected] of Object.entries( RUBY_ENGINE_LOCK.sourceAssets, )) { await assertHash(path.join(pack, name), expected, `Packed Ruby ${name}`); } const metadata = JSON.parse( await readFile(path.join(pack, "engine-metadata.json"), "utf8"), ); const expected = await expectedMetadata(projectRoot, pack); if (JSON.stringify(metadata) !== JSON.stringify(expected)) { throw new Error("Ruby engine metadata does not match the pinned pack."); } await smokeRuby(pack, projectRoot); return metadata; } export async function buildRubyPack({ projectRoot = root, output = path.join(root, ".engine-build", "ruby"), force = false, } = {}) { const sources = await verifySources(projectRoot); const target = path.resolve(output); const existing = await lstat(target).catch(() => null); if (existing && !force) { throw new Error(`${target} exists; pass --force to replace it.`); } if (existing && (existing.isSymbolicLink() || !existing.isDirectory())) { throw new Error("Ruby pack output must be a real directory when replaced."); } await mkdir(path.dirname(target), { recursive: true }); const stage = await mkdtemp(path.join(path.dirname(target), ".ruby-pack-")); try { for (const [name, components] of Object.entries(SOURCE_FILES)) { await copyFile( path.join(sources.nodeModules, ...components), path.join(stage, name), fsConstants.COPYFILE_EXCL, ); } const metadata = await expectedMetadata(projectRoot, stage); await writeFile( path.join(stage, "engine-metadata.json"), `${JSON.stringify(metadata, null, 2)}\n`, { flag: "wx", mode: 0o644 }, ); const checksumLines = []; for (const name of CHECKSUM_FILES) { checksumLines.push( `${await sha256File(path.join(stage, name))} ${name}`, ); } await writeFile( path.join(stage, "SHA256SUMS"), `${checksumLines.join("\n")}\n`, { flag: "wx", mode: 0o644 }, ); await verifyRubyPack(stage, projectRoot); if (existing) await rm(target, { recursive: true }); await rename(stage, target); } catch (error) { await rm(stage, { recursive: true, force: true }); throw error; } return { output: target, metadata: await verifyRubyPack(target, projectRoot), }; } export async function installRubyPack(source, projectRoot = root) { const sourcePath = await assertRealDirectory(source, "Ruby staged pack"); const target = path.join(projectRoot, "public", "engines", "ruby"); if (sourcePath === (await realpath(target).catch(() => ""))) { throw new Error("Ruby staged pack and installation target must differ."); } const metadata = await verifyRubyPack(sourcePath, projectRoot); await mkdir(path.dirname(target), { recursive: true }); const stage = await mkdtemp( path.join(path.dirname(target), ".ruby-install-"), ); let backup; try { for (const name of PACK_FILES) { await copyFile( path.join(sourcePath, name), path.join(stage, name), fsConstants.COPYFILE_EXCL, ); } await verifyRubyPack(stage, projectRoot); const existing = await lstat(target).catch(() => null); if (existing) { if (existing.isSymbolicLink() || !existing.isDirectory()) { throw new Error("Installed Ruby pack target must be a real directory."); } backup = `${target}.backup-${process.pid}-${Date.now()}`; await rename(target, backup); } await rename(stage, target); if (backup) await rm(backup, { recursive: true }); } catch (error) { await rm(stage, { recursive: true, force: true }); if (backup && !(await lstat(target).catch(() => null))) { await rename(backup, target); } throw error; } return { output: target, metadata }; } function parseArguments(arguments_) { let output = path.join(root, ".engine-build", "ruby"); let force = false; for (let index = 0; index < arguments_.length; index += 1) { const argument = arguments_[index]; if (argument === "--force") { if (force) throw new Error("--force was provided more than once."); force = true; } else if (argument === "--output") { const value = arguments_[index + 1]; if (!value || value.startsWith("--")) { throw new Error("--output requires a directory."); } output = path.resolve(root, value); index += 1; } else { throw new Error(`Unknown Ruby engine-pack argument: ${argument}`); } } return { output, force }; } async function main() { const result = await buildRubyPack(parseArguments(process.argv.slice(2))); console.log( `Built verified ${result.metadata.engineVersion} pack at ${result.output}.`, ); } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { await main(); }