538 lines
18 KiB
JavaScript
538 lines
18 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import {
|
|
constants as fsConstants,
|
|
copyFile,
|
|
lstat,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
realpath,
|
|
rename,
|
|
rm,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
import { File, PreopenDirectory } from "@bjorn3/browser_wasi_shim";
|
|
import { DefaultRubyVM } from "@ruby/wasm-wasi/dist/browser";
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
|
|
export const RUBY_ENGINE_LOCK = Object.freeze({
|
|
bridgeAbi: 2,
|
|
rubyVersion: "4.0.0",
|
|
platform: "wasm32-wasi",
|
|
packageVersion: "2.9.3-2.9.4",
|
|
maximumPatternUtf16: 65_536,
|
|
maximumSubjectBytes: 16_777_216,
|
|
maximumReplacementUtf16: 65_536,
|
|
maximumOutputBytes: 67_108_864,
|
|
maximumMatches: 10_000,
|
|
maximumCaptureRows: 100_000,
|
|
maximumCaptureGroups: 1_000,
|
|
npm: Object.freeze({
|
|
runtime: Object.freeze({
|
|
name: "@ruby/4.0-wasm-wasi",
|
|
version: "2.9.3-2.9.4",
|
|
resolved:
|
|
"https://registry.npmjs.org/@ruby/4.0-wasm-wasi/-/4.0-wasm-wasi-2.9.3-2.9.4.tgz",
|
|
integrity:
|
|
"sha512-USLEWmCjM/DIota/9c7Mwzzl3v8mvmg4x24m+zMq8+vLx5+AdOS6dyXvsL/B8+XjQn5UgYhh3c8ekaUdWFkUMA==",
|
|
packageJsonSha256:
|
|
"a03a9475d54494a351c9c37ee9aca2db05f2652098907938628522d8a0f43807",
|
|
}),
|
|
host: Object.freeze({
|
|
name: "@ruby/wasm-wasi",
|
|
version: "2.9.3-2.9.4",
|
|
resolved:
|
|
"https://registry.npmjs.org/@ruby/wasm-wasi/-/wasm-wasi-2.9.3-2.9.4.tgz",
|
|
integrity:
|
|
"sha512-WxW9wON/TIf+8Ktng8qDJeV/6iH8kw+YwxOsOyXdAdLJgfYDPAXOqpIVd/96y2C9V8VJ2yqZm/IRv6nLeV6EKg==",
|
|
packageJsonSha256:
|
|
"4a6eacec227c5cf81f580435d0105933a456e9685cb3f1821db967307431aa20",
|
|
}),
|
|
wasiShim: Object.freeze({
|
|
name: "@bjorn3/browser_wasi_shim",
|
|
version: "0.4.2",
|
|
integrity:
|
|
"sha512-/iHkCVUG3VbcbmEHn5iIUpIrh7a7WPiwZ3sHy4HZKZzBdSadwdddYDZAII2zBvQYV0Lfi8naZngPCN7WPHI/hA==",
|
|
packageJsonSha256:
|
|
"4025e157abfadd44037158899001a4ea0407651c2ee9c8a9f51d5c9d0ecf613c",
|
|
}),
|
|
tslib: Object.freeze({
|
|
name: "tslib",
|
|
version: "2.8.1",
|
|
integrity:
|
|
"sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
|
packageJsonSha256:
|
|
"f02b4851df316f29bdfcd404aa1eb41e4e216b5859015f81fcd3e91caf8be931",
|
|
}),
|
|
}),
|
|
sourceAssets: Object.freeze({
|
|
"ruby.wasm":
|
|
"fa1d5af1b4d601191739e6c4980d060d0d202181a7c83abf4930b3ca35b90f94",
|
|
"LICENSE.ruby-wasm.txt":
|
|
"90357d3794c968704914d42a52354a83f2d8b10cb43df3b63ef1ca0e5bbc0bf2",
|
|
"NOTICE.txt":
|
|
"343c246a6e1f1234e29e51707a54799ea82b50d3a2a41c5221fa12058b2395b2",
|
|
"LICENSE.browser-wasi-shim-MIT.txt":
|
|
"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3",
|
|
"LICENSE.browser-wasi-shim-Apache-2.0.txt":
|
|
"c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
|
"LICENSE.tslib.txt":
|
|
"210b19e543130388c68654b7497e967119ce17145f66ab7d85688fbd70f08751",
|
|
}),
|
|
});
|
|
|
|
const SOURCE_FILES = Object.freeze({
|
|
"ruby.wasm": ["@ruby", "4.0-wasm-wasi", "dist", "ruby.wasm"],
|
|
"LICENSE.ruby-wasm.txt": ["@ruby", "4.0-wasm-wasi", "dist", "LICENSE"],
|
|
"NOTICE.txt": ["@ruby", "4.0-wasm-wasi", "dist", "NOTICE"],
|
|
"LICENSE.browser-wasi-shim-MIT.txt": [
|
|
"@bjorn3",
|
|
"browser_wasi_shim",
|
|
"LICENSE-MIT",
|
|
],
|
|
"LICENSE.browser-wasi-shim-Apache-2.0.txt": [
|
|
"@bjorn3",
|
|
"browser_wasi_shim",
|
|
"LICENSE-APACHE",
|
|
],
|
|
"LICENSE.tslib.txt": ["tslib", "LICENSE.txt"],
|
|
});
|
|
|
|
const PACK_FILES = Object.freeze(
|
|
[...Object.keys(SOURCE_FILES), "SHA256SUMS", "engine-metadata.json"].sort(),
|
|
);
|
|
const CHECKSUM_FILES = Object.freeze(
|
|
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
|
|
);
|
|
|
|
function compareText(left, right) {
|
|
return left < right ? -1 : left > right ? 1 : 0;
|
|
}
|
|
|
|
function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
async function sha256File(file) {
|
|
return sha256(await readFile(file));
|
|
}
|
|
|
|
async function assertRealDirectory(candidate, label) {
|
|
const details = await lstat(candidate).catch(() => null);
|
|
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
|
throw new Error(`${label} must be a real directory: ${candidate}`);
|
|
}
|
|
return realpath(candidate);
|
|
}
|
|
|
|
async function assertRegularFile(candidate, label) {
|
|
const details = await lstat(candidate).catch(() => null);
|
|
if (!details?.isFile() || details.isSymbolicLink()) {
|
|
throw new Error(
|
|
`${label} must be a regular, non-symlink file: ${candidate}`,
|
|
);
|
|
}
|
|
return realpath(candidate);
|
|
}
|
|
|
|
async function assertHash(file, expected, label) {
|
|
const actual = await sha256File(file);
|
|
if (actual !== expected) {
|
|
throw new Error(
|
|
`${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async function verifyPackage(projectRoot, descriptor) {
|
|
const directory = path.join(
|
|
projectRoot,
|
|
"node_modules",
|
|
...descriptor.name.split("/"),
|
|
);
|
|
await assertRealDirectory(directory, `${descriptor.name} npm package`);
|
|
const packageFile = path.join(directory, "package.json");
|
|
await assertRegularFile(packageFile, `${descriptor.name} package.json`);
|
|
await assertHash(
|
|
packageFile,
|
|
descriptor.packageJsonSha256,
|
|
`${descriptor.name} package.json`,
|
|
);
|
|
const packageJson = JSON.parse(await readFile(packageFile, "utf8"));
|
|
if (
|
|
packageJson.name !== descriptor.name ||
|
|
packageJson.version !== descriptor.version
|
|
) {
|
|
throw new Error(`${descriptor.name} npm package identity drifted.`);
|
|
}
|
|
return directory;
|
|
}
|
|
|
|
async function verifyNpmLock(projectRoot) {
|
|
const lock = JSON.parse(
|
|
await readFile(path.join(projectRoot, "package-lock.json"), "utf8"),
|
|
);
|
|
for (const descriptor of Object.values(RUBY_ENGINE_LOCK.npm)) {
|
|
const entry = lock.packages?.[`node_modules/${descriptor.name}`];
|
|
if (
|
|
entry?.version !== descriptor.version ||
|
|
entry?.integrity !== descriptor.integrity ||
|
|
(descriptor.resolved !== undefined &&
|
|
entry?.resolved !== descriptor.resolved)
|
|
) {
|
|
throw new Error(
|
|
`${descriptor.name}@${descriptor.version} is not exactly pinned in package-lock.json.`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function verifySources(projectRoot) {
|
|
await verifyNpmLock(projectRoot);
|
|
const runtime = await verifyPackage(
|
|
projectRoot,
|
|
RUBY_ENGINE_LOCK.npm.runtime,
|
|
);
|
|
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.host);
|
|
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.wasiShim);
|
|
await verifyPackage(projectRoot, RUBY_ENGINE_LOCK.npm.tslib);
|
|
|
|
const nodeModules = path.join(projectRoot, "node_modules");
|
|
for (const [output, components] of Object.entries(SOURCE_FILES)) {
|
|
const source = path.join(nodeModules, ...components);
|
|
await assertRegularFile(source, `Ruby source ${output}`);
|
|
await assertHash(
|
|
source,
|
|
RUBY_ENGINE_LOCK.sourceAssets[output],
|
|
`Ruby source ${output}`,
|
|
);
|
|
}
|
|
return { nodeModules, runtime };
|
|
}
|
|
|
|
async function fileRecord(directory, name) {
|
|
const value = await readFile(path.join(directory, name));
|
|
return { path: name, sha256: sha256(value), bytes: value.byteLength };
|
|
}
|
|
|
|
async function expectedMetadata(projectRoot, pack) {
|
|
const bridgePath = path.join(projectRoot, "engines", "ruby", "bridge.rb");
|
|
const bridge = await readFile(bridgePath);
|
|
const files = [];
|
|
for (const name of CHECKSUM_FILES) {
|
|
if (name !== "engine-metadata.json") {
|
|
files.push(await fileRecord(pack, name));
|
|
}
|
|
}
|
|
return {
|
|
schemaVersion: 1,
|
|
engine: "ruby",
|
|
engineVersion: `CRuby ${RUBY_ENGINE_LOCK.rubyVersion} Regexp`,
|
|
runtime: `ruby.wasm ${RUBY_ENGINE_LOCK.packageVersion}`,
|
|
platform: RUBY_ENGINE_LOCK.platform,
|
|
status: "production-runtime",
|
|
bridge: {
|
|
abiVersion: RUBY_ENGINE_LOCK.bridgeAbi,
|
|
sourceFile: {
|
|
path: "engines/ruby/bridge.rb",
|
|
sha256: sha256(bridge),
|
|
bytes: bridge.byteLength,
|
|
},
|
|
requestEncoding:
|
|
"bounded length-prefixed UTF-8 fields in a worker-local WASI memory file",
|
|
responseEncoding:
|
|
"bounded inert JSON metadata plus a worker-local bounded UTF-8 output file",
|
|
userSourceEvaluation: false,
|
|
replacement:
|
|
"native Ruby String#gsub/String#sub match iteration with a bounded CRuby 4.0.0 rb_reg_regsub-compatible expander",
|
|
nativeOffsetUnit: "Unicode code points",
|
|
maximumPatternUtf16: RUBY_ENGINE_LOCK.maximumPatternUtf16,
|
|
maximumSubjectBytes: RUBY_ENGINE_LOCK.maximumSubjectBytes,
|
|
maximumReplacementUtf16: RUBY_ENGINE_LOCK.maximumReplacementUtf16,
|
|
maximumOutputBytes: RUBY_ENGINE_LOCK.maximumOutputBytes,
|
|
maximumMatches: RUBY_ENGINE_LOCK.maximumMatches,
|
|
maximumCaptureRows: RUBY_ENGINE_LOCK.maximumCaptureRows,
|
|
maximumCaptureGroups: RUBY_ENGINE_LOCK.maximumCaptureGroups,
|
|
},
|
|
source: {
|
|
repository: "https://github.com/ruby/ruby.wasm",
|
|
packages: Object.values(RUBY_ENGINE_LOCK.npm).map(
|
|
({ name, version, integrity }) => ({ name, version, integrity }),
|
|
),
|
|
runtimeVariant: "minimal ruby.wasm (without bundled stdlib/debug)",
|
|
cruby: {
|
|
engine: "ruby",
|
|
version: RUBY_ENGINE_LOCK.rubyVersion,
|
|
platform: RUBY_ENGINE_LOCK.platform,
|
|
},
|
|
},
|
|
files,
|
|
};
|
|
}
|
|
|
|
async function smokeRuby(pack, projectRoot) {
|
|
const moduleBytes = await readFile(path.join(pack, "ruby.wasm"));
|
|
if (!WebAssembly.validate(moduleBytes)) {
|
|
throw new Error("Ruby runtime is not a valid WebAssembly module.");
|
|
}
|
|
const module = await WebAssembly.compile(moduleBytes);
|
|
const { vm, wasi } = await DefaultRubyVM(module, { consolePrint: false });
|
|
const requestFile = new File([]);
|
|
const outputFile = new File([]);
|
|
const rootDirectory = wasi.fds[3];
|
|
if (!(rootDirectory instanceof PreopenDirectory)) {
|
|
throw new Error("Ruby smoke runtime has no writable root filesystem.");
|
|
}
|
|
rootDirectory.dir.contents.set("regex-tools-request.bin", requestFile);
|
|
rootDirectory.dir.contents.set("regex-tools-output.bin", outputFile);
|
|
const bridgeSource = await readFile(
|
|
path.join(projectRoot, "engines", "ruby", "bridge.rb"),
|
|
"utf8",
|
|
);
|
|
const bridge = vm.eval(bridgeSource);
|
|
const encoder = new TextEncoder();
|
|
const call = (...values) => {
|
|
const fields = values.map((value) => encoder.encode(String(value)));
|
|
const payload = new Uint8Array(
|
|
8 + fields.reduce((total, field) => total + 4 + field.byteLength, 0),
|
|
);
|
|
payload.set(encoder.encode("RGXRUBY1"));
|
|
const view = new DataView(payload.buffer);
|
|
let offset = 8;
|
|
for (const field of fields) {
|
|
view.setUint32(offset, field.byteLength, false);
|
|
offset += 4;
|
|
payload.set(field, offset);
|
|
offset += field.byteLength;
|
|
}
|
|
requestFile.data = payload;
|
|
const response = JSON.parse(bridge.call("run").toString());
|
|
if (String(values[0]) === "replace" && response?.ok === true) {
|
|
response.output = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
outputFile.data,
|
|
);
|
|
}
|
|
return response;
|
|
};
|
|
const identity = call("identity", "", "", "", 1, 1, "", 1);
|
|
if (
|
|
identity?.ok !== true ||
|
|
identity.identity?.implementation !== "ruby" ||
|
|
identity.identity?.rubyVersion !== RUBY_ENGINE_LOCK.rubyVersion ||
|
|
identity.identity?.platform !== RUBY_ENGINE_LOCK.platform
|
|
) {
|
|
throw new Error("Ruby runtime failed its identity smoke test.");
|
|
}
|
|
const execution = call(
|
|
"execute",
|
|
"(?<word>é+)",
|
|
"g",
|
|
"😀éé x",
|
|
10,
|
|
10,
|
|
"",
|
|
1,
|
|
);
|
|
if (
|
|
execution?.ok !== true ||
|
|
execution.groupCount !== 1 ||
|
|
JSON.stringify(execution.groupNames) !== JSON.stringify([[1, "word"]]) ||
|
|
JSON.stringify(execution.matches?.[0]?.span) !== JSON.stringify([1, 3])
|
|
) {
|
|
throw new Error("Ruby runtime failed its native match smoke test.");
|
|
}
|
|
const replacement = call(
|
|
"replace",
|
|
"(?<word>é+)",
|
|
"g",
|
|
"😀éé x",
|
|
10,
|
|
10,
|
|
"[\\k<word>]",
|
|
1_024,
|
|
);
|
|
if (
|
|
replacement?.ok !== true ||
|
|
replacement.output !== "😀[éé] x" ||
|
|
replacement.outputTruncated !== false
|
|
) {
|
|
throw new Error("Ruby runtime failed its native replacement smoke test.");
|
|
}
|
|
}
|
|
|
|
export async function verifyRubyPack(directory, projectRoot = root) {
|
|
const pack = await assertRealDirectory(directory, "Ruby engine pack");
|
|
const names = (await readdir(pack, { withFileTypes: true }))
|
|
.map((entry) => {
|
|
if (!entry.isFile() || entry.isSymbolicLink()) {
|
|
throw new Error(
|
|
`Ruby pack contains a non-regular entry: ${entry.name}`,
|
|
);
|
|
}
|
|
return entry.name;
|
|
})
|
|
.sort(compareText);
|
|
if (JSON.stringify(names) !== JSON.stringify(PACK_FILES)) {
|
|
throw new Error(`Ruby pack must contain only: ${PACK_FILES.join(", ")}.`);
|
|
}
|
|
|
|
const checksumLines = (await readFile(path.join(pack, "SHA256SUMS"), "utf8"))
|
|
.trimEnd()
|
|
.split("\n");
|
|
const expectedLines = [];
|
|
for (const name of CHECKSUM_FILES) {
|
|
expectedLines.push(`${await sha256File(path.join(pack, name))} ${name}`);
|
|
}
|
|
if (JSON.stringify(checksumLines) !== JSON.stringify(expectedLines)) {
|
|
throw new Error("Ruby pack checksum manifest is stale or malformed.");
|
|
}
|
|
|
|
for (const [name, expected] of Object.entries(
|
|
RUBY_ENGINE_LOCK.sourceAssets,
|
|
)) {
|
|
await assertHash(path.join(pack, name), expected, `Packed Ruby ${name}`);
|
|
}
|
|
const metadata = JSON.parse(
|
|
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
|
);
|
|
const expected = await expectedMetadata(projectRoot, pack);
|
|
if (JSON.stringify(metadata) !== JSON.stringify(expected)) {
|
|
throw new Error("Ruby engine metadata does not match the pinned pack.");
|
|
}
|
|
await smokeRuby(pack, projectRoot);
|
|
return metadata;
|
|
}
|
|
|
|
export async function buildRubyPack({
|
|
projectRoot = root,
|
|
output = path.join(root, ".engine-build", "ruby"),
|
|
force = false,
|
|
} = {}) {
|
|
const sources = await verifySources(projectRoot);
|
|
const target = path.resolve(output);
|
|
const existing = await lstat(target).catch(() => null);
|
|
if (existing && !force) {
|
|
throw new Error(`${target} exists; pass --force to replace it.`);
|
|
}
|
|
if (existing && (existing.isSymbolicLink() || !existing.isDirectory())) {
|
|
throw new Error("Ruby pack output must be a real directory when replaced.");
|
|
}
|
|
|
|
await mkdir(path.dirname(target), { recursive: true });
|
|
const stage = await mkdtemp(path.join(path.dirname(target), ".ruby-pack-"));
|
|
try {
|
|
for (const [name, components] of Object.entries(SOURCE_FILES)) {
|
|
await copyFile(
|
|
path.join(sources.nodeModules, ...components),
|
|
path.join(stage, name),
|
|
fsConstants.COPYFILE_EXCL,
|
|
);
|
|
}
|
|
const metadata = await expectedMetadata(projectRoot, stage);
|
|
await writeFile(
|
|
path.join(stage, "engine-metadata.json"),
|
|
`${JSON.stringify(metadata, null, 2)}\n`,
|
|
{ flag: "wx", mode: 0o644 },
|
|
);
|
|
const checksumLines = [];
|
|
for (const name of CHECKSUM_FILES) {
|
|
checksumLines.push(
|
|
`${await sha256File(path.join(stage, name))} ${name}`,
|
|
);
|
|
}
|
|
await writeFile(
|
|
path.join(stage, "SHA256SUMS"),
|
|
`${checksumLines.join("\n")}\n`,
|
|
{ flag: "wx", mode: 0o644 },
|
|
);
|
|
await verifyRubyPack(stage, projectRoot);
|
|
if (existing) await rm(target, { recursive: true });
|
|
await rename(stage, target);
|
|
} catch (error) {
|
|
await rm(stage, { recursive: true, force: true });
|
|
throw error;
|
|
}
|
|
return {
|
|
output: target,
|
|
metadata: await verifyRubyPack(target, projectRoot),
|
|
};
|
|
}
|
|
|
|
export async function installRubyPack(source, projectRoot = root) {
|
|
const sourcePath = await assertRealDirectory(source, "Ruby staged pack");
|
|
const target = path.join(projectRoot, "public", "engines", "ruby");
|
|
if (sourcePath === (await realpath(target).catch(() => ""))) {
|
|
throw new Error("Ruby staged pack and installation target must differ.");
|
|
}
|
|
const metadata = await verifyRubyPack(sourcePath, projectRoot);
|
|
await mkdir(path.dirname(target), { recursive: true });
|
|
const stage = await mkdtemp(
|
|
path.join(path.dirname(target), ".ruby-install-"),
|
|
);
|
|
let backup;
|
|
try {
|
|
for (const name of PACK_FILES) {
|
|
await copyFile(
|
|
path.join(sourcePath, name),
|
|
path.join(stage, name),
|
|
fsConstants.COPYFILE_EXCL,
|
|
);
|
|
}
|
|
await verifyRubyPack(stage, projectRoot);
|
|
const existing = await lstat(target).catch(() => null);
|
|
if (existing) {
|
|
if (existing.isSymbolicLink() || !existing.isDirectory()) {
|
|
throw new Error("Installed Ruby pack target must be a real directory.");
|
|
}
|
|
backup = `${target}.backup-${process.pid}-${Date.now()}`;
|
|
await rename(target, backup);
|
|
}
|
|
await rename(stage, target);
|
|
if (backup) await rm(backup, { recursive: true });
|
|
} catch (error) {
|
|
await rm(stage, { recursive: true, force: true });
|
|
if (backup && !(await lstat(target).catch(() => null))) {
|
|
await rename(backup, target);
|
|
}
|
|
throw error;
|
|
}
|
|
return { output: target, metadata };
|
|
}
|
|
|
|
function parseArguments(arguments_) {
|
|
let output = path.join(root, ".engine-build", "ruby");
|
|
let force = false;
|
|
for (let index = 0; index < arguments_.length; index += 1) {
|
|
const argument = arguments_[index];
|
|
if (argument === "--force") {
|
|
if (force) throw new Error("--force was provided more than once.");
|
|
force = true;
|
|
} else if (argument === "--output") {
|
|
const value = arguments_[index + 1];
|
|
if (!value || value.startsWith("--")) {
|
|
throw new Error("--output requires a directory.");
|
|
}
|
|
output = path.resolve(root, value);
|
|
index += 1;
|
|
} else {
|
|
throw new Error(`Unknown Ruby engine-pack argument: ${argument}`);
|
|
}
|
|
}
|
|
return { output, force };
|
|
}
|
|
|
|
async function main() {
|
|
const result = await buildRubyPack(parseArguments(process.argv.slice(2)));
|
|
console.log(
|
|
`Built verified ${result.metadata.engineVersion} pack at ${result.output}.`,
|
|
);
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
|
await main();
|
|
}
|