feat: add multi-engine regex flavour support
This commit is contained in:
478
scripts/go-engine-pack.mjs
Normal file
478
scripts/go-engine-pack.mjs
Normal file
@@ -0,0 +1,478 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export const GO_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
engineVersion: "1.26.5",
|
||||
runtimeVersion: "go1.26.5",
|
||||
sourceDateEpoch: 1_783_452_544,
|
||||
source: Object.freeze({
|
||||
repository: "https://go.googlesource.com/go",
|
||||
tag: "go1.26.5",
|
||||
commit: "c19862e5f8415b4f24b189d065ed739517c548ba",
|
||||
tree: "0bb2fb1cc06c334c36a2a92d2f0b07fea7236d74",
|
||||
license: "BSD-3-Clause",
|
||||
}),
|
||||
toolchain: Object.freeze({
|
||||
archive: "https://go.dev/dl/go1.26.5.linux-amd64.tar.gz",
|
||||
archiveSha256:
|
||||
"5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053",
|
||||
licenseSha256:
|
||||
"911f8f5782931320f5b8d1160a76365b83aea6447ee6c04fa6d5591467db9dad",
|
||||
wasmExecSha256:
|
||||
"0c949f4996f9a89698e4b5c586de32249c3b69b7baadb64d220073cc04acba14",
|
||||
}),
|
||||
limits: Object.freeze({
|
||||
maximumPatternBytes: 262_144,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementBytes: 262_144,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
maximumRequestJSONBytes: 103_874_560,
|
||||
}),
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze([
|
||||
"LICENSE-Go.txt",
|
||||
"SHA256SUMS",
|
||||
"engine-metadata.json",
|
||||
"go-regex.wasm",
|
||||
"wasm_exec.mjs",
|
||||
]);
|
||||
const CHECKSUM_FILES = Object.freeze([
|
||||
"LICENSE-Go.txt",
|
||||
"engine-metadata.json",
|
||||
"go-regex.wasm",
|
||||
"wasm_exec.mjs",
|
||||
]);
|
||||
const SOURCE_FILES = Object.freeze([
|
||||
"engines/go/README.md",
|
||||
"engines/go/go.mod",
|
||||
"engines/go/main.go",
|
||||
]);
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a regular file: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
function commandLabel(command, arguments_) {
|
||||
return [command, ...arguments_]
|
||||
.map((part) =>
|
||||
/^[A-Za-z0-9_./:=,+-]+$/u.test(part) ? part : JSON.stringify(part),
|
||||
)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
const detail = [output, errorOutput].filter(Boolean).join("\n").trim();
|
||||
throw new Error(
|
||||
`${commandLabel(command, arguments_)} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}${detail ? `:\n${detail}` : "."}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
async function verifyToolchain(goRootDirectory) {
|
||||
const goRoot = await assertRealDirectory(
|
||||
goRootDirectory,
|
||||
"Go 1.26.5 toolchain",
|
||||
);
|
||||
const go = await assertRegularFile(path.join(goRoot, "bin", "go"), "go");
|
||||
await access(go, fsConstants.X_OK);
|
||||
const version = await run(go, ["version"]);
|
||||
if (version.stdout.trim() !== "go version go1.26.5 linux/amd64") {
|
||||
throw new Error("The Go pack requires official go1.26.5 linux/amd64.");
|
||||
}
|
||||
const license = await assertRegularFile(
|
||||
path.join(goRoot, "LICENSE"),
|
||||
"Go licence",
|
||||
);
|
||||
const wasmExec = await assertRegularFile(
|
||||
path.join(goRoot, "lib", "wasm", "wasm_exec.js"),
|
||||
"Go wasm_exec.js",
|
||||
);
|
||||
if ((await sha256File(license)) !== GO_LOCK.toolchain.licenseSha256) {
|
||||
throw new Error("The Go toolchain licence SHA-256 is not pinned go1.26.5.");
|
||||
}
|
||||
if ((await sha256File(wasmExec)) !== GO_LOCK.toolchain.wasmExecSha256) {
|
||||
throw new Error("wasm_exec.js does not match official go1.26.5.");
|
||||
}
|
||||
return { goRoot, go, license, wasmExec };
|
||||
}
|
||||
|
||||
function buildEnvironment(goRoot) {
|
||||
const environment = { ...process.env };
|
||||
for (const variable of [
|
||||
"GOFLAGS",
|
||||
"GOENV",
|
||||
"GOEXPERIMENT",
|
||||
"GOMOD",
|
||||
"GOTOOLDIR",
|
||||
"GOVERSION",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.CGO_ENABLED = "0";
|
||||
environment.GOARCH = "wasm";
|
||||
environment.GOENV = "off";
|
||||
environment.GOOS = "js";
|
||||
environment.GOROOT = goRoot;
|
||||
environment.GOTOOLCHAIN = "local";
|
||||
environment.GOWORK = "off";
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.SOURCE_DATE_EPOCH = String(GO_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceMetadata(root) {
|
||||
return Promise.all(
|
||||
SOURCE_FILES.map(async (relativePath) => {
|
||||
const contents = await readFile(path.join(root, relativePath));
|
||||
return {
|
||||
path: relativePath,
|
||||
sha256: sha256(contents),
|
||||
bytes: contents.byteLength,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fileMetadata(pack, name) {
|
||||
const contents = await readFile(path.join(pack, name));
|
||||
return { path: name, sha256: sha256(contents), bytes: contents.byteLength };
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "go",
|
||||
engineName: "Go standard-library regexp",
|
||||
engineVersion: GO_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
"Go 1.26.5 standard-library regexp (RE2 syntax), official js/wasm runtime",
|
||||
status: "production-native-runtime",
|
||||
bridge: {
|
||||
abiVersion: GO_LOCK.bridgeAbi,
|
||||
offsetUnit: "utf8-byte",
|
||||
requestEncoding:
|
||||
"Exact UTF-8-bounded JSON values with worst-case escape allowance; fixed callbacks; base64 UTF-8 replacement output; no source evaluation",
|
||||
supportedApplicationFlags: ["g", "i", "m", "s", "U"],
|
||||
replacement:
|
||||
"Bounded streaming expansion with regexp.ExpandString grammar, parity-checked against the native API.",
|
||||
limits: { ...GO_LOCK.limits },
|
||||
sourceFiles: await sourceMetadata(root),
|
||||
},
|
||||
source: {
|
||||
repository: GO_LOCK.source.repository,
|
||||
tag: GO_LOCK.source.tag,
|
||||
commitSha1: GO_LOCK.source.commit,
|
||||
treeSha1: GO_LOCK.source.tree,
|
||||
license: GO_LOCK.source.license,
|
||||
},
|
||||
toolchain: {
|
||||
version: GO_LOCK.runtimeVersion,
|
||||
host: "linux/amd64",
|
||||
target: "js/wasm",
|
||||
archive: GO_LOCK.toolchain.archive,
|
||||
archiveSha256: GO_LOCK.toolchain.archiveSha256,
|
||||
wasmExecSha256: GO_LOCK.toolchain.wasmExecSha256,
|
||||
build:
|
||||
"CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -buildvcs=false -ldflags=-s,-w,-buildid=",
|
||||
},
|
||||
sourceDateEpoch: GO_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(
|
||||
["LICENSE-Go.txt", "go-regex.wasm", "wasm_exec.mjs"].map((name) =>
|
||||
fileMetadata(pack, name),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function checksumText(pack) {
|
||||
const lines = await Promise.all(
|
||||
CHECKSUM_FILES.map(
|
||||
async (name) => `${await sha256File(path.join(pack, name))} ${name}`,
|
||||
),
|
||||
);
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function equalJson(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
|
||||
async function smokePack(pack) {
|
||||
const runtimeUrl = new URL(`file://${path.join(pack, "wasm_exec.mjs")}`).href;
|
||||
const wasmPath = path.join(pack, "go-regex.wasm");
|
||||
const smoke = `
|
||||
import { readFile } from "node:fs/promises";
|
||||
await import(process.argv[1]);
|
||||
let install;
|
||||
const ready = new Promise((resolve) => { install = resolve; });
|
||||
globalThis.__regexToolsInstallGoBridge = (identity, run) =>
|
||||
install({ identity, run });
|
||||
const runtime = new globalThis.Go();
|
||||
const module = await WebAssembly.instantiate(
|
||||
await readFile(process.argv[2]),
|
||||
runtime.importObject,
|
||||
);
|
||||
void runtime.run(module.instance);
|
||||
const bridge = await ready;
|
||||
const identity = JSON.parse(bridge.identity());
|
||||
if (
|
||||
identity.abi !== 1 ||
|
||||
identity.ok !== true ||
|
||||
identity.identity.goVersion !== "go1.26.5" ||
|
||||
identity.identity.selfTest !== true
|
||||
) throw new Error("Go identity self-test failed.");
|
||||
const result = JSON.parse(bridge.run(JSON.stringify({
|
||||
abi: 1,
|
||||
operation: "replace",
|
||||
pattern: "(?P<word>😀+)",
|
||||
flags: "g",
|
||||
options: {},
|
||||
subject: "x😀😀",
|
||||
scanAll: false,
|
||||
maximumMatches: 10,
|
||||
maximumCaptureRows: 10,
|
||||
replacement: "\${word}-$$",
|
||||
maximumOutputBytes: 100,
|
||||
})));
|
||||
if (
|
||||
result.ok !== true ||
|
||||
result.matches[0].span[0] !== 1 ||
|
||||
result.matches[0].span[1] !== 9 ||
|
||||
Buffer.from(result.outputBase64, "base64").toString("utf8") !== "x😀😀-$"
|
||||
) throw new Error("Go execution self-test failed.");
|
||||
process.exit(0);
|
||||
`;
|
||||
await run(process.execPath, [
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
smoke,
|
||||
runtimeUrl,
|
||||
wasmPath,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function verifyGoPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const pack = await assertRealDirectory(packDirectory, "Go engine pack");
|
||||
const entries = (await readdir(pack, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== PACK_FILES.length ||
|
||||
entries.some(
|
||||
(entry, index) =>
|
||||
entry.name !== PACK_FILES[index] ||
|
||||
!entry.isFile() ||
|
||||
entry.isSymbolicLink(),
|
||||
)
|
||||
) {
|
||||
throw new Error(`The Go pack must contain only: ${PACK_FILES.join(", ")}.`);
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "LICENSE-Go.txt"))) !==
|
||||
GO_LOCK.toolchain.licenseSha256
|
||||
) {
|
||||
throw new Error("The staged Go licence does not match go1.26.5.");
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "wasm_exec.mjs"))) !==
|
||||
GO_LOCK.toolchain.wasmExecSha256
|
||||
) {
|
||||
throw new Error("The staged wasm_exec runtime does not match go1.26.5.");
|
||||
}
|
||||
const wasm = await readFile(path.join(pack, "go-regex.wasm"));
|
||||
if (
|
||||
wasm.byteLength < 100_000 ||
|
||||
wasm.subarray(0, 4).toString("hex") !== "0061736d"
|
||||
) {
|
||||
throw new Error("The staged Go WebAssembly module is invalid.");
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("Go engine-metadata.json is invalid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (!equalJson(metadata, await expectedMetadata(repositoryRoot, pack))) {
|
||||
throw new Error(
|
||||
"The Go metadata does not match the pinned build contract.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
|
||||
(await checksumText(pack))
|
||||
) {
|
||||
throw new Error("The Go SHA256SUMS file is incorrect.");
|
||||
}
|
||||
await smokePack(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function replaceDirectory(stage, target, label) {
|
||||
const details = await lstat(target).catch(() => null);
|
||||
if (details && (!details.isDirectory() || details.isSymbolicLink())) {
|
||||
throw new Error(`${label} must be a real directory.`);
|
||||
}
|
||||
if (!details) {
|
||||
await rename(stage, target);
|
||||
return;
|
||||
}
|
||||
const backup = `${target}.replaced-${process.pid}`;
|
||||
await rename(target, backup);
|
||||
try {
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rename(backup, target);
|
||||
throw error;
|
||||
}
|
||||
await rm(backup, { recursive: true });
|
||||
}
|
||||
|
||||
export async function buildGoPack(root, goRootDirectory, outputDirectory) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const engine = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "engines", "go"),
|
||||
"Go engine source",
|
||||
);
|
||||
const toolchain = await verifyToolchain(goRootDirectory);
|
||||
const environment = buildEnvironment(toolchain.goRoot);
|
||||
const buildParent = path.join(repositoryRoot, ".engine-build");
|
||||
await mkdir(buildParent, { recursive: true });
|
||||
const temporary = await mkdtemp(path.join(buildParent, "go-build-"));
|
||||
const first = path.join(temporary, "first.wasm");
|
||||
const second = path.join(temporary, "second.wasm");
|
||||
const buildArguments = [
|
||||
"build",
|
||||
"-trimpath",
|
||||
"-buildvcs=false",
|
||||
"-ldflags=-s -w -buildid=",
|
||||
];
|
||||
try {
|
||||
await run(toolchain.go, [...buildArguments, "-o", first, "."], {
|
||||
cwd: engine,
|
||||
env: environment,
|
||||
});
|
||||
await run(toolchain.go, [...buildArguments, "-o", second, "."], {
|
||||
cwd: engine,
|
||||
env: environment,
|
||||
});
|
||||
if ((await sha256File(first)) !== (await sha256File(second))) {
|
||||
throw new Error("Two pinned Go builds produced different module hashes.");
|
||||
}
|
||||
|
||||
const target = path.resolve(
|
||||
repositoryRoot,
|
||||
outputDirectory ?? path.join(".engine-build", "go"),
|
||||
);
|
||||
const stage = await mkdtemp(path.join(buildParent, "go-pack-"));
|
||||
await copyFile(toolchain.license, path.join(stage, "LICENSE-Go.txt"));
|
||||
await copyFile(first, path.join(stage, "go-regex.wasm"));
|
||||
await copyFile(toolchain.wasmExec, path.join(stage, "wasm_exec.mjs"));
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(stage, "SHA256SUMS"), await checksumText(stage));
|
||||
await verifyGoPack(stage, repositoryRoot);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
await replaceDirectory(stage, target, "Go staged pack");
|
||||
return { metadata, output: target };
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function installGoPack(sourceDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const source = await assertRealDirectory(sourceDirectory, "Go staged pack");
|
||||
const metadata = await verifyGoPack(source, repositoryRoot);
|
||||
const engineParent = path.join(repositoryRoot, "public", "engines");
|
||||
await mkdir(engineParent, { recursive: true });
|
||||
const stage = await mkdtemp(path.join(engineParent, "go-install-"));
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(path.join(source, name), path.join(stage, name));
|
||||
}
|
||||
await verifyGoPack(stage, repositoryRoot);
|
||||
const output = path.join(engineParent, "go");
|
||||
await replaceDirectory(stage, output, "Installed Go engine pack");
|
||||
return { metadata, output };
|
||||
}
|
||||
Reference in New Issue
Block a user