feat: add Python and Java regex engines

This commit is contained in:
2026-07-27 02:02:21 +02:00
parent 4951c966d4
commit 7079cde15f
95 changed files with 8866 additions and 251 deletions

View File

@@ -40,15 +40,17 @@ const entries = (await readdir(engineDirectory)).sort();
if (
typeof packageJson.version !== "string" ||
entries.length !== 2 ||
entries.length !== 4 ||
entries[0] !== "README.md" ||
entries[1] !== "pcre2"
entries[1] !== "java" ||
entries[2] !== "pcre2" ||
entries[3] !== "python"
) {
throw new Error(
"The production build requires the documented PCRE2 runtime pack.",
"The production build requires the documented Java, PCRE2 and Python runtime packs.",
);
}
console.log(
"The checked-in PCRE2 runtime pack is present; no release-time download or engine compilation is needed.",
"The checked-in Java, PCRE2 and Python runtime packs are present; no release-time download or engine compilation is needed.",
);

View File

@@ -0,0 +1,16 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { installJavaPack } from "./java-engine-pack.mjs";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const arguments_ = process.argv.slice(2);
if (arguments_.length > 1) {
throw new Error(
"Usage: node scripts/install-java-engine.mjs [.engine-build/java]",
);
}
const source = path.resolve(root, arguments_[0] ?? ".engine-build/java");
const result = await installJavaPack(source, root);
console.log(
`Installed verified ${result.metadata.semanticIdentity} assets at ${result.output}.`,
);

View File

@@ -0,0 +1,16 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { installPythonPack } from "./python-engine-pack.mjs";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const arguments_ = process.argv.slice(2);
if (arguments_.length > 1) {
throw new Error(
"Usage: node scripts/install-python-engine.mjs [.engine-build/python]",
);
}
const source = path.resolve(root, arguments_[0] ?? ".engine-build/python");
const result = await installPythonPack(source, root);
console.log(
`Installed verified ${result.metadata.runtime} / ${result.metadata.engineVersion} assets at ${result.output}.`,
);

View File

@@ -0,0 +1,603 @@
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import {
access,
constants as fsConstants,
copyFile,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
realpath,
rename,
rm,
stat,
writeFile,
} from "node:fs/promises";
import { homedir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
export const JAVA_LOCK = Object.freeze({
bridgeAbi: 1,
engineIdentity: "TeaVM java.util.regex 0.15.0",
engineVersion: "0.15.0",
sourceDateEpoch: 1_780_759_630,
maximumPatternUtf16: 65_536,
maximumSubjectBytes: 16_777_216,
maximumReplacementUtf16: 65_536,
maximumMatches: 10_000,
maximumCaptureRows: 100_000,
maximumCaptureGroups: 1_000,
maximumOutputBytes: 67_108_864,
source: Object.freeze({
repository: "https://github.com/konsoletyper/teavm.git",
tag: "0.15.0",
commit: "ee91b03e616c4b45401cd11fb0cd7eb0daf6649b",
tree: "1cdc5c332d5809828c8952f96d73f9e7504a0cec",
license: "Apache-2.0",
licenseSha256:
"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
noticeSha256:
"6c2b3372a6beee7fbaad482248ac0f5ad2b099d92ff2a528c04261cc9c0636f4",
}),
toolchain: Object.freeze({
mavenVersion: "3.9.15",
javaVersion: "25.0.3",
artifacts: Object.freeze([
Object.freeze({
coordinate: "org.teavm:teavm-classlib:0.15.0",
relativePath:
"org/teavm/teavm-classlib/0.15.0/teavm-classlib-0.15.0.jar",
sha256:
"7d805be4670a892a34c89d9d750a8afd65e2f8201b7b3a10343634ea19d1c410",
}),
Object.freeze({
coordinate: "org.teavm:teavm-jso:0.15.0",
relativePath: "org/teavm/teavm-jso/0.15.0/teavm-jso-0.15.0.jar",
sha256:
"58b337b93924106bf819c45890c7c231a0c740145afe77795f1e0f099f621d71",
}),
Object.freeze({
coordinate: "org.teavm:teavm-maven-plugin:0.15.0",
relativePath:
"org/teavm/teavm-maven-plugin/0.15.0/teavm-maven-plugin-0.15.0.jar",
sha256:
"1133e16e45f1fd3cb93129fd96d142e1f5ef3608891d83b2273442d5446a3879",
}),
]),
}),
});
const PACK_FILES = Object.freeze([
"LICENSE.txt",
"NOTICE.txt",
"SHA256SUMS",
"engine-metadata.json",
"java-regex.mjs",
]);
const CHECKSUM_FILES = Object.freeze([
"LICENSE.txt",
"NOTICE.txt",
"engine-metadata.json",
"java-regex.mjs",
]);
const BRIDGE_SOURCE_FILES = Object.freeze([
"engines/java/pom.xml",
"engines/java/README.md",
"engines/java/LICENSE-TeaVM.txt",
"engines/java/NOTICE-TeaVM.txt",
"engines/java/src/main/java/de/addideas/regextools/javaengine/RegexBridge.java",
]);
const EXPORTED_FUNCTIONS = Object.freeze([
"bridgeAbiVersion",
"engineIdentity",
"execute",
"replace",
"selfTest",
]);
function sha256(data) {
return createHash("sha256").update(data).digest("hex");
}
async function sha256File(file) {
return sha256(await readFile(file));
}
async function assertRealDirectory(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isDirectory() || details.isSymbolicLink()) {
throw new Error(`${label} must be a real directory: ${candidate}`);
}
return realpath(candidate);
}
async function assertRegularFile(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isFile() || details.isSymbolicLink()) {
throw new Error(
`${label} must be a regular, non-symlink file: ${candidate}`,
);
}
return realpath(candidate);
}
async function resolveExecutable(name, environment = process.env) {
for (const directory of (environment.PATH ?? "").split(path.delimiter)) {
if (!directory) continue;
const candidate = path.join(directory, name);
try {
await access(candidate, fsConstants.X_OK);
if ((await stat(candidate)).isFile()) return realpath(candidate);
} catch {
// Continue through PATH.
}
}
throw new Error(`${name} is required on PATH for the TeaVM Java build.`);
}
function commandLabel(command, arguments_) {
return [command, ...arguments_]
.map((part) =>
/^[A-Za-z0-9_./:=,+-]+$/u.test(part) ? part : JSON.stringify(part),
)
.join(" ");
}
async function run(command, arguments_, options = {}) {
const child = spawn(command, arguments_, {
cwd: options.cwd,
env: options.env,
stdio: ["ignore", "pipe", "pipe"],
});
const stdout = [];
const stderr = [];
child.stdout.on("data", (chunk) => stdout.push(chunk));
child.stderr.on("data", (chunk) => stderr.push(chunk));
const result = await new Promise((resolve, reject) => {
child.once("error", reject);
child.once("close", (code, signal) => resolve({ code, signal }));
});
const output = Buffer.concat(stdout).toString("utf8");
const errorOutput = Buffer.concat(stderr).toString("utf8");
if (result.code !== 0) {
const detail = [output, errorOutput].filter(Boolean).join("\n").trim();
throw new Error(
`${commandLabel(command, arguments_)} failed with ${
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
}${detail ? `:\n${detail}` : "."}`,
);
}
return { stdout: output, stderr: errorOutput };
}
async function verifyToolchain() {
const maven = await resolveExecutable("mvn");
const java = await resolveExecutable("java");
const mavenVersion = await run(maven, ["--version"]);
const javaVersion = await run(java, ["-version"]);
if (
!mavenVersion.stdout.startsWith(
`Apache Maven ${JAVA_LOCK.toolchain.mavenVersion} `,
)
) {
throw new Error(
`The Java pack requires Maven ${JAVA_LOCK.toolchain.mavenVersion}.`,
);
}
const javaText = `${javaVersion.stdout}\n${javaVersion.stderr}`;
if (
!javaText.includes(`openjdk version "${JAVA_LOCK.toolchain.javaVersion}`)
) {
throw new Error(
`The Java pack requires OpenJDK ${JAVA_LOCK.toolchain.javaVersion}.`,
);
}
const repository = path.join(homedir(), ".m2", "repository");
for (const artifact of JAVA_LOCK.toolchain.artifacts) {
const file = await assertRegularFile(
path.join(repository, artifact.relativePath),
artifact.coordinate,
);
const actual = await sha256File(file);
if (actual !== artifact.sha256) {
throw new Error(
`${artifact.coordinate} SHA-256 mismatch: expected ${artifact.sha256}, got ${actual}.`,
);
}
}
return { maven };
}
function cleanBuildEnvironment() {
const environment = { ...process.env };
for (const variable of [
"CLASSPATH",
"JAVA_TOOL_OPTIONS",
"MAVEN_ARGS",
"MAVEN_OPTS",
"_JAVA_OPTIONS",
]) {
delete environment[variable];
}
environment.LANG = "C";
environment.LC_ALL = "C";
environment.SOURCE_DATE_EPOCH = String(JAVA_LOCK.sourceDateEpoch);
environment.TZ = "UTC";
return environment;
}
async function sourceFiles(root) {
return Promise.all(
BRIDGE_SOURCE_FILES.map(async (relativePath) => {
const file = await assertRegularFile(
path.join(root, relativePath),
`Java bridge source ${relativePath}`,
);
const contents = await readFile(file);
return {
path: relativePath,
sha256: sha256(contents),
bytes: contents.byteLength,
};
}),
);
}
async function fileMetadata(pack, name) {
const contents = await readFile(path.join(pack, name));
return { path: name, sha256: sha256(contents), bytes: contents.byteLength };
}
async function expectedMetadata(root, pack) {
return {
schemaVersion: 1,
engine: "java",
engineName: "TeaVM java.util.regex",
engineVersion: JAVA_LOCK.engineVersion,
semanticIdentity:
"TeaVM 0.15.0 classlib java.util.regex; Apache Harmony-derived; not OpenJDK",
status: "production-compatibility-runtime",
bridge: {
abiVersion: JAVA_LOCK.bridgeAbi,
maximumPatternUtf16: JAVA_LOCK.maximumPatternUtf16,
maximumSubjectBytes: JAVA_LOCK.maximumSubjectBytes,
maximumReplacementUtf16: JAVA_LOCK.maximumReplacementUtf16,
maximumMatches: JAVA_LOCK.maximumMatches,
maximumCaptureRows: JAVA_LOCK.maximumCaptureRows,
maximumCaptureGroups: JAVA_LOCK.maximumCaptureGroups,
maximumOutputBytes: JAVA_LOCK.maximumOutputBytes,
offsetUnit: "utf16",
exports: [...EXPORTED_FUNCTIONS],
sourceFiles: await sourceFiles(root),
},
source: {
repository: JAVA_LOCK.source.repository,
tag: JAVA_LOCK.source.tag,
commitSha1: JAVA_LOCK.source.commit,
treeSha1: JAVA_LOCK.source.tree,
license: JAVA_LOCK.source.license,
upstreamLicenseSha256: JAVA_LOCK.source.licenseSha256,
upstreamNoticeSha256: JAVA_LOCK.source.noticeSha256,
},
toolchain: {
mavenVersion: JAVA_LOCK.toolchain.mavenVersion,
javaVersion: JAVA_LOCK.toolchain.javaVersion,
teaVmArtifacts: JAVA_LOCK.toolchain.artifacts.map(
({ coordinate, sha256: digest }) => ({
coordinate,
sha256: digest,
}),
),
},
configuration: {
target: "JavaScript ES2015 module",
minified: true,
optimizationLevel: "ADVANCED",
sourceMaps: false,
filesystem: false,
supportedApplicationFlags: ["g", "i", "d", "m", "s", "u", "x", "U"],
unixLines: "d maps to TeaVM Pattern.UNIX_LINES.",
unicodeCharacterClassCompatibility:
"TeaVM 0.15.0 lacks OpenJDK UNICODE_CHARACTER_CLASS; U implies Unicode case folding only.",
replacement:
"TeaVM 0.15.0 Matcher appendReplacement/appendTail semantics: only single-digit $n references; ${name} is rejected; not OpenJDK Matcher parity.",
},
sourceDateEpoch: JAVA_LOCK.sourceDateEpoch,
files: await Promise.all(
["LICENSE.txt", "NOTICE.txt", "java-regex.mjs"].map((name) =>
fileMetadata(pack, name),
),
),
};
}
async function expectedChecksums(pack) {
const lines = await Promise.all(
CHECKSUM_FILES.map(
async (name) => `${await sha256File(path.join(pack, name))} ${name}`,
),
);
return `${lines.join("\n")}\n`;
}
function equalJson(left, right) {
return JSON.stringify(left) === JSON.stringify(right);
}
function readField(payload, state) {
const colon = payload.indexOf(":", state.cursor);
if (colon < 0) throw new Error("Malformed Java smoke-test payload.");
const length = Number.parseInt(payload.slice(state.cursor, colon), 10);
if (!Number.isSafeInteger(length) || length < 0) {
throw new Error("Malformed Java smoke-test field length.");
}
const start = colon + 1;
const end = start + length;
if (end > payload.length) {
throw new Error("Truncated Java smoke-test payload.");
}
state.cursor = end;
return payload.slice(start, end);
}
async function smokeJavaPack(pack) {
const moduleFile = path.join(pack, "java-regex.mjs");
const imported = await import(
`${pathToFileURL(moduleFile).href}?verify=${await sha256File(moduleFile)}`
);
for (const name of EXPORTED_FUNCTIONS) {
if (typeof imported[name] !== "function") {
throw new Error(`The Java module is missing export ${name}.`);
}
}
if (
imported.bridgeAbiVersion() !== JAVA_LOCK.bridgeAbi ||
imported.engineIdentity() !== JAVA_LOCK.engineIdentity ||
imported.selfTest() !== 0
) {
throw new Error("The TeaVM Java bridge identity or self-test failed.");
}
const execution = imported.execute("(?<=a)(b)\\1", "abb", 0, true, 10, 10);
const executionState = { cursor: 0 };
const executionFields = Array.from({ length: 9 }, () =>
readField(execution, executionState),
);
if (
executionState.cursor !== execution.length ||
JSON.stringify(executionFields) !==
JSON.stringify(["1", "ok", "1", "0", "1", "1", "3", "1", "2"])
) {
throw new Error("The TeaVM Java matching smoke test failed.");
}
const replacement = imported.replace("(a)", "a a", "<$1>", 0, true, 10, 10);
const replacementState = { cursor: 0 };
const replacementFields = [];
while (replacementState.cursor < replacement.length) {
replacementFields.push(readField(replacement, replacementState));
}
if (
replacementFields.at(-2) !== "0" ||
replacementFields.at(-1) !== "<a> <a>"
) {
throw new Error("The TeaVM Java replacement smoke test failed.");
}
}
export async function verifyJavaPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const pack = await assertRealDirectory(packDirectory, "Java engine pack");
const entries = (await readdir(pack, { withFileTypes: true })).sort(
(left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
if (
entries.length !== PACK_FILES.length ||
entries.some(
(entry, index) =>
entry.name !== PACK_FILES[index] ||
!entry.isFile() ||
entry.isSymbolicLink(),
)
) {
throw new Error(
`The staged Java pack must contain only: ${PACK_FILES.join(", ")}.`,
);
}
if (
(await sha256File(path.join(pack, "LICENSE.txt"))) !==
JAVA_LOCK.source.licenseSha256
) {
throw new Error("The staged TeaVM licence does not match tag 0.15.0.");
}
const moduleText = await readFile(path.join(pack, "java-regex.mjs"), "utf8");
if (
moduleText.includes("sourceMappingURL") ||
moduleText.includes("/mnt/") ||
moduleText.includes("/home/") ||
!moduleText.includes("as bridgeAbiVersion") ||
!moduleText.includes("as engineIdentity") ||
!moduleText.includes("as execute") ||
!moduleText.includes("as replace") ||
!moduleText.includes("as selfTest")
) {
throw new Error(
"java-regex.mjs leaks a build path, has a source map, or lacks bridge exports.",
);
}
let metadata;
try {
metadata = JSON.parse(
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
);
} catch (error) {
throw new Error("Java engine-metadata.json is not valid JSON.", {
cause: error,
});
}
const expected = await expectedMetadata(repositoryRoot, pack);
if (!equalJson(metadata, expected)) {
throw new Error(
"The staged Java metadata does not match the pinned build contract.",
);
}
if (
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
(await expectedChecksums(pack))
) {
throw new Error("The staged Java SHA256SUMS file is incorrect.");
}
await smokeJavaPack(pack);
return metadata;
}
async function replaceDirectory(stage, target, label) {
const targetDetails = await lstat(target).catch(() => null);
if (
targetDetails &&
(!targetDetails.isDirectory() || targetDetails.isSymbolicLink())
) {
throw new Error(`${label} must be a real directory.`);
}
if (!targetDetails) {
await rename(stage, target);
return;
}
const backup = `${target}.replaced-${process.pid}`;
await rename(target, backup);
try {
await rename(stage, target);
} catch (error) {
await rename(backup, target);
throw error;
}
await rm(backup, { recursive: true });
}
export async function buildJavaPack(root, outputDirectory) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const engine = await assertRealDirectory(
path.join(repositoryRoot, "engines", "java"),
"Java engine source",
);
const { maven } = await verifyToolchain();
const environment = cleanBuildEnvironment();
const buildArguments = [
"--batch-mode",
"--no-transfer-progress",
"--offline",
"clean",
"package",
];
await run(maven, buildArguments, { cwd: engine, env: environment });
const generated = await assertRegularFile(
path.join(engine, "target", "generated", "java-regex.mjs"),
"generated TeaVM module",
);
const firstBuild = await readFile(generated);
await run(maven, buildArguments, { cwd: engine, env: environment });
const secondBuild = await readFile(generated);
if (sha256(firstBuild) !== sha256(secondBuild)) {
throw new Error(
"Two clean TeaVM builds produced different Java module hashes.",
);
}
const output = path.resolve(
repositoryRoot,
outputDirectory ?? path.join(".engine-build", "java"),
);
await mkdir(path.dirname(output), { recursive: true });
const stage = await mkdtemp(path.join(path.dirname(output), ".java-build-"));
try {
await copyFile(
path.join(engine, "LICENSE-TeaVM.txt"),
path.join(stage, "LICENSE.txt"),
fsConstants.COPYFILE_EXCL,
);
await copyFile(
path.join(engine, "NOTICE-TeaVM.txt"),
path.join(stage, "NOTICE.txt"),
fsConstants.COPYFILE_EXCL,
);
await writeFile(path.join(stage, "java-regex.mjs"), secondBuild, {
flag: "wx",
});
const metadata = await expectedMetadata(repositoryRoot, stage);
await writeFile(
path.join(stage, "engine-metadata.json"),
`${JSON.stringify(metadata, null, 2)}\n`,
{ encoding: "utf8", flag: "wx" },
);
await writeFile(
path.join(stage, "SHA256SUMS"),
await expectedChecksums(stage),
{ encoding: "utf8", flag: "wx" },
);
await verifyJavaPack(stage, repositoryRoot);
await replaceDirectory(stage, output, "Java build output");
return {
output,
metadata,
moduleSha256: sha256(secondBuild),
moduleBytes: secondBuild.byteLength,
};
} finally {
await rm(stage, { recursive: true, force: true });
}
}
export async function installJavaPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const source = await assertRealDirectory(packDirectory, "verified Java pack");
const metadata = await verifyJavaPack(source, repositoryRoot);
const engineRoot = await assertRealDirectory(
path.join(repositoryRoot, "public", "engines"),
"public engine directory",
);
const target = path.join(engineRoot, "java");
const stage = await mkdtemp(path.join(engineRoot, ".java-install-"));
try {
for (const name of PACK_FILES) {
await copyFile(
path.join(source, name),
path.join(stage, name),
fsConstants.COPYFILE_EXCL,
);
}
await verifyJavaPack(stage, repositoryRoot);
await replaceDirectory(stage, target, "public/engines/java");
return { output: target, metadata };
} finally {
await rm(stage, { recursive: true, force: true });
}
}
const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : "";
if (invokedFile === fileURLToPath(import.meta.url)) {
if (process.argv.length > 3) {
throw new Error(
"Usage: node scripts/java-engine-pack.mjs [.engine-build/java]",
);
}
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const result = await buildJavaPack(root, process.argv[2]);
console.log(
`Built deterministic ${result.metadata.semanticIdentity} module: ${result.moduleSha256} (${result.moduleBytes.toLocaleString()} bytes).`,
);
}

View File

@@ -18,7 +18,7 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const zipEpoch = new Date(1980, 0, 1, 0, 0, 0);
const gplVersion3TextSha256 =
"fb981668c18a279e285fc4d83fba1e836cc84dd4daa73c9697d3cfd2d8aca6e0";
const expectedApplicationVersion = "0.2.0";
const expectedApplicationVersion = "0.3.0";
const requiredRootFiles = [
"LICENSE",
"README.md",
@@ -387,11 +387,25 @@ async function collectReleaseEntries() {
"SOURCE.md",
"THIRD_PARTY_NOTICES.md",
"engines/README.md",
"engines/java/LICENSE.txt",
"engines/java/NOTICE.txt",
"engines/java/SHA256SUMS",
"engines/java/engine-metadata.json",
"engines/java/java-regex.mjs",
"engines/pcre2/LICENSE.txt",
"engines/pcre2/SHA256SUMS",
"engines/pcre2/engine-metadata.json",
"engines/pcre2/pcre2.mjs",
"engines/pcre2/pcre2.wasm",
"engines/python/LICENSE.cpython.txt",
"engines/python/LICENSE.pyodide.txt",
"engines/python/SHA256SUMS",
"engines/python/engine-metadata.json",
"engines/python/pyodide-lock.json",
"engines/python/pyodide.asm.mjs",
"engines/python/pyodide.asm.wasm",
"engines/python/pyodide.mjs",
"engines/python/python_stdlib.zip",
"LICENSES/Emscripten-MIT.txt",
"LICENSES/PCRE2.txt",
"LICENSES/build/rolldown-1.1.5/LICENSE",

View File

@@ -0,0 +1,572 @@
import { createHash } from "node:crypto";
import {
constants as fsConstants,
copyFile,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
realpath,
rename,
rm,
writeFile,
} from "node:fs/promises";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
export const PYTHON_ENGINE_LOCK = Object.freeze({
bridgeAbi: 1,
pythonVersion: "3.14.2",
pyodideVersion: "314.0.3",
maximumPatternUtf16: 65_536,
maximumSubjectBytes: 16_777_216,
maximumReplacementUtf16: 65_536,
maximumOutputBytes: 67_108_864,
maximumMatches: 10_000,
maximumCaptureRows: 100_000,
maximumCaptureGroups: 1_000,
maximumNativeExpansionCodePoints: 16_777_216,
npm: Object.freeze({
package: "pyodide@314.0.3",
resolved: "https://registry.npmjs.org/pyodide/-/pyodide-314.0.3.tgz",
integrity:
"sha512-sK40My6m8tmBUYtYH9au9rXUeh9x0wfahtHdOlGmJxZDsKBGKtP6KznyFB2+u/klbQTdDionR0uaVd176zVQzQ==",
packageJsonSha256:
"0393eb01a0707cddc251f1a66090998773fe1aa3cfde66eedc96e83e43861845",
}),
pyodide: Object.freeze({
repository: "https://github.com/pyodide/pyodide.git",
tag: "314.0.3",
commit: "ac57031be7564f864d061cb37c5c152e59f83ad4",
license: "MPL-2.0",
licenseUrl:
"https://raw.githubusercontent.com/pyodide/pyodide/314.0.3/LICENSE",
licenseSha256:
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
}),
cpython: Object.freeze({
repository: "https://github.com/python/cpython.git",
tag: "v3.14.2",
tagObject: "a1d0069daf8e85b25a0c3f96abc43182be6d429e",
commit: "df793163d5821791d4e7caf88885a2c11a107986",
license: "Python-2.0",
licenseUrl:
"https://raw.githubusercontent.com/python/cpython/v3.14.2/LICENSE",
licenseSha256:
"b0e25a78cffb43f4d92de8b61ccfa1f1f98ecbc22330b54b5251e7b6ba010231",
}),
emscriptenVersion: "5.0.3",
abiVersion: "2026_0",
lockPythonVersion: "3.14.0",
assets: Object.freeze({
"pyodide-lock.json":
"c963d22858f6bcb8f41586a2142f03905ab370c88ea22a86a2736e95fac2a8f3",
"pyodide.asm.mjs":
"1a9775427ef6e8abaa7db88ece0515422d1886915ae5c9093776410c865dfd8d",
"pyodide.asm.wasm":
"e7f8fac36f8bf11085309cbc5c829b3ec3057c18bf1d73b05a6741612d63cdbf",
"pyodide.mjs":
"5cfc46f5dcbaf2a16f26e2363f441873eb424762609cc03db00d6a2ace4d00e5",
"python_stdlib.zip":
"444c770dfd75a32097fc0a7d5c1413fd3140601f49c3a1f2e9af0376fcd124b4",
}),
strippedLoaderSha256:
"640ac750bf7cf42ee58b524faa09f803732d2237753cd630d6878edda8a107bf",
});
const PACK_FILES = Object.freeze(
[
"LICENSE.cpython.txt",
"LICENSE.pyodide.txt",
"SHA256SUMS",
"engine-metadata.json",
...Object.keys(PYTHON_ENGINE_LOCK.assets),
].sort(),
);
const CHECKSUM_FILES = Object.freeze(
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
);
function sha256(data) {
return createHash("sha256").update(data).digest("hex");
}
async function sha256File(file) {
return sha256(await readFile(file));
}
async function assertRealDirectory(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isDirectory() || details.isSymbolicLink()) {
throw new Error(`${label} must be a real directory: ${candidate}`);
}
return realpath(candidate);
}
async function assertRegularFile(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isFile() || details.isSymbolicLink()) {
throw new Error(
`${label} must be a regular, non-symlink file: ${candidate}`,
);
}
return realpath(candidate);
}
async function assertHash(file, expected, label) {
const actual = await sha256File(file);
if (actual !== expected) {
throw new Error(
`${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`,
);
}
}
function withoutSourceMapReference(source) {
const result = source.replace(
/\n\/\/# sourceMappingURL=pyodide\.mjs\.map\s*$/u,
"\n",
);
if (result === source) {
throw new Error(
"The pinned Pyodide loader no longer has the expected source-map trailer.",
);
}
return result;
}
async function verifyPyodideSource(sourceDirectory) {
const source = await assertRealDirectory(
sourceDirectory,
"Pyodide npm package",
);
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
await assertRegularFile(path.join(source, name), `Pyodide ${name}`);
await assertHash(path.join(source, name), expected, `Pyodide ${name}`);
}
const packageFile = path.join(source, "package.json");
await assertHash(
packageFile,
PYTHON_ENGINE_LOCK.npm.packageJsonSha256,
"Pyodide package.json",
);
const packageJson = JSON.parse(await readFile(packageFile, "utf8"));
if (
packageJson.name !== "pyodide" ||
packageJson.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
packageJson.license !== PYTHON_ENGINE_LOCK.pyodide.license ||
packageJson.repository?.url !== "git+https://github.com/pyodide/pyodide.git"
) {
throw new Error("The Pyodide npm package identity is not pinned.");
}
const lock = JSON.parse(
await readFile(path.join(source, "pyodide-lock.json"), "utf8"),
);
if (
lock.info?.abi_version !== PYTHON_ENGINE_LOCK.abiVersion ||
lock.info?.arch !== "wasm32" ||
lock.info?.python !== PYTHON_ENGINE_LOCK.lockPythonVersion ||
lock.info?.platform !==
`emscripten_${PYTHON_ENGINE_LOCK.emscriptenVersion.replaceAll(".", "_")}`
) {
throw new Error("The Pyodide lock-file platform identity is not pinned.");
}
return source;
}
async function fetchPinned(url, expectedHash, label) {
const response = await fetch(url, {
redirect: "follow",
signal: AbortSignal.timeout(30_000),
});
if (!response.ok) {
throw new Error(`${label} download failed with HTTP ${response.status}.`);
}
const value = Buffer.from(await response.arrayBuffer());
const actual = sha256(value);
if (actual !== expectedHash) {
throw new Error(
`${label} SHA-256 mismatch: expected ${expectedHash}, got ${actual}.`,
);
}
return value;
}
async function fileRecord(directory, name) {
const file = path.join(directory, name);
const value = await readFile(file);
return {
path: name,
sha256: sha256(value),
bytes: value.byteLength,
};
}
async function expectedMetadata(root, pack) {
const bridgePath = path.join(root, "engines", "python", "bridge.py");
const bridge = await readFile(bridgePath);
const files = [];
for (const name of CHECKSUM_FILES) {
if (name !== "engine-metadata.json") {
files.push(await fileRecord(pack, name));
}
}
return {
schemaVersion: 1,
engine: "python",
engineVersion: `CPython ${PYTHON_ENGINE_LOCK.pythonVersion} re`,
runtime: `Pyodide ${PYTHON_ENGINE_LOCK.pyodideVersion}`,
status: "production-runtime",
bridge: {
abiVersion: PYTHON_ENGINE_LOCK.bridgeAbi,
sourceFile: {
path: "engines/python/bridge.py",
sha256: sha256(bridge),
bytes: bridge.byteLength,
},
userValuesAreFunctionArguments: true,
regexModule: "re",
supportModules: {
serialization: "json",
runtimeIdentity: "sys",
},
maximumPatternUtf16: PYTHON_ENGINE_LOCK.maximumPatternUtf16,
maximumSubjectBytes: PYTHON_ENGINE_LOCK.maximumSubjectBytes,
maximumReplacementUtf16: PYTHON_ENGINE_LOCK.maximumReplacementUtf16,
maximumOutputBytes: PYTHON_ENGINE_LOCK.maximumOutputBytes,
maximumMatches: PYTHON_ENGINE_LOCK.maximumMatches,
maximumCaptureRows: PYTHON_ENGINE_LOCK.maximumCaptureRows,
maximumCaptureGroups: PYTHON_ENGINE_LOCK.maximumCaptureGroups,
maximumNativeExpansionCodePoints:
PYTHON_ENGINE_LOCK.maximumNativeExpansionCodePoints,
},
source: {
pyodide: {
repository: PYTHON_ENGINE_LOCK.pyodide.repository,
tag: PYTHON_ENGINE_LOCK.pyodide.tag,
commitSha1: PYTHON_ENGINE_LOCK.pyodide.commit,
license: PYTHON_ENGINE_LOCK.pyodide.license,
npmPackage: PYTHON_ENGINE_LOCK.npm.package,
npmResolved: PYTHON_ENGINE_LOCK.npm.resolved,
npmIntegrity: PYTHON_ENGINE_LOCK.npm.integrity,
},
cpython: {
repository: PYTHON_ENGINE_LOCK.cpython.repository,
tag: PYTHON_ENGINE_LOCK.cpython.tag,
tagObjectSha1: PYTHON_ENGINE_LOCK.cpython.tagObject,
commitSha1: PYTHON_ENGINE_LOCK.cpython.commit,
license: PYTHON_ENGINE_LOCK.cpython.license,
},
},
toolchain: {
abiVersion: PYTHON_ENGINE_LOCK.abiVersion,
architecture: "wasm32",
emscriptenVersion: PYTHON_ENGINE_LOCK.emscriptenVersion,
lockPythonVersion: PYTHON_ENGINE_LOCK.lockPythonVersion,
executablePythonVersion: PYTHON_ENGINE_LOCK.pythonVersion,
threads: false,
},
packaging: {
loaderTransformation:
"Removed only the upstream sourceMappingURL trailer; executable JavaScript is unchanged.",
upstreamLoaderSha256: PYTHON_ENGINE_LOCK.assets["pyodide.mjs"],
packedLoaderSha256: PYTHON_ENGINE_LOCK.strippedLoaderSha256,
externalPackagesIncluded: false,
lockPythonVersionNote:
"The upstream lock's 3.14.0 field identifies its CPython 3.14 ABI baseline; the executable runtime self-identifies as 3.14.2.",
},
files,
};
}
function equalJson(left, right) {
return JSON.stringify(left) === JSON.stringify(right);
}
async function expectedChecksums(pack) {
const lines = [];
for (const name of CHECKSUM_FILES) {
lines.push(`${await sha256File(path.join(pack, name))} ${name}`);
}
return `${lines.join("\n")}\n`;
}
async function smokePythonPack(pack) {
await WebAssembly.compile(
await readFile(path.join(pack, "pyodide.asm.wasm")),
);
const moduleUrl = pathToFileURL(path.join(pack, "pyodide.mjs"));
moduleUrl.searchParams.set("regex-tools-verification", String(Date.now()));
const imported = await import(moduleUrl.href);
if (
imported.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
typeof imported.loadPyodide !== "function"
) {
throw new Error("The staged Pyodide module identity is invalid.");
}
const runtime = await imported.loadPyodide({
indexURL: `${pack}${path.sep}`,
stdout: () => undefined,
stderr: () => undefined,
});
const result = JSON.parse(
runtime.runPython(`
import json, re, sys
expression = re.compile(r"(?P<emoji>😀)|(?P<word>[a-z]+)", re.ASCII)
matches = [
[match.span(0), match.lastgroup]
for match in expression.finditer("😀 abc")
]
replacement_match = re.compile(r"(?P<word>[a-z]+)").search("abc")
json.dumps({
"python": ".".join(str(part) for part in sys.version_info[:3]),
"implementation": sys.implementation.name,
"matches": matches,
"replacement": replacement_match.expand(r"[\\g<word>]"),
})
`),
);
if (
result.python !== PYTHON_ENGINE_LOCK.pythonVersion ||
result.implementation !== "cpython" ||
JSON.stringify(result.matches) !==
JSON.stringify([
[[0, 1], "emoji"],
[[2, 5], "word"],
]) ||
result.replacement !== "[abc]"
) {
throw new Error("The staged CPython re runtime smoke test failed.");
}
}
export async function verifyPythonPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const pack = await assertRealDirectory(packDirectory, "Python engine pack");
const entries = (await readdir(pack, { withFileTypes: true })).sort(
(left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
if (
entries.length !== PACK_FILES.length ||
entries.some(
(entry, index) =>
entry.name !== PACK_FILES[index] ||
!entry.isFile() ||
entry.isSymbolicLink(),
)
) {
throw new Error(
`The staged Python pack must contain only: ${PACK_FILES.join(", ")}.`,
);
}
await assertHash(
path.join(pack, "LICENSE.pyodide.txt"),
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
"staged Pyodide licence",
);
await assertHash(
path.join(pack, "LICENSE.cpython.txt"),
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
"staged CPython licence",
);
await assertHash(
path.join(pack, "pyodide.mjs"),
PYTHON_ENGINE_LOCK.strippedLoaderSha256,
"staged Pyodide loader",
);
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
if (name !== "pyodide.mjs") {
await assertHash(path.join(pack, name), expected, `staged ${name}`);
}
}
const loader = await readFile(path.join(pack, "pyodide.mjs"), "utf8");
if (
loader.includes("sourceMappingURL") ||
loader.includes("/mnt/DATA/") ||
loader.includes("/home/zemion/")
) {
throw new Error(
"The staged Pyodide loader leaks a source-map or local build path.",
);
}
let metadata;
try {
metadata = JSON.parse(
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
);
} catch (error) {
throw new Error("engine-metadata.json is not valid JSON.", {
cause: error,
});
}
const expected = await expectedMetadata(repositoryRoot, pack);
if (!equalJson(metadata, expected)) {
throw new Error(
"The staged Python metadata does not match the pinned pack contract.",
);
}
if (
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
(await expectedChecksums(pack))
) {
throw new Error("The staged Python SHA256SUMS file is incorrect.");
}
await smokePythonPack(pack);
return metadata;
}
async function replaceDirectory(stage, target, label) {
const targetDetails = await lstat(target).catch(() => null);
if (
targetDetails &&
(!targetDetails.isDirectory() || targetDetails.isSymbolicLink())
) {
throw new Error(`${label} must be a real directory.`);
}
if (!targetDetails) {
await rename(stage, target);
return;
}
const backup = `${target}.replaced-${process.pid}-${Date.now()}`;
await rename(target, backup);
try {
await rename(stage, target);
} catch (error) {
await rename(backup, target);
throw error;
}
await rm(backup, { recursive: true });
}
export async function installPythonPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const sourcePack = await assertRealDirectory(
packDirectory,
"verified Python engine pack",
);
const metadata = await verifyPythonPack(sourcePack, repositoryRoot);
const engineRoot = await assertRealDirectory(
path.join(repositoryRoot, "public", "engines"),
"public engine directory",
);
const target = path.join(engineRoot, "python");
const stage = await mkdtemp(path.join(engineRoot, ".python-install-"));
try {
for (const name of PACK_FILES) {
await copyFile(
path.join(sourcePack, name),
path.join(stage, name),
fsConstants.COPYFILE_EXCL,
);
}
await verifyPythonPack(stage, repositoryRoot);
await replaceDirectory(stage, target, "public/engines/python");
return { output: target, metadata };
} finally {
await rm(stage, { recursive: true, force: true });
}
}
export async function buildPythonPack(options, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const source = await verifyPyodideSource(options.sourceDirectory);
const output = path.resolve(options.outputDirectory);
await mkdir(path.dirname(output), { recursive: true });
const stage = await mkdtemp(
path.join(path.dirname(output), ".python-pack-build-"),
);
try {
const [pyodideLicense, cpythonLicense] = await Promise.all([
fetchPinned(
PYTHON_ENGINE_LOCK.pyodide.licenseUrl,
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
"Pyodide licence",
),
fetchPinned(
PYTHON_ENGINE_LOCK.cpython.licenseUrl,
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
"CPython licence",
),
]);
await writeFile(path.join(stage, "LICENSE.pyodide.txt"), pyodideLicense, {
flag: "wx",
});
await writeFile(path.join(stage, "LICENSE.cpython.txt"), cpythonLicense, {
flag: "wx",
});
for (const name of Object.keys(PYTHON_ENGINE_LOCK.assets)) {
if (name === "pyodide.mjs") {
const upstream = await readFile(path.join(source, name), "utf8");
const packed = withoutSourceMapReference(upstream);
if (sha256(packed) !== PYTHON_ENGINE_LOCK.strippedLoaderSha256) {
throw new Error("The deterministic Pyodide loader output changed.");
}
await writeFile(path.join(stage, name), packed, { flag: "wx" });
} else {
await copyFile(
path.join(source, name),
path.join(stage, name),
fsConstants.COPYFILE_EXCL,
);
}
}
const metadata = await expectedMetadata(repositoryRoot, stage);
await writeFile(
path.join(stage, "engine-metadata.json"),
`${JSON.stringify(metadata, null, 2)}\n`,
{ flag: "wx" },
);
await writeFile(
path.join(stage, "SHA256SUMS"),
await expectedChecksums(stage),
{ flag: "wx" },
);
await verifyPythonPack(stage, repositoryRoot);
await replaceDirectory(stage, output, "Python pack output");
return { output, metadata };
} finally {
await rm(stage, { recursive: true, force: true });
}
}
const invokedFile = process.argv[1]
? pathToFileURL(path.resolve(process.argv[1])).href
: "";
if (import.meta.url === invokedFile) {
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const arguments_ = process.argv.slice(2);
if (arguments_.length > 2) {
throw new Error(
"Usage: node scripts/python-engine-pack.mjs [node_modules/pyodide] [.engine-build/python]",
);
}
const result = await buildPythonPack(
{
sourceDirectory: path.resolve(
root,
arguments_[0] ?? "node_modules/pyodide",
),
outputDirectory: path.resolve(
root,
arguments_[1] ?? ".engine-build/python",
),
},
root,
);
console.log(
`Built and verified ${result.metadata.runtime} / ${result.metadata.engineVersion} at ${result.output}.`,
);
}

View File

@@ -26,6 +26,14 @@ const testCatalogue = {
theme: { mode: "system", brand: "add·ideas" },
apps: [{ manifest: "./toolbox-app.json", enabled: true }],
};
const productionSecurityHeaders = {
"Content-Security-Policy":
"default-src 'self'; base-uri 'self'; object-src 'none'; frame-ancestors 'none'; form-action 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; media-src 'self' blob:; font-src 'self' data:; connect-src 'self'; worker-src 'self' blob:; manifest-src 'self'",
"Cross-Origin-Embedder-Policy": "require-corp",
"Cross-Origin-Opener-Policy": "same-origin",
"Cross-Origin-Resource-Policy": "same-origin",
"X-Content-Type-Options": "nosniff",
};
function safeFile(requestPath) {
const decoded = decodeURIComponent(requestPath);
@@ -50,6 +58,7 @@ const server = createServer(async (request, response) => {
response.writeHead(200, {
"Content-Type": "application/json; charset=utf-8",
"Cache-Control": "no-cache",
...productionSecurityHeaders,
});
response.end(JSON.stringify(testCatalogue));
return;
@@ -66,7 +75,7 @@ const server = createServer(async (request, response) => {
"Content-Type":
mediaTypes.get(path.extname(file)) ?? "application/octet-stream",
"Cache-Control": "no-cache",
"Cross-Origin-Resource-Policy": "same-origin",
...productionSecurityHeaders,
});
response.end(content);
} catch {

View File

@@ -1,22 +1,28 @@
import { lstat, readFile, readdir } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { verifyJavaPack } from "./java-engine-pack.mjs";
import { verifyPcre2Pack } from "./pcre2-engine-pack.mjs";
import { verifyPythonPack } from "./python-engine-pack.mjs";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const arguments_ = process.argv.slice(2);
if (arguments_.length > 0) {
if (arguments_.length !== 2 || arguments_[0] !== "--pcre2-pack") {
const verifiers = new Map([
["--java-pack", verifyJavaPack],
["--pcre2-pack", verifyPcre2Pack],
["--python-pack", verifyPythonPack],
]);
const verifier = verifiers.get(arguments_[0]);
if (arguments_.length !== 2 || !verifier) {
throw new Error(
"Usage: node scripts/verify-engine-assets.mjs [--pcre2-pack PATH]",
"Usage: node scripts/verify-engine-assets.mjs [--java-pack|--pcre2-pack|--python-pack PATH]",
);
}
const pack = path.resolve(root, arguments_[1]);
const metadata = await verifyPcre2Pack(pack, root);
console.log(
`Verified staged PCRE2 ${metadata.engineVersion} pack and WebAssembly bridge smoke test.`,
);
const metadata = await verifier(pack, root);
console.log(`Verified staged ${metadata.engineVersion} engine pack.`);
process.exit(0);
}
@@ -31,21 +37,34 @@ const entries = (await readdir(engineDirectory, { withFileTypes: true })).sort(
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
if (
entries.length !== 2 ||
entries.length !== 4 ||
!entries[0]?.isFile() ||
entries[0].name !== "README.md" ||
!entries[1]?.isDirectory() ||
entries[1].name !== "pcre2"
entries[1].name !== "java" ||
!entries[2]?.isDirectory() ||
entries[2].name !== "pcre2" ||
!entries[3]?.isDirectory() ||
entries[3].name !== "python"
) {
throw new Error(
"The production build must contain only README.md and the declared PCRE2 pack.",
"The production build must contain only README.md and the declared Java, PCRE2 and Python packs.",
);
}
const notice = await readFile(path.join(engineDirectory, "README.md"), "utf8");
if (!notice.includes("native `RegExp`") || !notice.includes("PCRE2 10.47")) {
if (
!notice.includes("native `RegExp`") ||
!notice.includes("PCRE2 10.47") ||
!notice.includes("Pyodide 314.0.3") ||
!notice.includes("TeaVM 0.15.0")
) {
throw new Error("The engine asset notice does not describe this release.");
}
await verifyPcre2Pack(path.join(engineDirectory, "pcre2"), root);
await verifyPythonPack(path.join(engineDirectory, "python"), root);
await verifyJavaPack(path.join(engineDirectory, "java"), root);
console.log("Verified native ECMAScript and bundled PCRE2 engine assets.");
console.log(
"Verified native ECMAScript and bundled PCRE2, CPython/Pyodide and TeaVM Java engine assets.",
);