lotobo/regex-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

feat: add Python and Java regex engines

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-27 02:02:21 +02:00
parent 4951c966d4
commit 7079cde15f
95 changed files with 8866 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

572
scripts/python-engine-pack.mjs Normal file
View File

@@ -0,0 +1,572 @@
import { createHash } from "node:crypto";
import {
constants as fsConstants,
copyFile,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
realpath,
rename,
rm,
writeFile,
} from "node:fs/promises";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
export const PYTHON_ENGINE_LOCK = Object.freeze({
bridgeAbi: 1,
pythonVersion: "3.14.2",
pyodideVersion: "314.0.3",
maximumPatternUtf16: 65_536,
maximumSubjectBytes: 16_777_216,
maximumReplacementUtf16: 65_536,
maximumOutputBytes: 67_108_864,
maximumMatches: 10_000,
maximumCaptureRows: 100_000,
maximumCaptureGroups: 1_000,
maximumNativeExpansionCodePoints: 16_777_216,
npm: Object.freeze({
package: "pyodide@314.0.3",
resolved: "https://registry.npmjs.org/pyodide/-/pyodide-314.0.3.tgz",
integrity:
"sha512-sK40My6m8tmBUYtYH9au9rXUeh9x0wfahtHdOlGmJxZDsKBGKtP6KznyFB2+u/klbQTdDionR0uaVd176zVQzQ==",
packageJsonSha256:
"0393eb01a0707cddc251f1a66090998773fe1aa3cfde66eedc96e83e43861845",
}),
pyodide: Object.freeze({
repository: "https://github.com/pyodide/pyodide.git",
tag: "314.0.3",
commit: "ac57031be7564f864d061cb37c5c152e59f83ad4",
license: "MPL-2.0",
licenseUrl:
"https://raw.githubusercontent.com/pyodide/pyodide/314.0.3/LICENSE",
licenseSha256:
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
}),
cpython: Object.freeze({
repository: "https://github.com/python/cpython.git",
tag: "v3.14.2",
tagObject: "a1d0069daf8e85b25a0c3f96abc43182be6d429e",
commit: "df793163d5821791d4e7caf88885a2c11a107986",
license: "Python-2.0",
licenseUrl:
"https://raw.githubusercontent.com/python/cpython/v3.14.2/LICENSE",
licenseSha256:
"b0e25a78cffb43f4d92de8b61ccfa1f1f98ecbc22330b54b5251e7b6ba010231",
}),
emscriptenVersion: "5.0.3",
abiVersion: "2026_0",
lockPythonVersion: "3.14.0",
assets: Object.freeze({
"pyodide-lock.json":
"c963d22858f6bcb8f41586a2142f03905ab370c88ea22a86a2736e95fac2a8f3",
"pyodide.asm.mjs":
"1a9775427ef6e8abaa7db88ece0515422d1886915ae5c9093776410c865dfd8d",
"pyodide.asm.wasm":
"e7f8fac36f8bf11085309cbc5c829b3ec3057c18bf1d73b05a6741612d63cdbf",
"pyodide.mjs":
"5cfc46f5dcbaf2a16f26e2363f441873eb424762609cc03db00d6a2ace4d00e5",
"python_stdlib.zip":
"444c770dfd75a32097fc0a7d5c1413fd3140601f49c3a1f2e9af0376fcd124b4",
}),
strippedLoaderSha256:
"640ac750bf7cf42ee58b524faa09f803732d2237753cd630d6878edda8a107bf",
});
const PACK_FILES = Object.freeze(
[
"LICENSE.cpython.txt",
"LICENSE.pyodide.txt",
"SHA256SUMS",
"engine-metadata.json",
...Object.keys(PYTHON_ENGINE_LOCK.assets),
].sort(),
);
const CHECKSUM_FILES = Object.freeze(
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
);
function sha256(data) {
return createHash("sha256").update(data).digest("hex");
}
async function sha256File(file) {
return sha256(await readFile(file));
}
async function assertRealDirectory(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isDirectory() || details.isSymbolicLink()) {
throw new Error(`${label} must be a real directory: ${candidate}`);
}
return realpath(candidate);
}
async function assertRegularFile(candidate, label) {
const details = await lstat(candidate).catch(() => null);
if (!details?.isFile() || details.isSymbolicLink()) {
throw new Error(
`${label} must be a regular, non-symlink file: ${candidate}`,
);
}
return realpath(candidate);
}
async function assertHash(file, expected, label) {
const actual = await sha256File(file);
if (actual !== expected) {
throw new Error(
`${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`,
);
}
}
function withoutSourceMapReference(source) {
const result = source.replace(
/\n\/\/# sourceMappingURL=pyodide\.mjs\.map\s*$/u,
"\n",
);
if (result === source) {
throw new Error(
"The pinned Pyodide loader no longer has the expected source-map trailer.",
);
}
return result;
}
async function verifyPyodideSource(sourceDirectory) {
const source = await assertRealDirectory(
sourceDirectory,
"Pyodide npm package",
);
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
await assertRegularFile(path.join(source, name), `Pyodide ${name}`);
await assertHash(path.join(source, name), expected, `Pyodide ${name}`);
}
const packageFile = path.join(source, "package.json");
await assertHash(
packageFile,
PYTHON_ENGINE_LOCK.npm.packageJsonSha256,
"Pyodide package.json",
);
const packageJson = JSON.parse(await readFile(packageFile, "utf8"));
if (
packageJson.name !== "pyodide" ||
packageJson.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
packageJson.license !== PYTHON_ENGINE_LOCK.pyodide.license ||
packageJson.repository?.url !== "git+https://github.com/pyodide/pyodide.git"
) {
throw new Error("The Pyodide npm package identity is not pinned.");
}
const lock = JSON.parse(
await readFile(path.join(source, "pyodide-lock.json"), "utf8"),
);
if (
lock.info?.abi_version !== PYTHON_ENGINE_LOCK.abiVersion ||
lock.info?.arch !== "wasm32" ||
lock.info?.python !== PYTHON_ENGINE_LOCK.lockPythonVersion ||
lock.info?.platform !==
`emscripten_${PYTHON_ENGINE_LOCK.emscriptenVersion.replaceAll(".", "_")}`
) {
throw new Error("The Pyodide lock-file platform identity is not pinned.");
}
return source;
}
async function fetchPinned(url, expectedHash, label) {
const response = await fetch(url, {
redirect: "follow",
signal: AbortSignal.timeout(30_000),
});
if (!response.ok) {
throw new Error(`${label} download failed with HTTP ${response.status}.`);
}
const value = Buffer.from(await response.arrayBuffer());
const actual = sha256(value);
if (actual !== expectedHash) {
throw new Error(
`${label} SHA-256 mismatch: expected ${expectedHash}, got ${actual}.`,
);
}
return value;
}
async function fileRecord(directory, name) {
const file = path.join(directory, name);
const value = await readFile(file);
return {
path: name,
sha256: sha256(value),
bytes: value.byteLength,
};
}
async function expectedMetadata(root, pack) {
const bridgePath = path.join(root, "engines", "python", "bridge.py");
const bridge = await readFile(bridgePath);
const files = [];
for (const name of CHECKSUM_FILES) {
if (name !== "engine-metadata.json") {
files.push(await fileRecord(pack, name));
}
}
return {
schemaVersion: 1,
engine: "python",
engineVersion: `CPython ${PYTHON_ENGINE_LOCK.pythonVersion} re`,
runtime: `Pyodide ${PYTHON_ENGINE_LOCK.pyodideVersion}`,
status: "production-runtime",
bridge: {
abiVersion: PYTHON_ENGINE_LOCK.bridgeAbi,
sourceFile: {
path: "engines/python/bridge.py",
sha256: sha256(bridge),
bytes: bridge.byteLength,
},
userValuesAreFunctionArguments: true,
regexModule: "re",
supportModules: {
serialization: "json",
runtimeIdentity: "sys",
},
maximumPatternUtf16: PYTHON_ENGINE_LOCK.maximumPatternUtf16,
maximumSubjectBytes: PYTHON_ENGINE_LOCK.maximumSubjectBytes,
maximumReplacementUtf16: PYTHON_ENGINE_LOCK.maximumReplacementUtf16,
maximumOutputBytes: PYTHON_ENGINE_LOCK.maximumOutputBytes,
maximumMatches: PYTHON_ENGINE_LOCK.maximumMatches,
maximumCaptureRows: PYTHON_ENGINE_LOCK.maximumCaptureRows,
maximumCaptureGroups: PYTHON_ENGINE_LOCK.maximumCaptureGroups,
maximumNativeExpansionCodePoints:
PYTHON_ENGINE_LOCK.maximumNativeExpansionCodePoints,
},
source: {
pyodide: {
repository: PYTHON_ENGINE_LOCK.pyodide.repository,
tag: PYTHON_ENGINE_LOCK.pyodide.tag,
commitSha1: PYTHON_ENGINE_LOCK.pyodide.commit,
license: PYTHON_ENGINE_LOCK.pyodide.license,
npmPackage: PYTHON_ENGINE_LOCK.npm.package,
npmResolved: PYTHON_ENGINE_LOCK.npm.resolved,
npmIntegrity: PYTHON_ENGINE_LOCK.npm.integrity,
},
cpython: {
repository: PYTHON_ENGINE_LOCK.cpython.repository,
tag: PYTHON_ENGINE_LOCK.cpython.tag,
tagObjectSha1: PYTHON_ENGINE_LOCK.cpython.tagObject,
commitSha1: PYTHON_ENGINE_LOCK.cpython.commit,
license: PYTHON_ENGINE_LOCK.cpython.license,
},
},
toolchain: {
abiVersion: PYTHON_ENGINE_LOCK.abiVersion,
architecture: "wasm32",
emscriptenVersion: PYTHON_ENGINE_LOCK.emscriptenVersion,
lockPythonVersion: PYTHON_ENGINE_LOCK.lockPythonVersion,
executablePythonVersion: PYTHON_ENGINE_LOCK.pythonVersion,
threads: false,
},
packaging: {
loaderTransformation:
"Removed only the upstream sourceMappingURL trailer; executable JavaScript is unchanged.",
upstreamLoaderSha256: PYTHON_ENGINE_LOCK.assets["pyodide.mjs"],
packedLoaderSha256: PYTHON_ENGINE_LOCK.strippedLoaderSha256,
externalPackagesIncluded: false,
lockPythonVersionNote:
"The upstream lock's 3.14.0 field identifies its CPython 3.14 ABI baseline; the executable runtime self-identifies as 3.14.2.",
},
files,
};
}
function equalJson(left, right) {
return JSON.stringify(left) === JSON.stringify(right);
}
async function expectedChecksums(pack) {
const lines = [];
for (const name of CHECKSUM_FILES) {
lines.push(`${await sha256File(path.join(pack, name))} ${name}`);
}
return `${lines.join("\n")}\n`;
}
async function smokePythonPack(pack) {
await WebAssembly.compile(
await readFile(path.join(pack, "pyodide.asm.wasm")),
);
const moduleUrl = pathToFileURL(path.join(pack, "pyodide.mjs"));
moduleUrl.searchParams.set("regex-tools-verification", String(Date.now()));
const imported = await import(moduleUrl.href);
if (
imported.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
typeof imported.loadPyodide !== "function"
) {
throw new Error("The staged Pyodide module identity is invalid.");
}
const runtime = await imported.loadPyodide({
indexURL: `${pack}${path.sep}`,
stdout: () => undefined,
stderr: () => undefined,
});
const result = JSON.parse(
runtime.runPython(`
import json, re, sys
expression = re.compile(r"(?P<emoji>😀)|(?P<word>[a-z]+)", re.ASCII)
matches = [
[match.span(0), match.lastgroup]
for match in expression.finditer("😀 abc")
]
replacement_match = re.compile(r"(?P<word>[a-z]+)").search("abc")
json.dumps({
"python": ".".join(str(part) for part in sys.version_info[:3]),
"implementation": sys.implementation.name,
"matches": matches,
"replacement": replacement_match.expand(r"[\\g<word>]"),
})
`),
);
if (
result.python !== PYTHON_ENGINE_LOCK.pythonVersion ||
result.implementation !== "cpython" ||
JSON.stringify(result.matches) !==
JSON.stringify([
[[0, 1], "emoji"],
[[2, 5], "word"],
]) ||
result.replacement !== "[abc]"
) {
throw new Error("The staged CPython re runtime smoke test failed.");
}
}
export async function verifyPythonPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const pack = await assertRealDirectory(packDirectory, "Python engine pack");
const entries = (await readdir(pack, { withFileTypes: true })).sort(
(left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
if (
entries.length !== PACK_FILES.length ||
entries.some(
(entry, index) =>
entry.name !== PACK_FILES[index] ||
!entry.isFile() ||
entry.isSymbolicLink(),
)
) {
throw new Error(
`The staged Python pack must contain only: ${PACK_FILES.join(", ")}.`,
);
}
await assertHash(
path.join(pack, "LICENSE.pyodide.txt"),
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
"staged Pyodide licence",
);
await assertHash(
path.join(pack, "LICENSE.cpython.txt"),
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
"staged CPython licence",
);
await assertHash(
path.join(pack, "pyodide.mjs"),
PYTHON_ENGINE_LOCK.strippedLoaderSha256,
"staged Pyodide loader",
);
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
if (name !== "pyodide.mjs") {
await assertHash(path.join(pack, name), expected, `staged ${name}`);
}
}
const loader = await readFile(path.join(pack, "pyodide.mjs"), "utf8");
if (
loader.includes("sourceMappingURL") ||
loader.includes("/mnt/DATA/") ||
loader.includes("/home/zemion/")
) {
throw new Error(
"The staged Pyodide loader leaks a source-map or local build path.",
);
}
let metadata;
try {
metadata = JSON.parse(
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
);
} catch (error) {
throw new Error("engine-metadata.json is not valid JSON.", {
cause: error,
});
}
const expected = await expectedMetadata(repositoryRoot, pack);
if (!equalJson(metadata, expected)) {
throw new Error(
"The staged Python metadata does not match the pinned pack contract.",
);
}
if (
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
(await expectedChecksums(pack))
) {
throw new Error("The staged Python SHA256SUMS file is incorrect.");
}
await smokePythonPack(pack);
return metadata;
}
async function replaceDirectory(stage, target, label) {
const targetDetails = await lstat(target).catch(() => null);
if (
targetDetails &&
(!targetDetails.isDirectory() || targetDetails.isSymbolicLink())
) {
throw new Error(`${label} must be a real directory.`);
}
if (!targetDetails) {
await rename(stage, target);
return;
}
const backup = `${target}.replaced-${process.pid}-${Date.now()}`;
await rename(target, backup);
try {
await rename(stage, target);
} catch (error) {
await rename(backup, target);
throw error;
}
await rm(backup, { recursive: true });
}
export async function installPythonPack(packDirectory, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const sourcePack = await assertRealDirectory(
packDirectory,
"verified Python engine pack",
);
const metadata = await verifyPythonPack(sourcePack, repositoryRoot);
const engineRoot = await assertRealDirectory(
path.join(repositoryRoot, "public", "engines"),
"public engine directory",
);
const target = path.join(engineRoot, "python");
const stage = await mkdtemp(path.join(engineRoot, ".python-install-"));
try {
for (const name of PACK_FILES) {
await copyFile(
path.join(sourcePack, name),
path.join(stage, name),
fsConstants.COPYFILE_EXCL,
);
}
await verifyPythonPack(stage, repositoryRoot);
await replaceDirectory(stage, target, "public/engines/python");
return { output: target, metadata };
} finally {
await rm(stage, { recursive: true, force: true });
}
}
export async function buildPythonPack(options, root) {
const repositoryRoot = await assertRealDirectory(
root,
"Regex Tools repository",
);
const source = await verifyPyodideSource(options.sourceDirectory);
const output = path.resolve(options.outputDirectory);
await mkdir(path.dirname(output), { recursive: true });
const stage = await mkdtemp(
path.join(path.dirname(output), ".python-pack-build-"),
);
try {
const [pyodideLicense, cpythonLicense] = await Promise.all([
fetchPinned(
PYTHON_ENGINE_LOCK.pyodide.licenseUrl,
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
"Pyodide licence",
),
fetchPinned(
PYTHON_ENGINE_LOCK.cpython.licenseUrl,
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
"CPython licence",
),
]);
await writeFile(path.join(stage, "LICENSE.pyodide.txt"), pyodideLicense, {
flag: "wx",
});
await writeFile(path.join(stage, "LICENSE.cpython.txt"), cpythonLicense, {
flag: "wx",
});
for (const name of Object.keys(PYTHON_ENGINE_LOCK.assets)) {
if (name === "pyodide.mjs") {
const upstream = await readFile(path.join(source, name), "utf8");
const packed = withoutSourceMapReference(upstream);
if (sha256(packed) !== PYTHON_ENGINE_LOCK.strippedLoaderSha256) {
throw new Error("The deterministic Pyodide loader output changed.");
}
await writeFile(path.join(stage, name), packed, { flag: "wx" });
} else {
await copyFile(
path.join(source, name),
path.join(stage, name),
fsConstants.COPYFILE_EXCL,
);
}
}
const metadata = await expectedMetadata(repositoryRoot, stage);
await writeFile(
path.join(stage, "engine-metadata.json"),
`${JSON.stringify(metadata, null, 2)}\n`,
{ flag: "wx" },
);
await writeFile(
path.join(stage, "SHA256SUMS"),
await expectedChecksums(stage),
{ flag: "wx" },
);
await verifyPythonPack(stage, repositoryRoot);
await replaceDirectory(stage, output, "Python pack output");
return { output, metadata };
} finally {
await rm(stage, { recursive: true, force: true });
}
}
const invokedFile = process.argv[1]
? pathToFileURL(path.resolve(process.argv[1])).href
: "";
if (import.meta.url === invokedFile) {
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const arguments_ = process.argv.slice(2);
if (arguments_.length > 2) {
throw new Error(
"Usage: node scripts/python-engine-pack.mjs [node_modules/pyodide] [.engine-build/python]",
);
}
const result = await buildPythonPack(
{
sourceDirectory: path.resolve(
root,
arguments_[0] ?? "node_modules/pyodide",
),
outputDirectory: path.resolve(
root,
arguments_[1] ?? ".engine-build/python",
),
},
root,
);
console.log(
`Built and verified ${result.metadata.runtime} / ${result.metadata.engineVersion} at ${result.output}.`,
);
}