feat: add Python and Java regex engines
This commit is contained in:
@@ -40,15 +40,17 @@ const entries = (await readdir(engineDirectory)).sort();
|
||||
|
||||
if (
|
||||
typeof packageJson.version !== "string" ||
|
||||
entries.length !== 2 ||
|
||||
entries.length !== 4 ||
|
||||
entries[0] !== "README.md" ||
|
||||
entries[1] !== "pcre2"
|
||||
entries[1] !== "java" ||
|
||||
entries[2] !== "pcre2" ||
|
||||
entries[3] !== "python"
|
||||
) {
|
||||
throw new Error(
|
||||
"The production build requires the documented PCRE2 runtime pack.",
|
||||
"The production build requires the documented Java, PCRE2 and Python runtime packs.",
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
"The checked-in PCRE2 runtime pack is present; no release-time download or engine compilation is needed.",
|
||||
"The checked-in Java, PCRE2 and Python runtime packs are present; no release-time download or engine compilation is needed.",
|
||||
);
|
||||
|
||||
16
scripts/install-java-engine.mjs
Normal file
16
scripts/install-java-engine.mjs
Normal file
@@ -0,0 +1,16 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installJavaPack } from "./java-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-java-engine.mjs [.engine-build/java]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/java");
|
||||
const result = await installJavaPack(source, root);
|
||||
console.log(
|
||||
`Installed verified ${result.metadata.semanticIdentity} assets at ${result.output}.`,
|
||||
);
|
||||
16
scripts/install-python-engine.mjs
Normal file
16
scripts/install-python-engine.mjs
Normal file
@@ -0,0 +1,16 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installPythonPack } from "./python-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 1) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/install-python-engine.mjs [.engine-build/python]",
|
||||
);
|
||||
}
|
||||
const source = path.resolve(root, arguments_[0] ?? ".engine-build/python");
|
||||
const result = await installPythonPack(source, root);
|
||||
console.log(
|
||||
`Installed verified ${result.metadata.runtime} / ${result.metadata.engineVersion} assets at ${result.output}.`,
|
||||
);
|
||||
603
scripts/java-engine-pack.mjs
Normal file
603
scripts/java-engine-pack.mjs
Normal file
@@ -0,0 +1,603 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
export const JAVA_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
engineIdentity: "TeaVM java.util.regex 0.15.0",
|
||||
engineVersion: "0.15.0",
|
||||
sourceDateEpoch: 1_780_759_630,
|
||||
maximumPatternUtf16: 65_536,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementUtf16: 65_536,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
source: Object.freeze({
|
||||
repository: "https://github.com/konsoletyper/teavm.git",
|
||||
tag: "0.15.0",
|
||||
commit: "ee91b03e616c4b45401cd11fb0cd7eb0daf6649b",
|
||||
tree: "1cdc5c332d5809828c8952f96d73f9e7504a0cec",
|
||||
license: "Apache-2.0",
|
||||
licenseSha256:
|
||||
"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
|
||||
noticeSha256:
|
||||
"6c2b3372a6beee7fbaad482248ac0f5ad2b099d92ff2a528c04261cc9c0636f4",
|
||||
}),
|
||||
toolchain: Object.freeze({
|
||||
mavenVersion: "3.9.15",
|
||||
javaVersion: "25.0.3",
|
||||
artifacts: Object.freeze([
|
||||
Object.freeze({
|
||||
coordinate: "org.teavm:teavm-classlib:0.15.0",
|
||||
relativePath:
|
||||
"org/teavm/teavm-classlib/0.15.0/teavm-classlib-0.15.0.jar",
|
||||
sha256:
|
||||
"7d805be4670a892a34c89d9d750a8afd65e2f8201b7b3a10343634ea19d1c410",
|
||||
}),
|
||||
Object.freeze({
|
||||
coordinate: "org.teavm:teavm-jso:0.15.0",
|
||||
relativePath: "org/teavm/teavm-jso/0.15.0/teavm-jso-0.15.0.jar",
|
||||
sha256:
|
||||
"58b337b93924106bf819c45890c7c231a0c740145afe77795f1e0f099f621d71",
|
||||
}),
|
||||
Object.freeze({
|
||||
coordinate: "org.teavm:teavm-maven-plugin:0.15.0",
|
||||
relativePath:
|
||||
"org/teavm/teavm-maven-plugin/0.15.0/teavm-maven-plugin-0.15.0.jar",
|
||||
sha256:
|
||||
"1133e16e45f1fd3cb93129fd96d142e1f5ef3608891d83b2273442d5446a3879",
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze([
|
||||
"LICENSE.txt",
|
||||
"NOTICE.txt",
|
||||
"SHA256SUMS",
|
||||
"engine-metadata.json",
|
||||
"java-regex.mjs",
|
||||
]);
|
||||
const CHECKSUM_FILES = Object.freeze([
|
||||
"LICENSE.txt",
|
||||
"NOTICE.txt",
|
||||
"engine-metadata.json",
|
||||
"java-regex.mjs",
|
||||
]);
|
||||
const BRIDGE_SOURCE_FILES = Object.freeze([
|
||||
"engines/java/pom.xml",
|
||||
"engines/java/README.md",
|
||||
"engines/java/LICENSE-TeaVM.txt",
|
||||
"engines/java/NOTICE-TeaVM.txt",
|
||||
"engines/java/src/main/java/de/addideas/regextools/javaengine/RegexBridge.java",
|
||||
]);
|
||||
const EXPORTED_FUNCTIONS = Object.freeze([
|
||||
"bridgeAbiVersion",
|
||||
"engineIdentity",
|
||||
"execute",
|
||||
"replace",
|
||||
"selfTest",
|
||||
]);
|
||||
|
||||
function sha256(data) {
|
||||
return createHash("sha256").update(data).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`${label} must be a regular, non-symlink file: ${candidate}`,
|
||||
);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function resolveExecutable(name, environment = process.env) {
|
||||
for (const directory of (environment.PATH ?? "").split(path.delimiter)) {
|
||||
if (!directory) continue;
|
||||
const candidate = path.join(directory, name);
|
||||
try {
|
||||
await access(candidate, fsConstants.X_OK);
|
||||
if ((await stat(candidate)).isFile()) return realpath(candidate);
|
||||
} catch {
|
||||
// Continue through PATH.
|
||||
}
|
||||
}
|
||||
throw new Error(`${name} is required on PATH for the TeaVM Java build.`);
|
||||
}
|
||||
|
||||
function commandLabel(command, arguments_) {
|
||||
return [command, ...arguments_]
|
||||
.map((part) =>
|
||||
/^[A-Za-z0-9_./:=,+-]+$/u.test(part) ? part : JSON.stringify(part),
|
||||
)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
async function run(command, arguments_, options = {}) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
child.stdout.on("data", (chunk) => stdout.push(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const output = Buffer.concat(stdout).toString("utf8");
|
||||
const errorOutput = Buffer.concat(stderr).toString("utf8");
|
||||
if (result.code !== 0) {
|
||||
const detail = [output, errorOutput].filter(Boolean).join("\n").trim();
|
||||
throw new Error(
|
||||
`${commandLabel(command, arguments_)} failed with ${
|
||||
result.signal ? `signal ${result.signal}` : `exit code ${result.code}`
|
||||
}${detail ? `:\n${detail}` : "."}`,
|
||||
);
|
||||
}
|
||||
return { stdout: output, stderr: errorOutput };
|
||||
}
|
||||
|
||||
async function verifyToolchain() {
|
||||
const maven = await resolveExecutable("mvn");
|
||||
const java = await resolveExecutable("java");
|
||||
const mavenVersion = await run(maven, ["--version"]);
|
||||
const javaVersion = await run(java, ["-version"]);
|
||||
if (
|
||||
!mavenVersion.stdout.startsWith(
|
||||
`Apache Maven ${JAVA_LOCK.toolchain.mavenVersion} `,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`The Java pack requires Maven ${JAVA_LOCK.toolchain.mavenVersion}.`,
|
||||
);
|
||||
}
|
||||
const javaText = `${javaVersion.stdout}\n${javaVersion.stderr}`;
|
||||
if (
|
||||
!javaText.includes(`openjdk version "${JAVA_LOCK.toolchain.javaVersion}`)
|
||||
) {
|
||||
throw new Error(
|
||||
`The Java pack requires OpenJDK ${JAVA_LOCK.toolchain.javaVersion}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const repository = path.join(homedir(), ".m2", "repository");
|
||||
for (const artifact of JAVA_LOCK.toolchain.artifacts) {
|
||||
const file = await assertRegularFile(
|
||||
path.join(repository, artifact.relativePath),
|
||||
artifact.coordinate,
|
||||
);
|
||||
const actual = await sha256File(file);
|
||||
if (actual !== artifact.sha256) {
|
||||
throw new Error(
|
||||
`${artifact.coordinate} SHA-256 mismatch: expected ${artifact.sha256}, got ${actual}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { maven };
|
||||
}
|
||||
|
||||
function cleanBuildEnvironment() {
|
||||
const environment = { ...process.env };
|
||||
for (const variable of [
|
||||
"CLASSPATH",
|
||||
"JAVA_TOOL_OPTIONS",
|
||||
"MAVEN_ARGS",
|
||||
"MAVEN_OPTS",
|
||||
"_JAVA_OPTIONS",
|
||||
]) {
|
||||
delete environment[variable];
|
||||
}
|
||||
environment.LANG = "C";
|
||||
environment.LC_ALL = "C";
|
||||
environment.SOURCE_DATE_EPOCH = String(JAVA_LOCK.sourceDateEpoch);
|
||||
environment.TZ = "UTC";
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function sourceFiles(root) {
|
||||
return Promise.all(
|
||||
BRIDGE_SOURCE_FILES.map(async (relativePath) => {
|
||||
const file = await assertRegularFile(
|
||||
path.join(root, relativePath),
|
||||
`Java bridge source ${relativePath}`,
|
||||
);
|
||||
const contents = await readFile(file);
|
||||
return {
|
||||
path: relativePath,
|
||||
sha256: sha256(contents),
|
||||
bytes: contents.byteLength,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fileMetadata(pack, name) {
|
||||
const contents = await readFile(path.join(pack, name));
|
||||
return { path: name, sha256: sha256(contents), bytes: contents.byteLength };
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "java",
|
||||
engineName: "TeaVM java.util.regex",
|
||||
engineVersion: JAVA_LOCK.engineVersion,
|
||||
semanticIdentity:
|
||||
"TeaVM 0.15.0 classlib java.util.regex; Apache Harmony-derived; not OpenJDK",
|
||||
status: "production-compatibility-runtime",
|
||||
bridge: {
|
||||
abiVersion: JAVA_LOCK.bridgeAbi,
|
||||
maximumPatternUtf16: JAVA_LOCK.maximumPatternUtf16,
|
||||
maximumSubjectBytes: JAVA_LOCK.maximumSubjectBytes,
|
||||
maximumReplacementUtf16: JAVA_LOCK.maximumReplacementUtf16,
|
||||
maximumMatches: JAVA_LOCK.maximumMatches,
|
||||
maximumCaptureRows: JAVA_LOCK.maximumCaptureRows,
|
||||
maximumCaptureGroups: JAVA_LOCK.maximumCaptureGroups,
|
||||
maximumOutputBytes: JAVA_LOCK.maximumOutputBytes,
|
||||
offsetUnit: "utf16",
|
||||
exports: [...EXPORTED_FUNCTIONS],
|
||||
sourceFiles: await sourceFiles(root),
|
||||
},
|
||||
source: {
|
||||
repository: JAVA_LOCK.source.repository,
|
||||
tag: JAVA_LOCK.source.tag,
|
||||
commitSha1: JAVA_LOCK.source.commit,
|
||||
treeSha1: JAVA_LOCK.source.tree,
|
||||
license: JAVA_LOCK.source.license,
|
||||
upstreamLicenseSha256: JAVA_LOCK.source.licenseSha256,
|
||||
upstreamNoticeSha256: JAVA_LOCK.source.noticeSha256,
|
||||
},
|
||||
toolchain: {
|
||||
mavenVersion: JAVA_LOCK.toolchain.mavenVersion,
|
||||
javaVersion: JAVA_LOCK.toolchain.javaVersion,
|
||||
teaVmArtifacts: JAVA_LOCK.toolchain.artifacts.map(
|
||||
({ coordinate, sha256: digest }) => ({
|
||||
coordinate,
|
||||
sha256: digest,
|
||||
}),
|
||||
),
|
||||
},
|
||||
configuration: {
|
||||
target: "JavaScript ES2015 module",
|
||||
minified: true,
|
||||
optimizationLevel: "ADVANCED",
|
||||
sourceMaps: false,
|
||||
filesystem: false,
|
||||
supportedApplicationFlags: ["g", "i", "d", "m", "s", "u", "x", "U"],
|
||||
unixLines: "d maps to TeaVM Pattern.UNIX_LINES.",
|
||||
unicodeCharacterClassCompatibility:
|
||||
"TeaVM 0.15.0 lacks OpenJDK UNICODE_CHARACTER_CLASS; U implies Unicode case folding only.",
|
||||
replacement:
|
||||
"TeaVM 0.15.0 Matcher appendReplacement/appendTail semantics: only single-digit $n references; ${name} is rejected; not OpenJDK Matcher parity.",
|
||||
},
|
||||
sourceDateEpoch: JAVA_LOCK.sourceDateEpoch,
|
||||
files: await Promise.all(
|
||||
["LICENSE.txt", "NOTICE.txt", "java-regex.mjs"].map((name) =>
|
||||
fileMetadata(pack, name),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function expectedChecksums(pack) {
|
||||
const lines = await Promise.all(
|
||||
CHECKSUM_FILES.map(
|
||||
async (name) => `${await sha256File(path.join(pack, name))} ${name}`,
|
||||
),
|
||||
);
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function equalJson(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
|
||||
function readField(payload, state) {
|
||||
const colon = payload.indexOf(":", state.cursor);
|
||||
if (colon < 0) throw new Error("Malformed Java smoke-test payload.");
|
||||
const length = Number.parseInt(payload.slice(state.cursor, colon), 10);
|
||||
if (!Number.isSafeInteger(length) || length < 0) {
|
||||
throw new Error("Malformed Java smoke-test field length.");
|
||||
}
|
||||
const start = colon + 1;
|
||||
const end = start + length;
|
||||
if (end > payload.length) {
|
||||
throw new Error("Truncated Java smoke-test payload.");
|
||||
}
|
||||
state.cursor = end;
|
||||
return payload.slice(start, end);
|
||||
}
|
||||
|
||||
async function smokeJavaPack(pack) {
|
||||
const moduleFile = path.join(pack, "java-regex.mjs");
|
||||
const imported = await import(
|
||||
`${pathToFileURL(moduleFile).href}?verify=${await sha256File(moduleFile)}`
|
||||
);
|
||||
for (const name of EXPORTED_FUNCTIONS) {
|
||||
if (typeof imported[name] !== "function") {
|
||||
throw new Error(`The Java module is missing export ${name}.`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
imported.bridgeAbiVersion() !== JAVA_LOCK.bridgeAbi ||
|
||||
imported.engineIdentity() !== JAVA_LOCK.engineIdentity ||
|
||||
imported.selfTest() !== 0
|
||||
) {
|
||||
throw new Error("The TeaVM Java bridge identity or self-test failed.");
|
||||
}
|
||||
|
||||
const execution = imported.execute("(?<=a)(b)\\1", "abb", 0, true, 10, 10);
|
||||
const executionState = { cursor: 0 };
|
||||
const executionFields = Array.from({ length: 9 }, () =>
|
||||
readField(execution, executionState),
|
||||
);
|
||||
if (
|
||||
executionState.cursor !== execution.length ||
|
||||
JSON.stringify(executionFields) !==
|
||||
JSON.stringify(["1", "ok", "1", "0", "1", "1", "3", "1", "2"])
|
||||
) {
|
||||
throw new Error("The TeaVM Java matching smoke test failed.");
|
||||
}
|
||||
|
||||
const replacement = imported.replace("(a)", "a a", "<$1>", 0, true, 10, 10);
|
||||
const replacementState = { cursor: 0 };
|
||||
const replacementFields = [];
|
||||
while (replacementState.cursor < replacement.length) {
|
||||
replacementFields.push(readField(replacement, replacementState));
|
||||
}
|
||||
if (
|
||||
replacementFields.at(-2) !== "0" ||
|
||||
replacementFields.at(-1) !== "<a> <a>"
|
||||
) {
|
||||
throw new Error("The TeaVM Java replacement smoke test failed.");
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyJavaPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const pack = await assertRealDirectory(packDirectory, "Java engine pack");
|
||||
const entries = (await readdir(pack, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== PACK_FILES.length ||
|
||||
entries.some(
|
||||
(entry, index) =>
|
||||
entry.name !== PACK_FILES[index] ||
|
||||
!entry.isFile() ||
|
||||
entry.isSymbolicLink(),
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`The staged Java pack must contain only: ${PACK_FILES.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await sha256File(path.join(pack, "LICENSE.txt"))) !==
|
||||
JAVA_LOCK.source.licenseSha256
|
||||
) {
|
||||
throw new Error("The staged TeaVM licence does not match tag 0.15.0.");
|
||||
}
|
||||
const moduleText = await readFile(path.join(pack, "java-regex.mjs"), "utf8");
|
||||
if (
|
||||
moduleText.includes("sourceMappingURL") ||
|
||||
moduleText.includes("/mnt/") ||
|
||||
moduleText.includes("/home/") ||
|
||||
!moduleText.includes("as bridgeAbiVersion") ||
|
||||
!moduleText.includes("as engineIdentity") ||
|
||||
!moduleText.includes("as execute") ||
|
||||
!moduleText.includes("as replace") ||
|
||||
!moduleText.includes("as selfTest")
|
||||
) {
|
||||
throw new Error(
|
||||
"java-regex.mjs leaks a build path, has a source map, or lacks bridge exports.",
|
||||
);
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("Java engine-metadata.json is not valid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const expected = await expectedMetadata(repositoryRoot, pack);
|
||||
if (!equalJson(metadata, expected)) {
|
||||
throw new Error(
|
||||
"The staged Java metadata does not match the pinned build contract.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
|
||||
(await expectedChecksums(pack))
|
||||
) {
|
||||
throw new Error("The staged Java SHA256SUMS file is incorrect.");
|
||||
}
|
||||
await smokeJavaPack(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function replaceDirectory(stage, target, label) {
|
||||
const targetDetails = await lstat(target).catch(() => null);
|
||||
if (
|
||||
targetDetails &&
|
||||
(!targetDetails.isDirectory() || targetDetails.isSymbolicLink())
|
||||
) {
|
||||
throw new Error(`${label} must be a real directory.`);
|
||||
}
|
||||
if (!targetDetails) {
|
||||
await rename(stage, target);
|
||||
return;
|
||||
}
|
||||
const backup = `${target}.replaced-${process.pid}`;
|
||||
await rename(target, backup);
|
||||
try {
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rename(backup, target);
|
||||
throw error;
|
||||
}
|
||||
await rm(backup, { recursive: true });
|
||||
}
|
||||
|
||||
export async function buildJavaPack(root, outputDirectory) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const engine = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "engines", "java"),
|
||||
"Java engine source",
|
||||
);
|
||||
const { maven } = await verifyToolchain();
|
||||
const environment = cleanBuildEnvironment();
|
||||
const buildArguments = [
|
||||
"--batch-mode",
|
||||
"--no-transfer-progress",
|
||||
"--offline",
|
||||
"clean",
|
||||
"package",
|
||||
];
|
||||
await run(maven, buildArguments, { cwd: engine, env: environment });
|
||||
const generated = await assertRegularFile(
|
||||
path.join(engine, "target", "generated", "java-regex.mjs"),
|
||||
"generated TeaVM module",
|
||||
);
|
||||
const firstBuild = await readFile(generated);
|
||||
await run(maven, buildArguments, { cwd: engine, env: environment });
|
||||
const secondBuild = await readFile(generated);
|
||||
if (sha256(firstBuild) !== sha256(secondBuild)) {
|
||||
throw new Error(
|
||||
"Two clean TeaVM builds produced different Java module hashes.",
|
||||
);
|
||||
}
|
||||
|
||||
const output = path.resolve(
|
||||
repositoryRoot,
|
||||
outputDirectory ?? path.join(".engine-build", "java"),
|
||||
);
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
const stage = await mkdtemp(path.join(path.dirname(output), ".java-build-"));
|
||||
try {
|
||||
await copyFile(
|
||||
path.join(engine, "LICENSE-TeaVM.txt"),
|
||||
path.join(stage, "LICENSE.txt"),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
await copyFile(
|
||||
path.join(engine, "NOTICE-TeaVM.txt"),
|
||||
path.join(stage, "NOTICE.txt"),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
await writeFile(path.join(stage, "java-regex.mjs"), secondBuild, {
|
||||
flag: "wx",
|
||||
});
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
{ encoding: "utf8", flag: "wx" },
|
||||
);
|
||||
await writeFile(
|
||||
path.join(stage, "SHA256SUMS"),
|
||||
await expectedChecksums(stage),
|
||||
{ encoding: "utf8", flag: "wx" },
|
||||
);
|
||||
await verifyJavaPack(stage, repositoryRoot);
|
||||
await replaceDirectory(stage, output, "Java build output");
|
||||
return {
|
||||
output,
|
||||
metadata,
|
||||
moduleSha256: sha256(secondBuild),
|
||||
moduleBytes: secondBuild.byteLength,
|
||||
};
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function installJavaPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const source = await assertRealDirectory(packDirectory, "verified Java pack");
|
||||
const metadata = await verifyJavaPack(source, repositoryRoot);
|
||||
const engineRoot = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "public", "engines"),
|
||||
"public engine directory",
|
||||
);
|
||||
const target = path.join(engineRoot, "java");
|
||||
const stage = await mkdtemp(path.join(engineRoot, ".java-install-"));
|
||||
try {
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(
|
||||
path.join(source, name),
|
||||
path.join(stage, name),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
}
|
||||
await verifyJavaPack(stage, repositoryRoot);
|
||||
await replaceDirectory(stage, target, "public/engines/java");
|
||||
return { output: target, metadata };
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : "";
|
||||
if (invokedFile === fileURLToPath(import.meta.url)) {
|
||||
if (process.argv.length > 3) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/java-engine-pack.mjs [.engine-build/java]",
|
||||
);
|
||||
}
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const result = await buildJavaPack(root, process.argv[2]);
|
||||
console.log(
|
||||
`Built deterministic ${result.metadata.semanticIdentity} module: ${result.moduleSha256} (${result.moduleBytes.toLocaleString()} bytes).`,
|
||||
);
|
||||
}
|
||||
@@ -18,7 +18,7 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const zipEpoch = new Date(1980, 0, 1, 0, 0, 0);
|
||||
const gplVersion3TextSha256 =
|
||||
"fb981668c18a279e285fc4d83fba1e836cc84dd4daa73c9697d3cfd2d8aca6e0";
|
||||
const expectedApplicationVersion = "0.2.0";
|
||||
const expectedApplicationVersion = "0.3.0";
|
||||
const requiredRootFiles = [
|
||||
"LICENSE",
|
||||
"README.md",
|
||||
@@ -387,11 +387,25 @@ async function collectReleaseEntries() {
|
||||
"SOURCE.md",
|
||||
"THIRD_PARTY_NOTICES.md",
|
||||
"engines/README.md",
|
||||
"engines/java/LICENSE.txt",
|
||||
"engines/java/NOTICE.txt",
|
||||
"engines/java/SHA256SUMS",
|
||||
"engines/java/engine-metadata.json",
|
||||
"engines/java/java-regex.mjs",
|
||||
"engines/pcre2/LICENSE.txt",
|
||||
"engines/pcre2/SHA256SUMS",
|
||||
"engines/pcre2/engine-metadata.json",
|
||||
"engines/pcre2/pcre2.mjs",
|
||||
"engines/pcre2/pcre2.wasm",
|
||||
"engines/python/LICENSE.cpython.txt",
|
||||
"engines/python/LICENSE.pyodide.txt",
|
||||
"engines/python/SHA256SUMS",
|
||||
"engines/python/engine-metadata.json",
|
||||
"engines/python/pyodide-lock.json",
|
||||
"engines/python/pyodide.asm.mjs",
|
||||
"engines/python/pyodide.asm.wasm",
|
||||
"engines/python/pyodide.mjs",
|
||||
"engines/python/python_stdlib.zip",
|
||||
"LICENSES/Emscripten-MIT.txt",
|
||||
"LICENSES/PCRE2.txt",
|
||||
"LICENSES/build/rolldown-1.1.5/LICENSE",
|
||||
|
||||
572
scripts/python-engine-pack.mjs
Normal file
572
scripts/python-engine-pack.mjs
Normal file
@@ -0,0 +1,572 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
constants as fsConstants,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
export const PYTHON_ENGINE_LOCK = Object.freeze({
|
||||
bridgeAbi: 1,
|
||||
pythonVersion: "3.14.2",
|
||||
pyodideVersion: "314.0.3",
|
||||
maximumPatternUtf16: 65_536,
|
||||
maximumSubjectBytes: 16_777_216,
|
||||
maximumReplacementUtf16: 65_536,
|
||||
maximumOutputBytes: 67_108_864,
|
||||
maximumMatches: 10_000,
|
||||
maximumCaptureRows: 100_000,
|
||||
maximumCaptureGroups: 1_000,
|
||||
maximumNativeExpansionCodePoints: 16_777_216,
|
||||
npm: Object.freeze({
|
||||
package: "pyodide@314.0.3",
|
||||
resolved: "https://registry.npmjs.org/pyodide/-/pyodide-314.0.3.tgz",
|
||||
integrity:
|
||||
"sha512-sK40My6m8tmBUYtYH9au9rXUeh9x0wfahtHdOlGmJxZDsKBGKtP6KznyFB2+u/klbQTdDionR0uaVd176zVQzQ==",
|
||||
packageJsonSha256:
|
||||
"0393eb01a0707cddc251f1a66090998773fe1aa3cfde66eedc96e83e43861845",
|
||||
}),
|
||||
pyodide: Object.freeze({
|
||||
repository: "https://github.com/pyodide/pyodide.git",
|
||||
tag: "314.0.3",
|
||||
commit: "ac57031be7564f864d061cb37c5c152e59f83ad4",
|
||||
license: "MPL-2.0",
|
||||
licenseUrl:
|
||||
"https://raw.githubusercontent.com/pyodide/pyodide/314.0.3/LICENSE",
|
||||
licenseSha256:
|
||||
"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5",
|
||||
}),
|
||||
cpython: Object.freeze({
|
||||
repository: "https://github.com/python/cpython.git",
|
||||
tag: "v3.14.2",
|
||||
tagObject: "a1d0069daf8e85b25a0c3f96abc43182be6d429e",
|
||||
commit: "df793163d5821791d4e7caf88885a2c11a107986",
|
||||
license: "Python-2.0",
|
||||
licenseUrl:
|
||||
"https://raw.githubusercontent.com/python/cpython/v3.14.2/LICENSE",
|
||||
licenseSha256:
|
||||
"b0e25a78cffb43f4d92de8b61ccfa1f1f98ecbc22330b54b5251e7b6ba010231",
|
||||
}),
|
||||
emscriptenVersion: "5.0.3",
|
||||
abiVersion: "2026_0",
|
||||
lockPythonVersion: "3.14.0",
|
||||
assets: Object.freeze({
|
||||
"pyodide-lock.json":
|
||||
"c963d22858f6bcb8f41586a2142f03905ab370c88ea22a86a2736e95fac2a8f3",
|
||||
"pyodide.asm.mjs":
|
||||
"1a9775427ef6e8abaa7db88ece0515422d1886915ae5c9093776410c865dfd8d",
|
||||
"pyodide.asm.wasm":
|
||||
"e7f8fac36f8bf11085309cbc5c829b3ec3057c18bf1d73b05a6741612d63cdbf",
|
||||
"pyodide.mjs":
|
||||
"5cfc46f5dcbaf2a16f26e2363f441873eb424762609cc03db00d6a2ace4d00e5",
|
||||
"python_stdlib.zip":
|
||||
"444c770dfd75a32097fc0a7d5c1413fd3140601f49c3a1f2e9af0376fcd124b4",
|
||||
}),
|
||||
strippedLoaderSha256:
|
||||
"640ac750bf7cf42ee58b524faa09f803732d2237753cd630d6878edda8a107bf",
|
||||
});
|
||||
|
||||
const PACK_FILES = Object.freeze(
|
||||
[
|
||||
"LICENSE.cpython.txt",
|
||||
"LICENSE.pyodide.txt",
|
||||
"SHA256SUMS",
|
||||
"engine-metadata.json",
|
||||
...Object.keys(PYTHON_ENGINE_LOCK.assets),
|
||||
].sort(),
|
||||
);
|
||||
const CHECKSUM_FILES = Object.freeze(
|
||||
PACK_FILES.filter((name) => name !== "SHA256SUMS"),
|
||||
);
|
||||
|
||||
function sha256(data) {
|
||||
return createHash("sha256").update(data).digest("hex");
|
||||
}
|
||||
|
||||
async function sha256File(file) {
|
||||
return sha256(await readFile(file));
|
||||
}
|
||||
|
||||
async function assertRealDirectory(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a real directory: ${candidate}`);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertRegularFile(candidate, label) {
|
||||
const details = await lstat(candidate).catch(() => null);
|
||||
if (!details?.isFile() || details.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`${label} must be a regular, non-symlink file: ${candidate}`,
|
||||
);
|
||||
}
|
||||
return realpath(candidate);
|
||||
}
|
||||
|
||||
async function assertHash(file, expected, label) {
|
||||
const actual = await sha256File(file);
|
||||
if (actual !== expected) {
|
||||
throw new Error(
|
||||
`${label} SHA-256 mismatch: expected ${expected}, got ${actual}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function withoutSourceMapReference(source) {
|
||||
const result = source.replace(
|
||||
/\n\/\/# sourceMappingURL=pyodide\.mjs\.map\s*$/u,
|
||||
"\n",
|
||||
);
|
||||
if (result === source) {
|
||||
throw new Error(
|
||||
"The pinned Pyodide loader no longer has the expected source-map trailer.",
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function verifyPyodideSource(sourceDirectory) {
|
||||
const source = await assertRealDirectory(
|
||||
sourceDirectory,
|
||||
"Pyodide npm package",
|
||||
);
|
||||
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
|
||||
await assertRegularFile(path.join(source, name), `Pyodide ${name}`);
|
||||
await assertHash(path.join(source, name), expected, `Pyodide ${name}`);
|
||||
}
|
||||
const packageFile = path.join(source, "package.json");
|
||||
await assertHash(
|
||||
packageFile,
|
||||
PYTHON_ENGINE_LOCK.npm.packageJsonSha256,
|
||||
"Pyodide package.json",
|
||||
);
|
||||
const packageJson = JSON.parse(await readFile(packageFile, "utf8"));
|
||||
if (
|
||||
packageJson.name !== "pyodide" ||
|
||||
packageJson.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
|
||||
packageJson.license !== PYTHON_ENGINE_LOCK.pyodide.license ||
|
||||
packageJson.repository?.url !== "git+https://github.com/pyodide/pyodide.git"
|
||||
) {
|
||||
throw new Error("The Pyodide npm package identity is not pinned.");
|
||||
}
|
||||
const lock = JSON.parse(
|
||||
await readFile(path.join(source, "pyodide-lock.json"), "utf8"),
|
||||
);
|
||||
if (
|
||||
lock.info?.abi_version !== PYTHON_ENGINE_LOCK.abiVersion ||
|
||||
lock.info?.arch !== "wasm32" ||
|
||||
lock.info?.python !== PYTHON_ENGINE_LOCK.lockPythonVersion ||
|
||||
lock.info?.platform !==
|
||||
`emscripten_${PYTHON_ENGINE_LOCK.emscriptenVersion.replaceAll(".", "_")}`
|
||||
) {
|
||||
throw new Error("The Pyodide lock-file platform identity is not pinned.");
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
async function fetchPinned(url, expectedHash, label) {
|
||||
const response = await fetch(url, {
|
||||
redirect: "follow",
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`${label} download failed with HTTP ${response.status}.`);
|
||||
}
|
||||
const value = Buffer.from(await response.arrayBuffer());
|
||||
const actual = sha256(value);
|
||||
if (actual !== expectedHash) {
|
||||
throw new Error(
|
||||
`${label} SHA-256 mismatch: expected ${expectedHash}, got ${actual}.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function fileRecord(directory, name) {
|
||||
const file = path.join(directory, name);
|
||||
const value = await readFile(file);
|
||||
return {
|
||||
path: name,
|
||||
sha256: sha256(value),
|
||||
bytes: value.byteLength,
|
||||
};
|
||||
}
|
||||
|
||||
async function expectedMetadata(root, pack) {
|
||||
const bridgePath = path.join(root, "engines", "python", "bridge.py");
|
||||
const bridge = await readFile(bridgePath);
|
||||
const files = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
if (name !== "engine-metadata.json") {
|
||||
files.push(await fileRecord(pack, name));
|
||||
}
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
engine: "python",
|
||||
engineVersion: `CPython ${PYTHON_ENGINE_LOCK.pythonVersion} re`,
|
||||
runtime: `Pyodide ${PYTHON_ENGINE_LOCK.pyodideVersion}`,
|
||||
status: "production-runtime",
|
||||
bridge: {
|
||||
abiVersion: PYTHON_ENGINE_LOCK.bridgeAbi,
|
||||
sourceFile: {
|
||||
path: "engines/python/bridge.py",
|
||||
sha256: sha256(bridge),
|
||||
bytes: bridge.byteLength,
|
||||
},
|
||||
userValuesAreFunctionArguments: true,
|
||||
regexModule: "re",
|
||||
supportModules: {
|
||||
serialization: "json",
|
||||
runtimeIdentity: "sys",
|
||||
},
|
||||
maximumPatternUtf16: PYTHON_ENGINE_LOCK.maximumPatternUtf16,
|
||||
maximumSubjectBytes: PYTHON_ENGINE_LOCK.maximumSubjectBytes,
|
||||
maximumReplacementUtf16: PYTHON_ENGINE_LOCK.maximumReplacementUtf16,
|
||||
maximumOutputBytes: PYTHON_ENGINE_LOCK.maximumOutputBytes,
|
||||
maximumMatches: PYTHON_ENGINE_LOCK.maximumMatches,
|
||||
maximumCaptureRows: PYTHON_ENGINE_LOCK.maximumCaptureRows,
|
||||
maximumCaptureGroups: PYTHON_ENGINE_LOCK.maximumCaptureGroups,
|
||||
maximumNativeExpansionCodePoints:
|
||||
PYTHON_ENGINE_LOCK.maximumNativeExpansionCodePoints,
|
||||
},
|
||||
source: {
|
||||
pyodide: {
|
||||
repository: PYTHON_ENGINE_LOCK.pyodide.repository,
|
||||
tag: PYTHON_ENGINE_LOCK.pyodide.tag,
|
||||
commitSha1: PYTHON_ENGINE_LOCK.pyodide.commit,
|
||||
license: PYTHON_ENGINE_LOCK.pyodide.license,
|
||||
npmPackage: PYTHON_ENGINE_LOCK.npm.package,
|
||||
npmResolved: PYTHON_ENGINE_LOCK.npm.resolved,
|
||||
npmIntegrity: PYTHON_ENGINE_LOCK.npm.integrity,
|
||||
},
|
||||
cpython: {
|
||||
repository: PYTHON_ENGINE_LOCK.cpython.repository,
|
||||
tag: PYTHON_ENGINE_LOCK.cpython.tag,
|
||||
tagObjectSha1: PYTHON_ENGINE_LOCK.cpython.tagObject,
|
||||
commitSha1: PYTHON_ENGINE_LOCK.cpython.commit,
|
||||
license: PYTHON_ENGINE_LOCK.cpython.license,
|
||||
},
|
||||
},
|
||||
toolchain: {
|
||||
abiVersion: PYTHON_ENGINE_LOCK.abiVersion,
|
||||
architecture: "wasm32",
|
||||
emscriptenVersion: PYTHON_ENGINE_LOCK.emscriptenVersion,
|
||||
lockPythonVersion: PYTHON_ENGINE_LOCK.lockPythonVersion,
|
||||
executablePythonVersion: PYTHON_ENGINE_LOCK.pythonVersion,
|
||||
threads: false,
|
||||
},
|
||||
packaging: {
|
||||
loaderTransformation:
|
||||
"Removed only the upstream sourceMappingURL trailer; executable JavaScript is unchanged.",
|
||||
upstreamLoaderSha256: PYTHON_ENGINE_LOCK.assets["pyodide.mjs"],
|
||||
packedLoaderSha256: PYTHON_ENGINE_LOCK.strippedLoaderSha256,
|
||||
externalPackagesIncluded: false,
|
||||
lockPythonVersionNote:
|
||||
"The upstream lock's 3.14.0 field identifies its CPython 3.14 ABI baseline; the executable runtime self-identifies as 3.14.2.",
|
||||
},
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
function equalJson(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
|
||||
async function expectedChecksums(pack) {
|
||||
const lines = [];
|
||||
for (const name of CHECKSUM_FILES) {
|
||||
lines.push(`${await sha256File(path.join(pack, name))} ${name}`);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
async function smokePythonPack(pack) {
|
||||
await WebAssembly.compile(
|
||||
await readFile(path.join(pack, "pyodide.asm.wasm")),
|
||||
);
|
||||
const moduleUrl = pathToFileURL(path.join(pack, "pyodide.mjs"));
|
||||
moduleUrl.searchParams.set("regex-tools-verification", String(Date.now()));
|
||||
const imported = await import(moduleUrl.href);
|
||||
if (
|
||||
imported.version !== PYTHON_ENGINE_LOCK.pyodideVersion ||
|
||||
typeof imported.loadPyodide !== "function"
|
||||
) {
|
||||
throw new Error("The staged Pyodide module identity is invalid.");
|
||||
}
|
||||
const runtime = await imported.loadPyodide({
|
||||
indexURL: `${pack}${path.sep}`,
|
||||
stdout: () => undefined,
|
||||
stderr: () => undefined,
|
||||
});
|
||||
const result = JSON.parse(
|
||||
runtime.runPython(`
|
||||
import json, re, sys
|
||||
expression = re.compile(r"(?P<emoji>😀)|(?P<word>[a-z]+)", re.ASCII)
|
||||
matches = [
|
||||
[match.span(0), match.lastgroup]
|
||||
for match in expression.finditer("😀 abc")
|
||||
]
|
||||
replacement_match = re.compile(r"(?P<word>[a-z]+)").search("abc")
|
||||
json.dumps({
|
||||
"python": ".".join(str(part) for part in sys.version_info[:3]),
|
||||
"implementation": sys.implementation.name,
|
||||
"matches": matches,
|
||||
"replacement": replacement_match.expand(r"[\\g<word>]"),
|
||||
})
|
||||
`),
|
||||
);
|
||||
if (
|
||||
result.python !== PYTHON_ENGINE_LOCK.pythonVersion ||
|
||||
result.implementation !== "cpython" ||
|
||||
JSON.stringify(result.matches) !==
|
||||
JSON.stringify([
|
||||
[[0, 1], "emoji"],
|
||||
[[2, 5], "word"],
|
||||
]) ||
|
||||
result.replacement !== "[abc]"
|
||||
) {
|
||||
throw new Error("The staged CPython re runtime smoke test failed.");
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyPythonPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const pack = await assertRealDirectory(packDirectory, "Python engine pack");
|
||||
const entries = (await readdir(pack, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== PACK_FILES.length ||
|
||||
entries.some(
|
||||
(entry, index) =>
|
||||
entry.name !== PACK_FILES[index] ||
|
||||
!entry.isFile() ||
|
||||
entry.isSymbolicLink(),
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`The staged Python pack must contain only: ${PACK_FILES.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
|
||||
await assertHash(
|
||||
path.join(pack, "LICENSE.pyodide.txt"),
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
|
||||
"staged Pyodide licence",
|
||||
);
|
||||
await assertHash(
|
||||
path.join(pack, "LICENSE.cpython.txt"),
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
|
||||
"staged CPython licence",
|
||||
);
|
||||
await assertHash(
|
||||
path.join(pack, "pyodide.mjs"),
|
||||
PYTHON_ENGINE_LOCK.strippedLoaderSha256,
|
||||
"staged Pyodide loader",
|
||||
);
|
||||
for (const [name, expected] of Object.entries(PYTHON_ENGINE_LOCK.assets)) {
|
||||
if (name !== "pyodide.mjs") {
|
||||
await assertHash(path.join(pack, name), expected, `staged ${name}`);
|
||||
}
|
||||
}
|
||||
const loader = await readFile(path.join(pack, "pyodide.mjs"), "utf8");
|
||||
if (
|
||||
loader.includes("sourceMappingURL") ||
|
||||
loader.includes("/mnt/DATA/") ||
|
||||
loader.includes("/home/zemion/")
|
||||
) {
|
||||
throw new Error(
|
||||
"The staged Pyodide loader leaks a source-map or local build path.",
|
||||
);
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(
|
||||
await readFile(path.join(pack, "engine-metadata.json"), "utf8"),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error("engine-metadata.json is not valid JSON.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const expected = await expectedMetadata(repositoryRoot, pack);
|
||||
if (!equalJson(metadata, expected)) {
|
||||
throw new Error(
|
||||
"The staged Python metadata does not match the pinned pack contract.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
(await readFile(path.join(pack, "SHA256SUMS"), "utf8")) !==
|
||||
(await expectedChecksums(pack))
|
||||
) {
|
||||
throw new Error("The staged Python SHA256SUMS file is incorrect.");
|
||||
}
|
||||
await smokePythonPack(pack);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function replaceDirectory(stage, target, label) {
|
||||
const targetDetails = await lstat(target).catch(() => null);
|
||||
if (
|
||||
targetDetails &&
|
||||
(!targetDetails.isDirectory() || targetDetails.isSymbolicLink())
|
||||
) {
|
||||
throw new Error(`${label} must be a real directory.`);
|
||||
}
|
||||
if (!targetDetails) {
|
||||
await rename(stage, target);
|
||||
return;
|
||||
}
|
||||
const backup = `${target}.replaced-${process.pid}-${Date.now()}`;
|
||||
await rename(target, backup);
|
||||
try {
|
||||
await rename(stage, target);
|
||||
} catch (error) {
|
||||
await rename(backup, target);
|
||||
throw error;
|
||||
}
|
||||
await rm(backup, { recursive: true });
|
||||
}
|
||||
|
||||
export async function installPythonPack(packDirectory, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const sourcePack = await assertRealDirectory(
|
||||
packDirectory,
|
||||
"verified Python engine pack",
|
||||
);
|
||||
const metadata = await verifyPythonPack(sourcePack, repositoryRoot);
|
||||
const engineRoot = await assertRealDirectory(
|
||||
path.join(repositoryRoot, "public", "engines"),
|
||||
"public engine directory",
|
||||
);
|
||||
const target = path.join(engineRoot, "python");
|
||||
const stage = await mkdtemp(path.join(engineRoot, ".python-install-"));
|
||||
try {
|
||||
for (const name of PACK_FILES) {
|
||||
await copyFile(
|
||||
path.join(sourcePack, name),
|
||||
path.join(stage, name),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
}
|
||||
await verifyPythonPack(stage, repositoryRoot);
|
||||
await replaceDirectory(stage, target, "public/engines/python");
|
||||
return { output: target, metadata };
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildPythonPack(options, root) {
|
||||
const repositoryRoot = await assertRealDirectory(
|
||||
root,
|
||||
"Regex Tools repository",
|
||||
);
|
||||
const source = await verifyPyodideSource(options.sourceDirectory);
|
||||
const output = path.resolve(options.outputDirectory);
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
const stage = await mkdtemp(
|
||||
path.join(path.dirname(output), ".python-pack-build-"),
|
||||
);
|
||||
try {
|
||||
const [pyodideLicense, cpythonLicense] = await Promise.all([
|
||||
fetchPinned(
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseUrl,
|
||||
PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
|
||||
"Pyodide licence",
|
||||
),
|
||||
fetchPinned(
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseUrl,
|
||||
PYTHON_ENGINE_LOCK.cpython.licenseSha256,
|
||||
"CPython licence",
|
||||
),
|
||||
]);
|
||||
await writeFile(path.join(stage, "LICENSE.pyodide.txt"), pyodideLicense, {
|
||||
flag: "wx",
|
||||
});
|
||||
await writeFile(path.join(stage, "LICENSE.cpython.txt"), cpythonLicense, {
|
||||
flag: "wx",
|
||||
});
|
||||
for (const name of Object.keys(PYTHON_ENGINE_LOCK.assets)) {
|
||||
if (name === "pyodide.mjs") {
|
||||
const upstream = await readFile(path.join(source, name), "utf8");
|
||||
const packed = withoutSourceMapReference(upstream);
|
||||
if (sha256(packed) !== PYTHON_ENGINE_LOCK.strippedLoaderSha256) {
|
||||
throw new Error("The deterministic Pyodide loader output changed.");
|
||||
}
|
||||
await writeFile(path.join(stage, name), packed, { flag: "wx" });
|
||||
} else {
|
||||
await copyFile(
|
||||
path.join(source, name),
|
||||
path.join(stage, name),
|
||||
fsConstants.COPYFILE_EXCL,
|
||||
);
|
||||
}
|
||||
}
|
||||
const metadata = await expectedMetadata(repositoryRoot, stage);
|
||||
await writeFile(
|
||||
path.join(stage, "engine-metadata.json"),
|
||||
`${JSON.stringify(metadata, null, 2)}\n`,
|
||||
{ flag: "wx" },
|
||||
);
|
||||
await writeFile(
|
||||
path.join(stage, "SHA256SUMS"),
|
||||
await expectedChecksums(stage),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
await verifyPythonPack(stage, repositoryRoot);
|
||||
await replaceDirectory(stage, output, "Python pack output");
|
||||
return { output, metadata };
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const invokedFile = process.argv[1]
|
||||
? pathToFileURL(path.resolve(process.argv[1])).href
|
||||
: "";
|
||||
if (import.meta.url === invokedFile) {
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
if (arguments_.length > 2) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/python-engine-pack.mjs [node_modules/pyodide] [.engine-build/python]",
|
||||
);
|
||||
}
|
||||
const result = await buildPythonPack(
|
||||
{
|
||||
sourceDirectory: path.resolve(
|
||||
root,
|
||||
arguments_[0] ?? "node_modules/pyodide",
|
||||
),
|
||||
outputDirectory: path.resolve(
|
||||
root,
|
||||
arguments_[1] ?? ".engine-build/python",
|
||||
),
|
||||
},
|
||||
root,
|
||||
);
|
||||
console.log(
|
||||
`Built and verified ${result.metadata.runtime} / ${result.metadata.engineVersion} at ${result.output}.`,
|
||||
);
|
||||
}
|
||||
@@ -26,6 +26,14 @@ const testCatalogue = {
|
||||
theme: { mode: "system", brand: "add·ideas" },
|
||||
apps: [{ manifest: "./toolbox-app.json", enabled: true }],
|
||||
};
|
||||
const productionSecurityHeaders = {
|
||||
"Content-Security-Policy":
|
||||
"default-src 'self'; base-uri 'self'; object-src 'none'; frame-ancestors 'none'; form-action 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; media-src 'self' blob:; font-src 'self' data:; connect-src 'self'; worker-src 'self' blob:; manifest-src 'self'",
|
||||
"Cross-Origin-Embedder-Policy": "require-corp",
|
||||
"Cross-Origin-Opener-Policy": "same-origin",
|
||||
"Cross-Origin-Resource-Policy": "same-origin",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
};
|
||||
|
||||
function safeFile(requestPath) {
|
||||
const decoded = decodeURIComponent(requestPath);
|
||||
@@ -50,6 +58,7 @@ const server = createServer(async (request, response) => {
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Cache-Control": "no-cache",
|
||||
...productionSecurityHeaders,
|
||||
});
|
||||
response.end(JSON.stringify(testCatalogue));
|
||||
return;
|
||||
@@ -66,7 +75,7 @@ const server = createServer(async (request, response) => {
|
||||
"Content-Type":
|
||||
mediaTypes.get(path.extname(file)) ?? "application/octet-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Cross-Origin-Resource-Policy": "same-origin",
|
||||
...productionSecurityHeaders,
|
||||
});
|
||||
response.end(content);
|
||||
} catch {
|
||||
|
||||
@@ -1,22 +1,28 @@
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { verifyJavaPack } from "./java-engine-pack.mjs";
|
||||
import { verifyPcre2Pack } from "./pcre2-engine-pack.mjs";
|
||||
import { verifyPythonPack } from "./python-engine-pack.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const arguments_ = process.argv.slice(2);
|
||||
|
||||
if (arguments_.length > 0) {
|
||||
if (arguments_.length !== 2 || arguments_[0] !== "--pcre2-pack") {
|
||||
const verifiers = new Map([
|
||||
["--java-pack", verifyJavaPack],
|
||||
["--pcre2-pack", verifyPcre2Pack],
|
||||
["--python-pack", verifyPythonPack],
|
||||
]);
|
||||
const verifier = verifiers.get(arguments_[0]);
|
||||
if (arguments_.length !== 2 || !verifier) {
|
||||
throw new Error(
|
||||
"Usage: node scripts/verify-engine-assets.mjs [--pcre2-pack PATH]",
|
||||
"Usage: node scripts/verify-engine-assets.mjs [--java-pack|--pcre2-pack|--python-pack PATH]",
|
||||
);
|
||||
}
|
||||
const pack = path.resolve(root, arguments_[1]);
|
||||
const metadata = await verifyPcre2Pack(pack, root);
|
||||
console.log(
|
||||
`Verified staged PCRE2 ${metadata.engineVersion} pack and WebAssembly bridge smoke test.`,
|
||||
);
|
||||
const metadata = await verifier(pack, root);
|
||||
console.log(`Verified staged ${metadata.engineVersion} engine pack.`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -31,21 +37,34 @@ const entries = (await readdir(engineDirectory, { withFileTypes: true })).sort(
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
if (
|
||||
entries.length !== 2 ||
|
||||
entries.length !== 4 ||
|
||||
!entries[0]?.isFile() ||
|
||||
entries[0].name !== "README.md" ||
|
||||
!entries[1]?.isDirectory() ||
|
||||
entries[1].name !== "pcre2"
|
||||
entries[1].name !== "java" ||
|
||||
!entries[2]?.isDirectory() ||
|
||||
entries[2].name !== "pcre2" ||
|
||||
!entries[3]?.isDirectory() ||
|
||||
entries[3].name !== "python"
|
||||
) {
|
||||
throw new Error(
|
||||
"The production build must contain only README.md and the declared PCRE2 pack.",
|
||||
"The production build must contain only README.md and the declared Java, PCRE2 and Python packs.",
|
||||
);
|
||||
}
|
||||
|
||||
const notice = await readFile(path.join(engineDirectory, "README.md"), "utf8");
|
||||
if (!notice.includes("native `RegExp`") || !notice.includes("PCRE2 10.47")) {
|
||||
if (
|
||||
!notice.includes("native `RegExp`") ||
|
||||
!notice.includes("PCRE2 10.47") ||
|
||||
!notice.includes("Pyodide 314.0.3") ||
|
||||
!notice.includes("TeaVM 0.15.0")
|
||||
) {
|
||||
throw new Error("The engine asset notice does not describe this release.");
|
||||
}
|
||||
await verifyPcre2Pack(path.join(engineDirectory, "pcre2"), root);
|
||||
await verifyPythonPack(path.join(engineDirectory, "python"), root);
|
||||
await verifyJavaPack(path.join(engineDirectory, "java"), root);
|
||||
|
||||
console.log("Verified native ECMAScript and bundled PCRE2 engine assets.");
|
||||
console.log(
|
||||
"Verified native ECMAScript and bundled PCRE2, CPython/Pyodide and TeaVM Java engine assets.",
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user