feat: add multi-engine regex flavour support

scripts/python-engine-pack.mjs

@@ -14,6 +14,11 @@ import {
 } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
+import {
+  createPythonSourceManifest,
+  PYTHON_LEGAL_RESOURCES,
+  transformPythonLegalResource,
+} from "./python-engine-provenance.mjs";
 
 export const PYTHON_ENGINE_LOCK = Object.freeze({
   bridgeAbi: 1,
@@ -77,9 +82,9 @@ export const PYTHON_ENGINE_LOCK = Object.freeze({
 
 const PACK_FILES = Object.freeze(
   [
-    "LICENSE.cpython.txt",
-    "LICENSE.pyodide.txt",
+    ...PYTHON_LEGAL_RESOURCES.map(({ output }) => output),
     "SHA256SUMS",
+    "SOURCE-MANIFEST.json",
     "engine-metadata.json",
     ...Object.keys(PYTHON_ENGINE_LOCK.assets),
   ].sort(),
@@ -203,9 +208,18 @@ async function fileRecord(directory, name) {
   };
 }
 
+async function expectedSourceManifest(pack) {
+  const legalFiles = [];
+  for (const { output } of PYTHON_LEGAL_RESOURCES) {
+    legalFiles.push(await fileRecord(pack, output));
+  }
+  return createPythonSourceManifest(legalFiles);
+}
+
 async function expectedMetadata(root, pack) {
   const bridgePath = path.join(root, "engines", "python", "bridge.py");
   const bridge = await readFile(bridgePath);
+  const sourceManifest = await fileRecord(pack, "SOURCE-MANIFEST.json");
   const files = [];
   for (const name of CHECKSUM_FILES) {
     if (name !== "engine-metadata.json") {
@@ -229,6 +243,7 @@ async function expectedMetadata(root, pack) {
       regexModule: "re",
       supportModules: {
         serialization: "json",
+        replacementOutputEncoding: "canonical RFC 4648 base64",
         runtimeIdentity: "sys",
       },
       maximumPatternUtf16: PYTHON_ENGINE_LOCK.maximumPatternUtf16,
@@ -242,6 +257,7 @@ async function expectedMetadata(root, pack) {
         PYTHON_ENGINE_LOCK.maximumNativeExpansionCodePoints,
     },
     source: {
+      manifest: sourceManifest,
       pyodide: {
         repository: PYTHON_ENGINE_LOCK.pyodide.repository,
         tag: PYTHON_ENGINE_LOCK.pyodide.tag,
@@ -273,6 +289,8 @@ async function expectedMetadata(root, pack) {
       upstreamLoaderSha256: PYTHON_ENGINE_LOCK.assets["pyodide.mjs"],
       packedLoaderSha256: PYTHON_ENGINE_LOCK.strippedLoaderSha256,
       externalPackagesIncluded: false,
+      legalClosure:
+        "SOURCE-MANIFEST.json inventories every loader and native base-runtime component, exact preferred-form input and patch, plus independently anchored legal text.",
       lockPythonVersionNote:
         "The upstream lock's 3.14.0 field identifies its CPython 3.14 ABI baseline; the executable runtime self-identifies as 3.14.2.",
     },
@@ -312,7 +330,7 @@ async function smokePythonPack(pack) {
   });
   const result = JSON.parse(
     runtime.runPython(`
-import json, re, sys
+import _zstd, decimal, json, pyexpat, re, sqlite3, sys, zlib
 expression = re.compile(r"(?P<emoji>😀)|(?P<word>[a-z]+)", re.ASCII)
 matches = [
     [match.span(0), match.lastgroup]
@@ -322,6 +340,11 @@ replacement_match = re.compile(r"(?P<word>[a-z]+)").search("abc")
 json.dumps({
     "python": ".".join(str(part) for part in sys.version_info[:3]),
     "implementation": sys.implementation.name,
+    "libmpdec": decimal.__libmpdec_version__,
+    "expat": pyexpat.EXPAT_VERSION,
+    "sqlite": sqlite3.sqlite_version,
+    "zlib": zlib.ZLIB_RUNTIME_VERSION,
+    "zstd": _zstd.zstd_version,
     "matches": matches,
     "replacement": replacement_match.expand(r"[\\g<word>]"),
 })
@@ -330,6 +353,11 @@ json.dumps({
   if (
     result.python !== PYTHON_ENGINE_LOCK.pythonVersion ||
     result.implementation !== "cpython" ||
+    result.libmpdec !== "2.5.1" ||
+    result.expat !== "expat_2.7.3" ||
+    result.sqlite !== "3.39.0" ||
+    result.zlib !== "1.3.1" ||
+    result.zstd !== "1.5.7" ||
     JSON.stringify(result.matches) !==
       JSON.stringify([
         [[0, 1], "emoji"],
@@ -365,16 +393,13 @@ export async function verifyPythonPack(packDirectory, root) {
     );
   }
 
-  await assertHash(
-    path.join(pack, "LICENSE.pyodide.txt"),
-    PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
-    "staged Pyodide licence",
-  );
-  await assertHash(
-    path.join(pack, "LICENSE.cpython.txt"),
-    PYTHON_ENGINE_LOCK.cpython.licenseSha256,
-    "staged CPython licence",
-  );
+  for (const resource of PYTHON_LEGAL_RESOURCES) {
+    await assertHash(
+      path.join(pack, resource.output),
+      resource.outputSha256,
+      `staged ${resource.component} legal material`,
+    );
+  }
   await assertHash(
     path.join(pack, "pyodide.mjs"),
     PYTHON_ENGINE_LOCK.strippedLoaderSha256,
@@ -406,6 +431,22 @@ export async function verifyPythonPack(packDirectory, root) {
       cause: error,
     });
   }
+  let sourceManifest;
+  try {
+    sourceManifest = JSON.parse(
+      await readFile(path.join(pack, "SOURCE-MANIFEST.json"), "utf8"),
+    );
+  } catch (error) {
+    throw new Error("SOURCE-MANIFEST.json is not valid JSON.", {
+      cause: error,
+    });
+  }
+  const expectedManifest = await expectedSourceManifest(pack);
+  if (!equalJson(sourceManifest, expectedManifest)) {
+    throw new Error(
+      "The staged Python source manifest does not match the pinned preferred-form and legal inventory.",
+    );
+  }
   const expected = await expectedMetadata(repositoryRoot, pack);
   if (!equalJson(metadata, expected)) {
     throw new Error(
@@ -489,24 +530,19 @@ export async function buildPythonPack(options, root) {
     path.join(path.dirname(output), ".python-pack-build-"),
   );
   try {
-    const [pyodideLicense, cpythonLicense] = await Promise.all([
-      fetchPinned(
-        PYTHON_ENGINE_LOCK.pyodide.licenseUrl,
-        PYTHON_ENGINE_LOCK.pyodide.licenseSha256,
-        "Pyodide licence",
-      ),
-      fetchPinned(
-        PYTHON_ENGINE_LOCK.cpython.licenseUrl,
-        PYTHON_ENGINE_LOCK.cpython.licenseSha256,
-        "CPython licence",
-      ),
-    ]);
-    await writeFile(path.join(stage, "LICENSE.pyodide.txt"), pyodideLicense, {
-      flag: "wx",
-    });
-    await writeFile(path.join(stage, "LICENSE.cpython.txt"), cpythonLicense, {
-      flag: "wx",
-    });
+    const legalFiles = await Promise.all(
+      PYTHON_LEGAL_RESOURCES.map(async (resource) => {
+        const sourceValue = await fetchPinned(
+          resource.url,
+          resource.sourceSha256,
+          `${resource.component} legal source`,
+        );
+        return [resource, transformPythonLegalResource(resource, sourceValue)];
+      }),
+    );
+    for (const [resource, value] of legalFiles) {
+      await writeFile(path.join(stage, resource.output), value, { flag: "wx" });
+    }
     for (const name of Object.keys(PYTHON_ENGINE_LOCK.assets)) {
       if (name === "pyodide.mjs") {
         const upstream = await readFile(path.join(source, name), "utf8");
@@ -523,6 +559,11 @@ export async function buildPythonPack(options, root) {
         );
       }
     }
+    await writeFile(
+      path.join(stage, "SOURCE-MANIFEST.json"),
+      `${JSON.stringify(await expectedSourceManifest(stage), null, 2)}\n`,
+      { flag: "wx" },
+    );
     const metadata = await expectedMetadata(repositoryRoot, stage);
     await writeFile(
       path.join(stage, "engine-metadata.json"),