Publish complete signed module catalogs

tools/release/govoplan_release/catalog_entry_synthesis.py

@@ -10,6 +10,8 @@ import os
 from pathlib import Path
 import subprocess
 import sys
+import tarfile
+import tempfile
 import tomllib
 from types import ModuleType
 from typing import Iterator
@@ -30,13 +32,14 @@ def synthesize_repository_catalog_entries(
     version: str,
     workspace: Path,
     repository_base: str,
+    source_ref: str | None = None,
 ) -> tuple[dict[str, object], ...]:
     """Build install entries from tagged, preflighted local source metadata.
 
-    The caller owns source-tag and worktree provenance checks. This function
-    accepts no hand-maintained module catalog registry: distribution metadata
-    identifies the runtime entry point and the runtime ``ModuleManifest`` is
-    the authoritative dependency/interface/frontend description.
+    Distribution metadata identifies the runtime entry point and the runtime
+    ``ModuleManifest`` is the authoritative dependency/interface/frontend
+    description. When ``source_ref`` is supplied, metadata is read from that
+    immutable Git tree rather than from the current checkout.
     """
 
     if os.getenv(_INSPECTION_CHILD) == "1":
@@ -45,6 +48,7 @@ def synthesize_repository_catalog_entries(
             version=version,
             workspace=workspace,
             repository_base=repository_base,
+            source_ref=source_ref,
         )
     command = (
         sys.executable,
@@ -59,6 +63,8 @@ def synthesize_repository_catalog_entries(
         "--repository-base",
         repository_base,
     )
+    if source_ref:
+        command = (*command, "--source-ref", source_ref)
     environment = os.environ.copy()
     environment[_INSPECTION_CHILD] = "1"
     release_root = str(Path(__file__).resolve().parents[1])
@@ -98,44 +104,46 @@ def synthesize_repository_catalog_entries_in_process(
     version: str,
     workspace: Path,
     repository_base: str,
+    source_ref: str | None = None,
 ) -> tuple[dict[str, object], ...]:
     specs = {item.name: item for item in load_repository_specs(include_website=False)}
     spec = specs.get(repo)
     if spec is None:
         raise ValueError(f"Cannot synthesize {repo}: repository is not registered.")
-    root = resolve_repo_path(spec, workspace)
-    project = read_project_metadata(root / "pyproject.toml")
-    package = required_text(project, "name", source=f"{repo}/pyproject.toml")
-    project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v")
-    expected_version = version.removeprefix("v")
-    if project_version != expected_version:
-        raise ValueError(
-            f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
-        )
-    description = optional_text(project.get("description"))
-    entry_points = module_entry_points(project, repo=repo)
-    entries: list[dict[str, object]] = []
-    for declared_module_id, target in sorted(entry_points.items()):
-        manifest = load_manifest(root=root, target=target, repo=repo)
-        if manifest.id != declared_module_id:
+    checkout_root = resolve_repo_path(spec, workspace)
+    with materialized_source_tree(checkout_root, source_ref=source_ref) as root:
+        project = read_project_metadata(root / "pyproject.toml")
+        package = required_text(project, "name", source=f"{repo}/pyproject.toml")
+        project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v")
+        expected_version = version.removeprefix("v")
+        if project_version != expected_version:
             raise ValueError(
-                f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}."
+                f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
             )
-        if manifest.version.removeprefix("v") != expected_version:
-            raise ValueError(
-                f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}."
+        description = optional_text(project.get("description"))
+        entry_points = module_entry_points(project, repo=repo)
+        entries: list[dict[str, object]] = []
+        for declared_module_id, target in sorted(entry_points.items()):
+            manifest = load_manifest(root=root, target=target, repo=repo)
+            if manifest.id != declared_module_id:
+                raise ValueError(
+                    f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}."
+                )
+            if manifest.version.removeprefix("v") != expected_version:
+                raise ValueError(
+                    f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}."
+                )
+            entry = manifest_catalog_entry(
+                manifest=manifest,
+                repo=repo,
+                package=package,
+                version=expected_version,
+                description=description,
+                root=root,
+                repository_base=repository_base.rstrip("/"),
             )
-        entry = manifest_catalog_entry(
-            manifest=manifest,
-            repo=repo,
-            package=package,
-            version=expected_version,
-            description=description,
-            root=root,
-            repository_base=repository_base.rstrip("/"),
-        )
-        entries.append(entry)
-    return tuple(entries)
+            entries.append(entry)
+        return tuple(entries)
 
 
 def manifest_catalog_entry(
@@ -156,8 +164,7 @@ def manifest_catalog_entry(
         "action": "install",
         "python_package": package,
         "python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}",
-        "license_features": [f"module.{manifest.id}"],
-        "tags": ["official"],
+        "tags": ["official", "open-source"],
     }
     if description:
         entry["description"] = description
@@ -218,6 +225,51 @@ def manifest_catalog_entry(
     return entry
 
 
+@contextmanager
+def materialized_source_tree(root: Path, *, source_ref: str | None) -> Iterator[Path]:
+    if not source_ref:
+        yield root
+        return
+    if not (root / ".git").exists():
+        raise ValueError(f"Cannot inspect {source_ref!r}: {root} is not a Git checkout.")
+    with tempfile.TemporaryDirectory(prefix="govoplan-catalog-source-") as value:
+        temporary = Path(value)
+        archive_path = temporary / "source.tar"
+        source_root = temporary / "source"
+        source_root.mkdir()
+        result = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(root),
+                "archive",
+                "--format=tar",
+                f"--output={archive_path}",
+                source_ref,
+            ],
+            check=False,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode != 0:
+            detail = result.stderr.strip() or "Git archive failed"
+            raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: {detail}")
+        if archive_path.stat().st_size > 256 * 1024 * 1024:
+            raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive exceeds 256 MiB.")
+        with tarfile.open(archive_path, mode="r:") as archive:
+            members = archive.getmembers()
+            if len(members) > 50_000:
+                raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive has too many entries.")
+            for member in members:
+                path = Path(member.name)
+                if path.is_absolute() or ".." in path.parts or member.issym() or member.islnk() or member.isdev():
+                    raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive contains an unsafe entry.")
+            archive.extractall(source_root, members=members, filter="data")
+        yield source_root
+
+
 def validate_initial_entry_closure(
     *,
     catalog_modules: list[object],
@@ -367,6 +419,7 @@ def main() -> int:
     parser.add_argument("--version", required=True)
     parser.add_argument("--workspace", type=Path, required=True)
     parser.add_argument("--repository-base", required=True)
+    parser.add_argument("--source-ref")
     args = parser.parse_args()
     try:
         entries = synthesize_repository_catalog_entries(
@@ -374,6 +427,7 @@ def main() -> int:
             version=args.version,
             workspace=args.workspace.resolve(),
             repository_base=args.repository_base,
+            source_ref=args.source_ref,
         )
     except ValueError as exc:
         print(str(exc), file=sys.stderr)