diff --git a/docs/operations/FULL_REGISTRY_CANDIDATES.md b/docs/operations/FULL_REGISTRY_CANDIDATES.md new file mode 100644 index 0000000..85c50c9 --- /dev/null +++ b/docs/operations/FULL_REGISTRY_CANDIDATES.md @@ -0,0 +1,138 @@ +# Full registry candidates / Vollständige Registry-Kandidaten + +## Operator workflow (EN) + +The canonical `release-catalog.py full-registry` command takes `--package-set`, +`--package-lock`, `--wheelhouse`, `--webui-packages`, `--output-dir`, and a +configured `--catalog-signing-key`. Generate the package set with +`generate-release-package-set.py --profile full` and download its exact artifacts +with `resolve-package-artifacts.py`; do not substitute locally rebuilt wheels. +The candidate compares the package set with the exact developer meta-package +pins, checks archive bytes and package metadata against the lock, and synthesizes +every entry from its immutable tagged manifest. Native package publication and +its CI authority remain trusted: this verifies the published artifact identity, +not independent reproducible-build equivalence to source. + +Pass `--selected-repository` once for each newly released repository, including +Core when it changes. These selected units must have clean, version-aligned +named branches whose HEAD equals the annotated local and remote release tag. +Other full-profile packages retain their exact older annotated tags; a later +workflow-only commit on `main` does not relabel those package contents or force +a version bump. Every source fetch/push endpoint must match the registered +origin, and all entries bind their source commit and annotated tag object. +Git replacement objects, caller Git configuration, and executable-path +redirection cannot substitute another tagged manifest tree. + +The fixed existing website catalog and keyring are authenticated before signing. +An older catalog without a signed keyring hash can be migrated only through this +complete rebuild, only when its signature verifies and its entire keyring +exactly matches the configured known signers. No old entries or artifact hashes +are reused. The new catalog signs the exact unchanged website keyring hash; +key rotation remains a separate reviewed operation. Selective candidates still +reject unpinned base keyrings. New candidate directories are private and +exclusive: a retry must choose a new directory, not overwrite a reviewed one. + +Run these commands on the trusted host, with an operator-private source workspace +and artifact directory. `RELEASE_PYTHON` must select its private environment and +`RELEASE_NPM` an absolute npm executable with a trusted sibling Node 22 binary. +In Flatpak, execute host commands through `flatpak-spawn --host`; sandbox and +host UID mappings are not interchangeable. Do not weaken trust gates or change +system-wide permissions. + +```sh +# These paths identify previously prepared private operator resources. +RELEASE_WORKSPACE=/path/to/private/workspace +RELEASE_PYTHON="$RELEASE_WORKSPACE/govoplan/.host-venv/bin/python" +RELEASE_NPM=/path/to/private/node22/bin/npm +ARTIFACT_ROOT=/path/to/private/artifacts +RELEASE_CANDIDATE=/path/to/private/new-candidate +RELEASE_VERSION=0.1.45 +RELEASE_TOOLS="$RELEASE_WORKSPACE/govoplan/tools/release" + +umask 077 +"$RELEASE_PYTHON" "$RELEASE_TOOLS/generate-release-package-set.py" \ + --version "$RELEASE_VERSION" --profile full \ + --workspace "$RELEASE_WORKSPACE" --output "$ARTIFACT_ROOT/packages.json" +PATH="$(dirname "$RELEASE_NPM"):/usr/bin:/bin" \ + "$RELEASE_PYTHON" "$RELEASE_TOOLS/resolve-package-artifacts.py" \ + --package-set "$ARTIFACT_ROOT/packages.json" \ + --wheelhouse "$ARTIFACT_ROOT/wheels" \ + --webui-packages "$ARTIFACT_ROOT/webui" \ + --lock-output "$ARTIFACT_ROOT/artifacts.lock.json" \ + --python "$RELEASE_PYTHON" --npm "$RELEASE_NPM" + +# Repeat --selected-repository for EVERY newly released unit, not just Core. +"$RELEASE_PYTHON" "$RELEASE_TOOLS/release-catalog.py" full-registry \ + --workspace-root "$RELEASE_WORKSPACE" \ + --package-set "$ARTIFACT_ROOT/packages.json" \ + --package-lock "$ARTIFACT_ROOT/artifacts.lock.json" \ + --wheelhouse "$ARTIFACT_ROOT/wheels" --webui-packages "$ARTIFACT_ROOT/webui" \ + --output-dir "$RELEASE_CANDIDATE" --selected-repository govoplan-core \ + --catalog-signing-key known-key=/path/to/private/known-key.pem --json + +"$RELEASE_PYTHON" "$RELEASE_TOOLS/release-catalog.py" publish-candidate \ + --workspace-root "$RELEASE_WORKSPACE" --candidate-dir "$RELEASE_CANDIDATE" \ + --channel stable --npm "$RELEASE_NPM" --build-web \ + --commit --tag --push --tag-name "catalog-v$RELEASE_VERSION" --json +``` + +The last command is a strict non-mutating preview because `--apply` is absent. +Review its output, then repeat it with `--apply` to publish. The website's locked +build dependencies must already be installed before `--build-web`. The publisher +sanitizes the build and Git environments and pushes the verified immutable +website commit/tag. Source tag publication and registry package availability +must be complete before candidate generation. + +Source tags, registry packages, and a signed module catalog do not imply that a +new runtime distribution exists. While runtime images are held, leave the Meta +Gitea runtime Release held too: a normal source-only Release can replace Gitea's +`releases/latest` discovery result despite having no deployment assets. The +deployer still requires an explicit signed manifest, digest, and trusted +keyring; it does not deploy a tag or module catalog directly. + +## Betriebsablauf (DE) + +`release-catalog.py full-registry` übernimmt den vollständigen Paketbestand, +die Registry-Sperrdatei, das Wheel-Verzeichnis, die WebUI-Archive und den +konfigurierten Signaturschlüssel. Zuerst mit +`generate-release-package-set.py --profile full` die exakten Meta-Paketversionen +ermitteln und mit `resolve-package-artifacts.py` die veröffentlichten Artefakte +herunterladen. Lokal neu gebaute Wheels sind kein Ersatz. Der Kandidat prüft +Paketidentitäten, Dateigrößen und Hashes und erzeugt alle Einträge aus den +unveränderlichen getaggten Manifesten. Die Registry und ihre veröffentlichende +CI bleiben eine Vertrauensgrundlage; dies ist kein unabhängiger Nachweis eines +reproduzierbaren Builds aus dem Quellcode. + +Jedes neu veröffentlichte Repository wird mit `--selected-repository` +angegeben. Nur diese Auswahl muss mit dem sauberen, versionsgleichen HEAD eines +benannten Branches und dem annotierten lokalen und entfernten Tag übereinstimmen. +Unveränderte Pakete behalten ihren ursprünglichen Tag, auch wenn auf `main` +bereits eine spätere Workflow-Korrektur liegt. Alle Quelladressen müssen dem +registrierten Ursprung entsprechen; Commit und annotiertes Tag-Objekt werden +für jeden Eintrag gebunden. Git-Ersetzungsobjekte oder fremde Git-Konfiguration +können dabei keinen anderen Manifestbaum unterschieben. + +Vor dem Signieren werden der bestehende Website-Katalog und sein Schlüsselbund +geprüft. Ein alter Katalog ohne signierten Schlüsselbund-Hash darf ausschließlich +durch diesen vollständigen Neuaufbau migriert werden: Seine Signatur muss gültig +sein und der gesamte Schlüsselbund exakt den konfigurierten bekannten Signierern +entsprechen. Alte Einträge oder Artefakt-Hashes werden nicht übernommen. Der neue +Katalog bindet den unveränderten Schlüsselbund-Hash; ein Schlüsselwechsel bleibt +ein eigener geprüfter Vorgang. Selektive Kandidaten verlangen weiterhin einen +bereits gebundenen Schlüsselbund. Kandidaten werden nur in neuen privaten +Verzeichnissen erzeugt und niemals überschrieben. + +Das obige Befehlsbeispiel wird auf dem vertrauenswürdigen Host ausgeführt. Dafür +die private Python-Umgebung und einen absoluten `--npm`-Pfad zu Node 22 verwenden; +unter Flatpak die Host-Werkzeuge über `flatpak-spawn --host` aufrufen. Die +gesperrten Website-Build-Abhängigkeiten vorher installieren. Keine +Vertrauensprüfung umgehen und keine globalen Rechte ändern. Die Artefaktordner +müssen privat und bei der Auflösung leer sein. Vor der Kandidatenerzeugung +müssen Quell-Tags und Registry-Pakete vollständig veröffentlicht sein. + +Die Veröffentlichung zunächst mit `publish-candidate --commit --tag --push +--build-web` ohne `--apply` prüfen und erst nach Prüfung mit `--apply` ausführen. +Solange Laufzeit-Images zurückgestellt sind, bleibt auch das Meta-Gitea-Runtime- +Release zurückgestellt: Ein reines Quellcode-Release könnte sonst als neuestes +Release erscheinen. Eine Installation benötigt weiterhin ein signiertes +Laufzeitmanifest, dessen Digest und einen explizit vertrauenswürdigen Schlüsselbund. diff --git a/docs/operations/PACKAGE_REGISTRY_RELEASES.md b/docs/operations/PACKAGE_REGISTRY_RELEASES.md index ad1e809..6d4597d 100644 --- a/docs/operations/PACKAGE_REGISTRY_RELEASES.md +++ b/docs/operations/PACKAGE_REGISTRY_RELEASES.md @@ -189,8 +189,13 @@ than invoking `pip`, `npm`, or Git on the target host. ## Public module directory -`tools/release/publish-release-catalog.sh` resolves the selected package set and -registry lock before it creates a catalog. Catalog entries are synthesized from +For an operator-reviewed full publication, use +`tools/release/release-catalog.py full-registry` followed by the same tool's +`publish-candidate` command. Resolve the package set and registry lock first; +the older direct-write shell wrapper is not the strict candidate publication +path. See [Full registry candidates / Vollständige Registry-Kandidaten](FULL_REGISTRY_CANDIDATES.md) +for the private host runtime, exact artifact checks, and legacy keyring transition. +Catalog entries are synthesized from the exact tagged module manifests, never from a hand-maintained module list or the current workspace. Each entry binds its Python wheel and optional WebUI tarball to the registry URL, filename, size, SHA-256, package identity, source diff --git a/docs/operations/RELEASE_CONSOLE.md b/docs/operations/RELEASE_CONSOLE.md index ca1e9de..61bd4b8 100644 --- a/docs/operations/RELEASE_CONSOLE.md +++ b/docs/operations/RELEASE_CONSOLE.md @@ -517,6 +517,15 @@ tree and requires byte-for-byte equality with those validated objects. Tags and remote branch updates then reference that exact commit SHA rather than the mutable worktree `HEAD`. +For a full registry-backed release, first build a fresh private candidate using +`release-catalog.py full-registry`. Pass `--selected-repository` for newly +released HEAD-bound units, not every unchanged package in the full profile. +The command independently checks all full-profile registry bytes and annotated +tag provenance, then feeds this same strict `publish-candidate` transaction. +It does not create Gitea runtime Releases or dispatch image builds. See +[Full registry candidates / Vollständige Registry-Kandidaten](FULL_REGISTRY_CANDIDATES.md) +for the complete EN/DE workflow and the narrowly scoped legacy keyring transition. + Published channels are expected below the public catalog base URL: - `https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json` diff --git a/tests/test_full_registry_catalog.py b/tests/test_full_registry_catalog.py new file mode 100644 index 0000000..0c8858a --- /dev/null +++ b/tests/test_full_registry_catalog.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +import base64 +from contextlib import ExitStack +import csv +from copy import deepcopy +import hashlib +import io +import json +import os +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import quote +import zipfile + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools/release")) + +from govoplan_release import full_catalog # noqa: E402 +from govoplan_release.artifact_identity import selected_artifact_identity_issues # noqa: E402 +from govoplan_release.catalog import canonical_hash # noqa: E402 +from govoplan_release.model import RepositorySpec # noqa: E402 +from govoplan_release.registry_reference import registry_entry_source # noqa: E402 +from govoplan_release.selective_catalog import ( # noqa: E402 + load_authenticated_catalog_base, public_key_base64, signature, +) +from govoplan_release.source_provenance import ( # noqa: E402 + SourceTagProvenanceIssue, catalog_source_selection, + registered_source_origin_issues, +) +from govoplan_release.version_alignment import candidate_catalog_version_issues # noqa: E402 + + +class FullRegistryCatalogTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.web = self.root / "addideas-govoplan-website" + self.wheels = self.root / "wheels" + self.npm = self.root / "npm" + self.wheels.mkdir(mode=0o700) + self.npm.mkdir(mode=0o700) + self.key = Ed25519PrivateKey.generate() + self.keypath = self.root / "key.pem" + self.keypath.write_bytes(self.key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + )) + self.keypath.chmod(0o600) + self.keyring = { + "keyring_version": "1", "keys": [{ + "key_id": "known-key", "public_key": public_key_base64(self.key), + "status": "active", + }], + } + self.base = { + "catalog_version": "1", "channel": "stable", "sequence": 1, + "core_release": {"version": "1.0.0"}, "modules": [], + "release": {}, + } + self.write_base() + self.package_set = { + "schema_version": "1", "release_version": "1.2.3", "profile": "full", + "registries": { + "python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple", + "npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/", + }, + "python": [self.package("govoplan-core"), self.package("govoplan-demo")], + "webui": [self.package("govoplan-core", webui=True)], + } + self.seal(self.package_set, "package_set_sha256") + self.lock = { + "schema_version": "1", "release_version": "1.2.3", "profile": "full", + "registries": self.package_set["registries"], + "package_set_sha256": self.package_set["package_set_sha256"], + "python": [], "webui": [], + } + for row in self.package_set["python"]: + path = self.wheel(row["name"]) + url = full_catalog._tool("resolve-package-artifacts")["_python_artifact_url"]( + self.package_set["registries"]["python"], package=row, filename=path.name, + ) + self.lock["python"].append(self.artifact(row, path, url)) + npm_package = self.package_set["webui"][0] + npm_path = self.npm / "govoplan-core-webui-1.2.3.tgz" + self.tarball(npm_path, "@govoplan/core-webui", "1.2.3") + url = self.package_set["registries"]["npm"] + quote(npm_package["name"], safe="") + "/-/1.2.3/core-webui-1.2.3.tgz" + row = self.artifact(npm_package, npm_path, url) + row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(npm_path.read_bytes()).digest()).decode() + self.lock["webui"].append(row) + self.seal(self.lock, "lock_sha256") + self.set_path = self.root / "package-set.json" + self.lock_path = self.root / "package-lock.json" + self.write_inputs() + self.output = self.root / "candidate" + + @staticmethod + def package(repo: str, *, webui: bool = False) -> dict: + row = { + "name": "@govoplan/core-webui" if webui else repo, "version": "1.2.3", + "repository": repo, "tag": "v1.2.3", + "commit": ("a" if repo == "govoplan-core" else "b") * 40, + } + if not webui: + row["extras"] = ["server"] if repo == "govoplan-core" else [] + return row + + @staticmethod + def artifact(package: dict, path: Path, url: str) -> dict: + encoded = path.read_bytes() + return {**package, "filename": path.name, "url": url, + "sha256": hashlib.sha256(encoded).hexdigest(), "size": len(encoded)} + + @staticmethod + def seal(payload: dict, field: str) -> None: + payload.pop(field, None) + payload[field] = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + def write_inputs(self) -> None: + self.set_path.write_text(json.dumps(self.package_set)) + self.lock_path.write_text(json.dumps(self.lock)) + + def write_base(self) -> None: + self.base.pop("signatures", None) + self.base["signatures"] = [signature(self.base, key_id="known-key", private_key=self.key)] + folder = self.web / "public/catalogs/v1" + (folder / "channels").mkdir(parents=True, exist_ok=True) + (folder / "channels/stable.json").write_text(json.dumps(self.base)) + (folder / "keyring.json").write_text(json.dumps(self.keyring)) + + def wheel(self, package: str) -> Path: + stem = package.replace("-", "_") + info = f"{stem}-1.2.3.dist-info" + files = { + f"{stem}/__init__.py": b"VALUE = 1\n", + f"{info}/METADATA": f"Metadata-Version: 2.1\nName: {package}\nVersion: 1.2.3\n".encode(), + f"{info}/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + } + record = io.StringIO() + writer = csv.writer(record, lineterminator="\n") + for name, value in files.items(): + writer.writerow((name, "", len(value))) + writer.writerow((f"{info}/RECORD", "", "")) + files[f"{info}/RECORD"] = record.getvalue().encode() + path = self.wheels / f"{stem}-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + for name, value in files.items(): + archive.writestr(name, value) + return path + + @staticmethod + def tarball(path: Path, name: str, version: str, *, duplicate: bool = False) -> None: + encoded = json.dumps({"name": name, "version": version}).encode() + with tarfile.open(path, "w:gz") as archive: + for _ in range(2 if duplicate else 1): + member = tarfile.TarInfo("package/package.json") + member.size = len(encoded) + archive.addfile(member, io.BytesIO(encoded)) + + def build(self, *, provenance_errors=(), origin_errors=()) -> dict: + registry_generator = full_catalog._tool("generate-release-catalog") + tools = {name: dict(full_catalog._tool(name)) for name in ( + "generate-release-catalog", "generate-release-package-set", "resolve-package-artifacts", + )} + tools["generate-release-package-set"]["generate_package_set"] = lambda **kwargs: self.package_set + self.provenance = { + row["repository"]: {"commit_sha": row["commit"], "tag_object_sha": str(index + 1) * 40} + for index, row in enumerate(self.package_set["python"]) + } + entry = { + "module_id": "demo", "name": "Demo", "version": "1.2.3", + "python_package": "govoplan-demo", + "python_ref": "govoplan-demo @ git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-demo.git@v1.2.3", + } + with ExitStack() as stack: + stack.enter_context(patch.object(full_catalog, "_tool", side_effect=lambda name: tools[name])) + stack.enter_context(patch.dict(registry_generator["_catalog_payload"].__globals__, { + "synthesize_repository_catalog_entries": lambda **kwargs: (dict(entry),), + })) + stack.enter_context(patch.object(full_catalog, "enforce_selected_version_alignment")) + stack.enter_context(patch.object(full_catalog, "registered_source_origin_issues", return_value=origin_errors)) + self.provenance_check = stack.enter_context(patch.object(full_catalog, "source_tag_provenance_issues", return_value=provenance_errors)) + stack.enter_context(patch.object(full_catalog, "selected_source_provenance", return_value=self.provenance)) + return full_catalog.build_full_registry_candidate( + package_set_path=self.set_path, package_lock_path=self.lock_path, + wheelhouse=self.wheels, webui_packages=self.npm, output_dir=self.output, + selected_repositories=("govoplan-core",), + signing_keys=(f"known-key={self.keypath}",), workspace_root=self.root, + ) + + def candidate(self) -> dict: + return json.loads((self.output / "channels/stable.json").read_text()) + + def test_full_candidate_uses_registry_bytes_and_preserves_unchanged_tag_provenance(self) -> None: + result = self.build() + candidate = self.candidate() + self.assertEqual("ready", result["status"]) + self.assertEqual(2, result["package_count"]) + self.assertEqual(1, result["selected_count"]) + self.assertEqual(self.keyring, json.loads((self.output / "keyring.json").read_text())) + self.assertEqual(canonical_hash(self.keyring), candidate["release"]["keyring_sha256"]) + self.assertEqual(2, len(candidate["release"]["artifacts"])) + self.assertIn("/pypi/files/", candidate["core_release"]["python_ref"]) + self.assertEqual((), candidate_catalog_version_issues(candidate)) + self.assertEqual((), selected_artifact_identity_issues(candidate)) + sources = catalog_source_selection(candidate) + self.assertEqual((), sources.issues) + self.assertEqual({"govoplan-core": "1.2.3"}, sources.selected_versions) + self.assertEqual({"govoplan-core": "1.2.3", "govoplan-demo": "1.2.3"}, sources.all_versions) + self.assertEqual("b" * 40, sources.selected_commits["govoplan-demo"]) + self.assertEqual("2" * 40, sources.selected_tag_objects["govoplan-demo"]) + self.assertEqual(2, self.provenance_check.call_count) + for call in self.provenance_check.call_args_list: + self.assertEqual({"govoplan-core"}, call.kwargs["require_head_repos"]) + for path in [self.output, *self.output.rglob("*")]: + self.assertEqual(0o700 if path.is_dir() else 0o600, path.stat().st_mode & 0o777) + + def test_legacy_base_is_authenticated_but_remains_rejected_by_selective(self) -> None: + self.build() + with self.assertRaisesRegex(ValueError, "does not pin"): + load_authenticated_catalog_base( + base_catalog=None, base_keyring=None, web_root=self.web, + channel="stable", public_base_url="https://unused.example", + signer_public_keys={"known-key": public_key_base64(self.key)}, + ) + + def test_injected_key_or_mismatched_pinned_keyring_is_rejected(self) -> None: + for mutation in ("extra-key", "bad-hash"): + with self.subTest(mutation=mutation): + if mutation == "extra-key": + self.keyring["keys"].append({"key_id": "injected", "status": "active", "public_key": public_key_base64(Ed25519PrivateKey.generate())}) + else: + self.keyring["keys"] = self.keyring["keys"][:1] + self.base["release"]["keyring_sha256"] = "f" * 64 + self.write_base() + with self.assertRaises(ValueError): + self.build() + self.assertFalse(self.output.exists()) + + def test_tampered_base_signature_is_rejected(self) -> None: + path = self.web / "public/catalogs/v1/channels/stable.json" + payload = json.loads(path.read_text()) + payload["sequence"] = 999 + path.write_text(json.dumps(payload)) + with self.assertRaisesRegex(ValueError, "signature verification"): + self.build() + + def test_wrong_registry_bytes_and_reused_candidate_fail_closed(self) -> None: + self.build() + original = (self.output / "channels/stable.json").read_bytes() + with self.assertRaisesRegex(ValueError, "must not already exist"): + self.build() + self.assertEqual(original, (self.output / "channels/stable.json").read_bytes()) + self.output = self.root / "candidate-2" + wheel = self.wheels / self.lock["python"][0]["filename"] + with wheel.open("ab") as stream: + stream.write(b"tampered") + with self.assertRaisesRegex(ValueError, "bytes differ"): + self.build() + self.assertFalse(self.output.exists()) + + def test_webui_identity_and_url_changes_are_rejected(self) -> None: + row = self.lock["webui"][0] + original = row["url"] + for url in ( + original + "?alternate=true", original.replace("/-/1.2.3/", "/-/9.9.9/"), + original.replace("/npm/", "/npm/../other/"), + original.replace("/npm/", "/npm/%2e%2e/other/"), + original.replace("/npm/", "/npm/%252e%252e/other/"), + ): + with self.subTest(url=url): + row["url"] = url + self.seal(self.lock, "lock_sha256") + self.write_inputs() + with self.assertRaisesRegex(ValueError, "URL differs"): + self.build() + + def test_duplicate_missing_and_symlinked_artifacts_are_rejected(self) -> None: + row = self.lock["webui"][0] + self.lock["webui"].append(dict(row)) + with self.assertRaisesRegex(ValueError, "duplicate/missing"): + full_catalog.verify_registry_artifacts(package_set=self.package_set, lock=self.lock, wheelhouse=self.wheels, webui_packages=self.npm) + self.lock["webui"].pop() + path = self.npm / row["filename"] + moved = self.root / "moved.tgz" + path.rename(moved) + path.symlink_to(moved) + with self.assertRaises(OSError): + full_catalog.verify_registry_artifacts(package_set=self.package_set, lock=self.lock, wheelhouse=self.wheels, webui_packages=self.npm) + + def test_archive_metadata_is_bounded_and_not_ambiguous(self) -> None: + path = self.npm / "duplicate.tgz" + self.tarball(path, "@govoplan/core-webui", "1.2.3", duplicate=True) + with self.assertRaisesRegex(ValueError, "duplicate"): + full_catalog._inspect_npm_metadata(path, name="@govoplan/core-webui", version="1.2.3") + + def test_origin_or_tag_provenance_failure_prevents_output(self) -> None: + issue = SourceTagProvenanceIssue("govoplan-core", "v1.2.3", "wrong immutable identity") + for kwargs in ({"origin_errors": (issue,)}, {"provenance_errors": (issue,)}): + with self.subTest(kwargs=kwargs), self.assertRaisesRegex(ValueError, "gate failed"): + self.build(**kwargs) + self.assertFalse(self.output.exists()) + + def test_registered_source_origin_requires_exact_fetch_and_push_targets(self) -> None: + repo = self.root / "govoplan-core" + repo.mkdir() + spec = RepositorySpec("govoplan-core", "system", "kernel", "git@example.test:trusted/core.git", "govoplan-core") + with patch("govoplan_release.source_provenance.load_repository_specs", return_value=(spec,)): + for targets in ((spec.remote, spec.remote), ("git@evil.test:core.git", spec.remote), (spec.remote, "git@evil.test:core.git")): + with self.subTest(targets=targets), patch("govoplan_release.source_provenance.git_text", side_effect=targets): + issues = registered_source_origin_issues(repo_versions={"govoplan-core": "1.2.3"}, workspace=self.root, remote="origin") + self.assertEqual(targets != (spec.remote, spec.remote), bool(issues)) + + def test_registry_metadata_cannot_cross_wire_webui_or_archive_identity(self) -> None: + self.build() + candidate = self.candidate() + entry = candidate["core_release"] + entry["webui_package"] = "@govoplan/files-webui" + with self.assertRaisesRegex(ValueError, "another source repository"): + registry_entry_source(entry) + candidate = self.candidate() + candidate["release"]["artifacts"][0]["archive_sha256"] = "f" * 64 + self.assertIn("matching inspected wheel", " ".join(selected_artifact_identity_issues(candidate))) + + def test_registry_version_and_selected_source_identity_must_agree(self) -> None: + self.build() + for field in ("commit_sha", "tag_object_sha"): + candidate = self.candidate() + candidate["release"]["selected_units"][0][field] = "f" * 40 + self.assertIn("differs", " ".join(issue.message for issue in catalog_source_selection(candidate).issues)) + candidate = self.candidate() + candidate["modules"][0]["artifact_integrity"]["python"]["git_ref"] = "v9.9.9" + self.assertTrue(candidate_catalog_version_issues(candidate)) + + def test_repeated_module_projections_must_bind_identical_python_and_webui_bytes(self) -> None: + self.build() + for kind in ("python", "webui"): + for reversed_order in (False, True): + with self.subTest(kind=kind, reversed_order=reversed_order): + candidate = self.candidate() + entry = deepcopy(candidate["core_release"]) + entry["module_id"] = "another-core-projection" + artifact = entry["artifact_integrity"][kind] + artifact["sha256"] = "f" * 64 + if kind == "python": + entry["python_ref"] = artifact["ref"] = entry["python_ref"].split("#sha256=", 1)[0] + "#sha256=" + "f" * 64 + if reversed_order: + original = candidate["core_release"] + candidate["core_release"] = entry + entry = original + candidate["modules"].append(entry) + self.assertIn(f"conflicting {kind}", " ".join(selected_artifact_identity_issues(candidate))) + self.assertTrue(candidate_catalog_version_issues(candidate)) + candidate = self.candidate() + repeated = deepcopy(candidate["modules"][0]) + repeated["module_id"] = "second-demo-projection" + candidate["modules"].append(repeated) + self.assertEqual((), selected_artifact_identity_issues(candidate)) + self.assertEqual((), candidate_catalog_version_issues(candidate)) + + def test_metadata_inspection_uses_the_opened_archive_not_a_replaced_path(self) -> None: + path = self.npm / "original.tgz" + replacement = self.npm / "replacement.tgz" + saved = self.npm / "saved.tgz" + self.tarball(path, "@govoplan/incorrect-webui", "1.2.3") + self.tarball(replacement, "@govoplan/core-webui", "1.2.3") + real_open = tarfile.open + + def replace_path(*args, **kwargs): + self.assertIn("fileobj", kwargs) + path.rename(saved) + replacement.rename(path) + return real_open(*args, **kwargs) + + with patch("govoplan_release.full_catalog.tarfile.open", side_effect=replace_path): + with self.assertRaisesRegex(ValueError, "metadata differs"): + full_catalog._inspect_npm_metadata(path, name="@govoplan/core-webui", version="1.2.3") + + def test_registry_url_provenance_rejects_package_version_and_traversal_mismatch(self) -> None: + self.build() + for old, new in ( + ("/govoplan-core/1.2.3/", "/govoplan-files/1.2.3/"), + ("/govoplan-core/1.2.3/", "/govoplan-core/9.9.9/"), + ("/pypi/files/", "/pypi/files/%2e%2e/"), + ("/pypi/files/", "/pypi/files/%252e%252e/"), + ): + with self.subTest(new=new): + entry = self.candidate()["core_release"] + artifact = entry["artifact_integrity"]["python"] + artifact["url"] = artifact["url"].replace(old, new) + artifact["ref"] = entry["python_ref"] = entry["python_ref"].replace(old, new) + with self.assertRaises(ValueError): + registry_entry_source(entry) + + def test_package_set_must_match_fixed_meta_pins_not_just_its_own_hash(self) -> None: + payload = deepcopy(self.package_set) + payload["python"][1]["version"] = "9.9.9" + self.seal(payload, "package_set_sha256") + self.set_path.write_text(json.dumps(payload)) + with self.assertRaisesRegex(ValueError, "exact Meta full pins"): + self.build() + self.assertFalse(self.output.exists()) + + def test_package_set_git_reads_ignore_caller_redirection(self) -> None: + tool = full_catalog._tool("generate-release-package-set") + with patch.dict(os.environ, {"GIT_DIR": "/outside", "GIT_CONFIG_GLOBAL": "/outside/config", "PATH": "/outside/bin"}): + with patch("subprocess.check_output", return_value="a" * 40 + "\n") as execute: + self.assertEqual("a" * 40, tool["_git"](self.root, "rev-parse", "HEAD")) + self.assertEqual("/usr/bin/git", execute.call_args.args[0][0]) + self.assertNotIn("GIT_DIR", execute.call_args.kwargs["env"]) + self.assertEqual(os.devnull, execute.call_args.kwargs["env"]["GIT_CONFIG_GLOBAL"]) + self.assertEqual("/usr/bin:/bin", execute.call_args.kwargs["env"]["PATH"]) + + def test_unchanged_real_annotated_ancestor_is_valid_but_selecting_it_requires_head(self) -> None: + from tests.test_release_source_provenance import git, git_text, make_repo + from govoplan_release.source_provenance import source_tag_provenance_issues + + workspace = self.root / "source-workspace" + workspace.mkdir() + repo, _remote = make_repo(workspace, self.root, "govoplan-access", "1.2.3") + git(repo, "tag", "-a", "v1.2.3", "-m", "reviewed immutable package") + git(repo, "push", "origin", "refs/tags/v1.2.3") + tagged_commit = git_text(repo, "rev-parse", "HEAD") + (repo / "workflow-only.txt").write_text("post-tag workflow repair\n") + git(repo, "add", "workflow-only.txt") + git(repo, "commit", "-m", "repair workflow without replacing package") + git(repo, "push", "origin", "main") + common = { + "repo_versions": {"govoplan-access": "1.2.3"}, "workspace": workspace, + "expected_commits": {"govoplan-access": tagged_commit}, + "expected_tag_objects": {"govoplan-access": git_text(repo, "rev-parse", "refs/tags/v1.2.3")}, + } + self.assertEqual((), source_tag_provenance_issues(**common)) + issues = source_tag_provenance_issues(**common, require_head_repos=("govoplan-access",)) + self.assertIn("not selected HEAD", " ".join(issue.message for issue in issues)) + + def test_tagged_manifest_synthesis_ignores_local_git_replacement_objects(self) -> None: + from tests.test_release_source_provenance import git, git_text, make_repo + from govoplan_release.catalog_entry_synthesis import materialized_source_tree + + workspace = self.root / "materialization-workspace" + workspace.mkdir() + repo, _remote = make_repo(workspace, self.root, "govoplan-access", "1.2.3") + original = (repo / "pyproject.toml").read_text() + tagged_commit = git_text(repo, "rev-parse", "HEAD") + git(repo, "tag", "-a", "v1.2.3", "-m", "reviewed immutable package") + (repo / "pyproject.toml").write_text(original.replace("1.2.3", "9.9.9")) + git(repo, "add", "pyproject.toml") + git(repo, "commit", "-m", "unreviewed replacement tree") + git(repo, "replace", tagged_commit, git_text(repo, "rev-parse", "HEAD")) + with patch.dict(os.environ, {"GIT_DIR": str(self.root / "outside"), "PATH": "/outside/bin"}): + with materialized_source_tree(repo, source_ref="v1.2.3") as source: + self.assertEqual(original, (source / "pyproject.toml").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/generate-release-package-set.py b/tools/release/generate-release-package-set.py index 3d8db15..ce675c2 100644 --- a/tools/release/generate-release-package-set.py +++ b/tools/release/generate-release-package-set.py @@ -9,6 +9,7 @@ import json from pathlib import Path import re import subprocess +import sys import tomllib @@ -216,10 +217,17 @@ def _extras(value: str | None) -> list[str]: def _git(repository: Path, *arguments: str) -> str: + release_root = str(Path(__file__).resolve().parent) + if release_root not in sys.path: + sys.path.insert(0, release_root) + from govoplan_release.git_state import sanitized_git_environment, scoped_git_command + return subprocess.check_output( - ["git", "-C", str(repository), *arguments], + scoped_git_command(repository, "-C", str(repository), *arguments), text=True, stderr=subprocess.DEVNULL, + env=sanitized_git_environment(), + timeout=30, ).strip() diff --git a/tools/release/govoplan_release/artifact_identity.py b/tools/release/govoplan_release/artifact_identity.py index 6fdd30a..b0b0cc0 100644 --- a/tools/release/govoplan_release/artifact_identity.py +++ b/tools/release/govoplan_release/artifact_identity.py @@ -14,6 +14,8 @@ import re import stat import zipfile +from .registry_reference import registry_artifact_conflicts, registry_entry_source + WHEEL_PAYLOAD_ALGORITHM = "govoplan-wheel-declared-payload-v1" INSTALLED_PAYLOAD_ALGORITHM = "govoplan-installed-record-payload-v1" @@ -185,10 +187,23 @@ def selected_artifact_identity_issues(payload: object) -> tuple[str, ...]: modules = payload.get("modules") if isinstance(modules, list): entries.extend(modules) + conflicts = registry_artifact_conflicts(entries) + if conflicts: + return conflicts package_by_repo: dict[str, tuple[str, str]] = {} + registry_artifacts: dict[str, dict[str, object]] = {} for entry in entries: if not isinstance(entry, dict): continue + try: + registry_source = registry_entry_source(entry) + except ValueError as exc: + return (str(exc),) + if registry_source is not None: + package = str(entry["python_package"]) + package_by_repo[registry_source.repository] = (package, registry_source.version) + registry_artifacts[package] = entry["artifact_integrity"]["python"] + continue python_ref = entry.get("python_ref") match = _PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None package_name = entry.get("python_package") @@ -219,6 +234,14 @@ def selected_artifact_identity_issues(payload: object) -> tuple[str, ...]: issues: list[str] = [] if malformed_artifacts: issues.append("release.artifacts contains malformed or duplicate identities") + for package, registry_artifact in registry_artifacts.items(): + artifact = artifacts_by_package.get(package) + if artifact is None or ( + artifact.get("archive_sha256") != registry_artifact.get("sha256") + or artifact.get("archive_size") != registry_artifact.get("size") + or artifact.get("package_version") != registry_artifact["registry_identity"].rsplit("@", 1)[-1] + ): + issues.append(f"registry artifact {package} has no matching inspected wheel byte identity") seen_repos: set[str] = set() for unit in selected_units: if not isinstance(unit, dict): diff --git a/tools/release/govoplan_release/catalog_entry_synthesis.py b/tools/release/govoplan_release/catalog_entry_synthesis.py index 93f3ce1..a7478ea 100644 --- a/tools/release/govoplan_release/catalog_entry_synthesis.py +++ b/tools/release/govoplan_release/catalog_entry_synthesis.py @@ -19,6 +19,7 @@ from typing import Iterator from govoplan_core.core.modules import ModuleManifest from govoplan_core.core.versioning import version_satisfies_range +from .git_state import sanitized_git_environment, scoped_git_command from .workspace import load_repository_specs, resolve_repo_path @@ -255,20 +256,19 @@ def materialized_source_tree(root: Path, *, source_ref: str | None) -> Iterator[ source_root = temporary / "source" source_root.mkdir() result = subprocess.run( - [ - "git", - "-C", - str(root), + scoped_git_command( + root, "-C", str(root), "archive", "--format=tar", f"--output={archive_path}", source_ref, - ], + ), check=False, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, timeout=30, + env=sanitized_git_environment(), ) if result.returncode != 0: detail = result.stderr.strip() or "Git archive failed" diff --git a/tools/release/govoplan_release/full_catalog.py b/tools/release/govoplan_release/full_catalog.py new file mode 100644 index 0000000..de0abc9 --- /dev/null +++ b/tools/release/govoplan_release/full_catalog.py @@ -0,0 +1,389 @@ +"""Build a private full-profile candidate from exact verified registry bytes.""" + +from __future__ import annotations + +import base64 +from datetime import UTC, datetime, timedelta +from functools import lru_cache +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import runpy +import stat +import tarfile +from typing import Any +from urllib.parse import quote + +from .artifact_identity import inspect_python_wheel, selected_artifact_identity_issues +from .candidate_artifact import ( + ensure_private_candidate_root, harden_private_candidate_tree, + validate_release_channel, +) +from .catalog import canonical_hash +from .module_directory import write_module_directory +from .selective_catalog import ( + authenticate_base_catalog_signatures, authenticate_base_keyring, + configured_signer_public_keys, enforce_selected_version_alignment, + next_sequence, parse_signing_key, read_bounded_json_source, signature, + validate_catalog_object, +) +from .source_provenance import ( + registered_source_origin_issues, selected_source_provenance, + source_tag_provenance_issues, +) +from .version_alignment import candidate_catalog_version_issues +from .workspace import META_ROOT, resolve_workspace_root, website_root + + +MAX_ARTIFACT_BYTES = 512 * 1024 * 1024 + + +@lru_cache(maxsize=3) +def _tool(name: str) -> dict[str, Any]: + # These are fixed, operator-controlled release modules, not caller paths. + if name not in { + "generate-release-package-set", "generate-release-catalog", + "resolve-package-artifacts", + }: + raise ValueError("unknown registry release tool") + return runpy.run_path(str(META_ROOT / "tools" / "release" / f"{name}.py")) + + +def authenticate_full_rebuild_base( + *, web_root: Path, channel: str, signer_public_keys: dict[str, str], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Authenticate the fixed website pair without reusing legacy entry data. + + Only this full rebuild may migrate a legacy catalog without a keyring hash. + In that case *every* active website key must exactly equal a configured + signer; injecting an additional website key cannot extend trust. Selective + candidates retain their stricter existing hash-pinned-base requirement. + """ + + root = web_root / "public" / "catalogs" / "v1" + catalog = read_bounded_json_source(root / "channels" / f"{channel}.json", label="published base catalog") + keyring = read_bounded_json_source(root / "keyring.json", label="published website keyring") + if not isinstance(catalog, dict) or not isinstance(keyring, dict): + raise ValueError("published website catalog/keyring must be objects") + trusted_keys = authenticate_base_keyring(keyring) + release = catalog.get("release") + pinned = release.get("keyring_sha256") if isinstance(release, dict) else None + if pinned is None: + if trusted_keys != signer_public_keys or len(keyring["keys"]) != len(trusted_keys): + raise ValueError("legacy full rebuild requires exactly the configured known website signers") + elif pinned != canonical_hash(keyring): + raise ValueError("published base catalog does not pin its exact website keyring") + if any(trusted_keys.get(key) != value for key, value in signer_public_keys.items()): + raise ValueError("full rebuild cannot introduce or replace a website signer") + authenticate_base_catalog_signatures( + catalog, base_trusted_keys=trusted_keys, configured_signers=signer_public_keys, + ) + validation = validate_catalog_object( + catalog, approved_channel=channel, signer_public_keys=signer_public_keys, + ) + if validation.get("valid") is not True: + raise ValueError(f"published base catalog failed validation: {validation.get('error')}") + return catalog, keyring + + +def build_full_registry_candidate( + *, package_set_path: Path, package_lock_path: Path, + wheelhouse: Path, webui_packages: Path, output_dir: Path, + selected_repositories: tuple[str, ...], signing_keys: tuple[str, ...], + workspace_root: Path | str | None = None, channel: str = "stable", + source_remote: str = "origin", public_base_url: str = "https://govoplan.add-ideas.de", + expires_days: int = 90, sequence: int | None = None, +) -> dict[str, object]: + channel = validate_release_channel(channel) + if not isinstance(expires_days, int) or isinstance(expires_days, bool) or not 1 <= expires_days <= 365: + raise ValueError("catalog expiry must be between 1 and 365 days") + workspace = resolve_workspace_root(workspace_root) + ensure_private_candidate_root(workspace) + output = output_dir.expanduser().absolute() + ensure_private_candidate_root(output.parent, create=True) + if output.exists() or output.is_symlink(): + raise ValueError("full candidate output must not already exist") + parsed_keys = tuple(parse_signing_key(value) for value in signing_keys) + if not parsed_keys: + raise ValueError("full candidate needs a configured signing key") + signer_keys = configured_signer_public_keys(parsed_keys) + base, keyring = authenticate_full_rebuild_base( + web_root=website_root(workspace), channel=channel, signer_public_keys=signer_keys, + ) + package_set = _hashed_json(package_set_path, "package_set_sha256") + lock = _hashed_json(package_lock_path, "lock_sha256") + generator = _tool("generate-release-catalog") + version = package_set.get("release_version") + if package_set.get("profile") != "full" or not isinstance(version, str): + raise ValueError("full candidate requires the complete full-profile package set") + expected = _tool("generate-release-package-set")["generate_package_set"]( + core_version=version, requirements=META_ROOT / "requirements-release.txt", + workspace=workspace, profile="full", + meta_package=META_ROOT / "packages/govoplan-meta/pyproject.toml", + ) + if package_set != expected: + raise ValueError("package set differs from exact Meta full pins or immutable tag metadata") + generator["_validate_release_inputs"](package_set, lock, core_version=version) + if lock.get("registries") != package_set.get("registries"): + raise ValueError("artifact lock uses different package registries") + versions = {row["repository"]: row["version"] for row in package_set["python"]} + origin_failures = registered_source_origin_issues( + repo_versions=versions, workspace=workspace, remote=source_remote, + ) + if origin_failures: + raise ValueError("Registered source origin gate failed: " + "; ".join(item.describe() for item in origin_failures)) + selected = set(selected_repositories) + if not selected or len(selected) != len(selected_repositories) or not selected <= versions.keys(): + raise ValueError("selected repositories must be unique members of the full package set") + selected_versions = {repo: versions[repo] for repo in sorted(selected)} + enforce_selected_version_alignment(repo_versions=selected_versions, workspace=workspace) + failures = source_tag_provenance_issues( + repo_versions=versions, workspace=workspace, remote=source_remote, + require_head_repos=selected, + ) + if failures: + raise ValueError("Full source provenance gate failed: " + "; ".join(item.describe() for item in failures)) + provenance = selected_source_provenance(repo_versions=versions, workspace=workspace) + wheel_identities = verify_registry_artifacts( + package_set=package_set, lock=lock, wheelhouse=wheelhouse, + webui_packages=webui_packages, + ) + generated_at = datetime.now(tz=UTC) + resolved_sequence = sequence if sequence is not None else next_sequence(base, generated_at=generated_at) + if isinstance(resolved_sequence, bool) or not isinstance(resolved_sequence, int) or resolved_sequence <= int(base.get("sequence") or 0): + raise ValueError("full candidate sequence must advance the authenticated published channel") + # Fresh tagged manifests, including unchanged tagged ancestors; no legacy + # entry, registry hash, source URL or dependency contract is carried over. + candidate = generator["_catalog_payload"]( + package_set=package_set, package_lock=lock, channel=channel, + sequence=resolved_sequence, generated_at=generated_at, + expires_at=generated_at + timedelta(days=expires_days), workspace=workspace, + public_base_url=public_base_url.rstrip("/"), + ) + for entry in [candidate["core_release"], *candidate["modules"]]: + repo = entry["python_package"] + entry["source"] = { + "repository": repo, "tag": f"v{versions[repo]}", + "commit": provenance[repo]["commit_sha"], + "tag_object_sha": provenance[repo]["tag_object_sha"], + "repository_url": f"https://git.add-ideas.de/GovOPlaN/{repo}", + "revision_url": f"https://git.add-ideas.de/GovOPlaN/{repo}/commit/{provenance[repo]['commit_sha']}", + } + candidate["release"].update({ + "selected_units": [ + {"repo": repo, "version": versions[repo], "tag": f"v{versions[repo]}", **provenance[repo]} + for repo in sorted(selected) + ], + "keyring_sha256": canonical_hash(keyring), + "artifacts": wheel_identities, + "base_catalog_sha256": canonical_hash(base), + }) + # Recheck the exact objects used by synthesis, including unchanged source + # ancestors, before signing. No late tag movement can change this candidate. + origin_failures = registered_source_origin_issues( + repo_versions=versions, workspace=workspace, remote=source_remote, + ) + if origin_failures: + raise ValueError("Registered source origin changed during synthesis: " + "; ".join(item.describe() for item in origin_failures)) + failures = source_tag_provenance_issues( + repo_versions=versions, workspace=workspace, remote=source_remote, + require_head_repos=selected, + expected_commits={repo: row["commit_sha"] for repo, row in provenance.items()}, + expected_tag_objects={repo: row["tag_object_sha"] for repo, row in provenance.items()}, + ) + if failures: + raise ValueError("Full source provenance changed during synthesis: " + "; ".join(item.describe() for item in failures)) + failures = candidate_catalog_version_issues(candidate) + identity_failures = selected_artifact_identity_issues(candidate) + if failures or identity_failures: + raise ValueError("full candidate identity validation failed: " + "; ".join( + [item.message for item in failures] + list(identity_failures) + )) + candidate["signatures"] = [signature(candidate, key_id=key, private_key=value) for key, value in parsed_keys] + validation = validate_catalog_object(candidate, approved_channel=channel, signer_public_keys=signer_keys) + if validation.get("valid") is not True: + raise ValueError(f"signed full candidate failed validation: {validation.get('error')}") + # Exclusive output creation preserves earlier reviewed candidates. + output.mkdir(mode=0o700) + (output / "channels").mkdir(mode=0o700) + catalog_path = output / "channels" / f"{channel}.json" + _write_private_json(catalog_path, candidate) + _write_private_json(output / "keyring.json", keyring) + write_module_directory( + catalog_payload=candidate, keyring_payload=keyring, output_root=output, + channel=channel, public_base_url=public_base_url, + ) + result = { + "status": "ready", "candidate_dir": str(output), "catalog_path": str(catalog_path), + "channel": channel, "sequence": resolved_sequence, + "package_count": len(package_set["python"]), "webui_count": len(package_set["webui"]), + "selected_count": len(selected), "candidate_catalog_hash": canonical_hash(candidate), + "candidate_keyring_hash": canonical_hash(keyring), "validation_valid": True, + } + _write_private_json(output / "summary.json", result) + harden_private_candidate_tree(output) + return result + + +def _hashed_json(path: Path, field: str) -> dict[str, Any]: + payload = read_bounded_json_source(path, label=field) + if not isinstance(payload, dict): + raise ValueError(f"{field} input must be an object") + unsigned = dict(payload) + expected = unsigned.pop(field, None) + # Registry tools use their established ASCII-escaped canonical form. + encoded = json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode() + if expected != hashlib.sha256(encoded).hexdigest(): + raise ValueError(f"{field} does not match its contents") + return payload + + +def verify_registry_artifacts( + *, package_set: dict[str, Any], lock: dict[str, Any], + wheelhouse: Path, webui_packages: Path, +) -> list[dict[str, object]]: + """Compare exact registry bytes, metadata and source bindings without installs.""" + + identities = [] + for group, root, suffix in (("python", wheelhouse, ".whl"), ("webui", webui_packages, ".tgz")): + ensure_private_candidate_root(root) + expected = {row["name"]: row for row in package_set[group]} + rows = lock[group] + if not isinstance(rows, list) or len(rows) != len(expected): + raise ValueError(f"artifact lock has duplicate/missing {group} rows") + filenames: set[str] = set() + seen: set[str] = set() + for row in rows: + selected = expected.get(row.get("name")) if isinstance(row, dict) else None + if selected is None or row["name"] in seen: + raise ValueError(f"artifact lock has unexpected/duplicate {group} identities") + seen.add(row["name"]) + for key in ("name", "version", "repository", "tag", "commit"): + if row.get(key) != selected[key]: + raise ValueError("artifact lock differs from selected source identity") + filename = row.get("filename") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,254}", filename) is None or not filename.endswith(suffix) or filename in filenames: + raise ValueError("artifact filename is invalid or duplicated") + filenames.add(filename) + path = root / filename + sha256, sha512, size = _hash_artifact(path) + if (sha256, size) != (row.get("sha256"), row.get("size")): + raise ValueError(f"registry artifact bytes differ from locked identity: {filename}") + if group == "python": + expected_url = _tool("resolve-package-artifacts")["_python_artifact_url"]( + package_set["registries"]["python"], package=selected, filename=filename, + ) + if row.get("url") != expected_url: + raise ValueError("Python artifact URL differs from the selected registry") + identity = inspect_python_wheel(path) + if (identity.package_name, identity.package_version, identity.archive_sha256, identity.archive_size) != (row["name"], row["version"], sha256, size): + raise ValueError("wheel metadata or bytes differ from the registry lock") + identities.append(identity.catalog_payload()) + else: + expected_url = ( + package_set["registries"]["npm"].rstrip("/") + "/" + + quote(row["name"], safe="") + "/-/" + + quote(row["version"], safe="") + "/" + + quote(row["name"].split("/", 1)[1] + "-" + row["version"] + ".tgz", safe="") + ) + if row.get("url") != expected_url: + raise ValueError("WebUI artifact URL differs from the selected registry") + if row.get("integrity") != "sha512-" + base64.b64encode(sha512).decode("ascii"): + raise ValueError("WebUI registry integrity differs from downloaded bytes") + _inspect_npm_metadata( + path, name=row["name"], version=row["version"], + expected_identity=(sha256, sha512, size), + ) + actual = set() + for index, path in enumerate(root.iterdir()): + if index >= 1000: + raise ValueError("registry artifact directory exceeds its inspection bound") + actual.add(path.name) + if actual != filenames: + raise ValueError(f"registry directory contains unexpected or missing {group} files") + return sorted(identities, key=lambda row: str(row["package_name"])) + + +def _hash_artifact(path: Path) -> tuple[str, bytes, int]: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + try: + initial = os.fstat(descriptor) + if not stat.S_ISREG(initial.st_mode) or not 0 < initial.st_size <= MAX_ARTIFACT_BYTES: + raise ValueError("registry artifact must be a bounded regular file") + digest, sri, total = hashlib.sha256(), hashlib.sha512(), 0 + while chunk := os.read(descriptor, 1024 * 1024): + total += len(chunk) + if total > MAX_ARTIFACT_BYTES: + raise ValueError("registry artifact exceeds its byte bound") + digest.update(chunk) + sri.update(chunk) + final = os.fstat(descriptor) + if (initial.st_ino, initial.st_size, initial.st_mtime_ns, initial.st_ctime_ns) != (final.st_ino, total, final.st_mtime_ns, final.st_ctime_ns): + raise ValueError("registry artifact changed while being inspected") + return digest.hexdigest(), sri.digest(), total + finally: + os.close(descriptor) + + +def _inspect_npm_metadata( + path: Path, *, name: str, version: str, + expected_identity: tuple[str, bytes, int] | None = None, +) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + try: + initial = os.fstat(descriptor) + if not stat.S_ISREG(initial.st_mode) or not 0 < initial.st_size <= MAX_ARTIFACT_BYTES: + raise ValueError("WebUI archive must be a bounded regular file") + digest, sri, total = hashlib.sha256(), hashlib.sha512(), 0 + while chunk := os.read(descriptor, 1024 * 1024): + total += len(chunk) + if total > MAX_ARTIFACT_BYTES: + raise ValueError("WebUI archive exceeds its byte bound") + digest.update(chunk) + sri.update(chunk) + if expected_identity is not None and (digest.hexdigest(), sri.digest(), total) != expected_identity: + raise ValueError("WebUI archive changed before metadata inspection") + os.lseek(descriptor, 0, os.SEEK_SET) + with os.fdopen(os.dup(descriptor), "rb") as stream: + _inspect_npm_stream(stream, name=name, version=version) + final = os.fstat(descriptor) + if (initial.st_ino, initial.st_size, initial.st_mtime_ns, initial.st_ctime_ns) != (final.st_ino, total, final.st_mtime_ns, final.st_ctime_ns): + raise ValueError("WebUI archive changed during metadata inspection") + finally: + os.close(descriptor) + + +def _inspect_npm_stream(stream: Any, *, name: str, version: str) -> None: + found = False + total = 0 + with tarfile.open(fileobj=stream, mode="r|gz") as archive: + for index, member in enumerate(archive): + total += member.size + parts = PurePosixPath(member.name).parts + if index >= 10000 or total > 1024 * 1024 * 1024 or member.size > 64 * 1024 * 1024: + raise ValueError("WebUI archive exceeds its inspection bound") + if not parts or parts[0] != "package" or ".." in parts or not (member.isfile() or member.isdir()): + raise ValueError("WebUI archive contains an unsafe member") + if member.name == "package/package.json": + if found or not member.isfile() or member.size > 1024 * 1024: + raise ValueError("WebUI archive has duplicate or oversized metadata") + stream = archive.extractfile(member) + if stream is None: + raise ValueError("WebUI archive metadata cannot be read") + metadata = json.loads(stream.read(1024 * 1024 + 1)) + if not isinstance(metadata, dict) or (metadata.get("name"), metadata.get("version")) != (name, version): + raise ValueError("WebUI metadata differs from the registry lock") + found = True + if not found: + raise ValueError("WebUI archive metadata is missing") + + +def _write_private_json(path: Path, payload: object) -> None: + encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + if len(encoded) > 16 * 1024 * 1024: + raise ValueError("candidate JSON exceeds its byte bound") + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) diff --git a/tools/release/govoplan_release/publisher.py b/tools/release/govoplan_release/publisher.py index ed24cc1..7a69b04 100644 --- a/tools/release/govoplan_release/publisher.py +++ b/tools/release/govoplan_release/publisher.py @@ -27,7 +27,10 @@ from .candidate_artifact import validate_release_channel from .model import CatalogPublishResult, CatalogPublishStep from .module_directory import module_directory_payloads from .selective_catalog import read_bounded_json_source, trusted_keys_from_keyring -from .source_provenance import catalog_source_selection, source_tag_provenance_issues +from .source_provenance import ( + catalog_source_selection, registered_source_origin_issues, + source_tag_provenance_issues, +) from .version_alignment import candidate_catalog_version_issues from .workspace import ( DEFAULT_WORKSPACE_ROOT, @@ -176,6 +179,22 @@ def publish_catalog_candidate( for issue in version_issues ) source_selection = catalog_source_selection(candidate_payload) + entries = [candidate_payload.get("core_release")] + if isinstance(candidate_payload.get("modules"), list): + entries.extend(candidate_payload["modules"]) + if any( + isinstance(entry, dict) + and isinstance(entry.get("python_ref"), str) + and " @ https://" in entry["python_ref"] + for entry in entries + ): + blockers.extend( + f"registered source origin: {issue.describe()}" + for issue in registered_source_origin_issues( + repo_versions=source_selection.all_versions, + workspace=workspace, remote=source_remote, + ) + ) blockers.extend( f"source provenance: {issue.describe()}" for issue in source_selection.issues diff --git a/tools/release/govoplan_release/registry_reference.py b/tools/release/govoplan_release/registry_reference.py new file mode 100644 index 0000000..bfd934e --- /dev/null +++ b/tools/release/govoplan_release/registry_reference.py @@ -0,0 +1,178 @@ +"""Strict source identities for immutable, registry-backed catalog entries.""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +import json +import re +from urllib.parse import unquote, urlsplit + + +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_COMMIT = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?\Z") +_REPO = re.compile(r"govoplan-[a-z0-9-]+\Z") +_VERSION = re.compile(r"\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?\Z") +_PYTHON = re.compile( + r"(?Pgovoplan-[a-z0-9-]+)(?:\[[a-z0-9_,.-]+\])? @ " + r"(?Phttps://\S+)#sha256=(?P[0-9a-f]{64})\Z" +) + + +@dataclass(frozen=True, slots=True) +class RegistrySource: + repository: str + version: str + commit: str + tag_object: str + + +def registry_artifact_conflicts(entries: list[object]) -> tuple[str, ...]: + """Allow repeated module projections only when package artifacts agree.""" + observed: dict[tuple[str, str], str] = {} + issues = [] + for entry in entries: + if not isinstance(entry, dict): + continue + try: + source = registry_entry_source(entry) + except ValueError as exc: + issues.append(str(exc)) + continue + if source is None: + continue + for kind, identity in entry["artifact_integrity"].items(): + if kind not in {"python", "webui"}: + continue + key = (source.repository, kind) + encoded = json.dumps(identity, sort_keys=True, separators=(",", ":")) + if observed.setdefault(key, encoded) != encoded: + issues.append(f"conflicting {kind} registry artifacts for {source.repository}") + return tuple(issues) + + +def registry_entry_source(entry: dict[str, object]) -> RegistrySource | None: + """Admit registry refs only with a complete matching artifact/source binding. + + Git-backed catalogs keep their existing validation path. An HTTPS Python + requirement cannot masquerade as a source-only/non-Python entry when its + registry provenance is missing or inconsistent. + """ + + ref = entry.get("python_ref") + if not isinstance(ref, str) or " @ https://" not in ref: + return None + match = _PYTHON.fullmatch(ref) + source = entry.get("source") + version = entry.get("version") + package = entry.get("python_package") + integrity = entry.get("artifact_integrity") + if ( + match is None + or not isinstance(source, dict) + or not isinstance(integrity, dict) + or not isinstance(version, str) + or _VERSION.fullmatch(version) is None + or package != match.group("name") + ): + raise ValueError("registry entry has no complete package/source identity") + repo = source.get("repository") + commit = source.get("commit") + tag_object = source.get("tag_object_sha") + if ( + not isinstance(repo, str) + or _REPO.fullmatch(repo) is None + or repo != package + or source.get("tag") != f"v{version}" + or not isinstance(commit, str) + or _COMMIT.fullmatch(commit) is None + or not isinstance(tag_object, str) + or _COMMIT.fullmatch(tag_object) is None + ): + raise ValueError("registry entry has invalid immutable tag provenance") + python = _artifact( + integrity.get("python"), ref=ref, package=package, version=version, + commit=commit, + ) + if python["url"] != match.group("url") or python["sha256"] != match.group("digest"): + raise ValueError("registry Python ref differs from its artifact identity") + webui_package = entry.get("webui_package") + webui_ref = entry.get("webui_ref") + if bool(webui_package) != bool(webui_ref): + raise ValueError("registry WebUI package and ref must be declared together") + if webui_package: + if not isinstance(webui_package, str) or re.fullmatch( + r"@govoplan/[a-z0-9-]+-webui", webui_package + ) is None or not isinstance(webui_ref, str): + raise ValueError("registry WebUI package identity is malformed") + if webui_package != f"@govoplan/{repo.removeprefix('govoplan-')}-webui": + raise ValueError("registry WebUI package belongs to another source repository") + webui = _artifact( + integrity.get("webui"), ref=webui_ref, package=webui_package, + version=version, commit=commit, + ) + if webui_ref != webui["url"]: + raise ValueError("registry WebUI ref differs from its artifact identity") + sri = webui.get("integrity") + try: + valid_sri = isinstance(sri, str) and sri.startswith("sha512-") and len( + base64.b64decode(sri[7:], validate=True) + ) == 64 + except ValueError: + valid_sri = False + if not valid_sri: + raise ValueError("registry WebUI artifact needs a SHA-512 integrity identity") + elif "webui" in integrity: + raise ValueError("registry entry carries an unexpected WebUI artifact") + return RegistrySource(repo, version, commit, tag_object) + + +def _artifact( + value: object, *, ref: str, package: str, version: str, commit: str, +) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError("registry entry is missing artifact integrity") + url = value.get("url") + filename = value.get("filename") + digest = value.get("sha256") + size = value.get("size") + parsed = urlsplit(url) if isinstance(url, str) else None + url_filename = unquote(parsed.path.rsplit("/", 1)[-1]) if parsed else "" + decoded_parts = unquote(parsed.path).split("/") if parsed else [] + expected_filenames = {url_filename} + expected_path = [package, version, url_filename] + if package.startswith("@govoplan/"): + # npm pack includes the scope in its local filename; the registry's + # immutable download URL uses the unscoped package basename. + expected_filenames = { + f"govoplan-{package.split('/', 1)[1]}-{version}.tgz", + } if url_filename == f"{package.split('/', 1)[1]}-{version}.tgz" else set() + expected_path = [*package.split("/"), "-", version, url_filename] + if ( + parsed is None + or parsed.scheme != "https" + or not parsed.netloc + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + or parsed.query + or any(part in {".", ".."} for part in unquote(parsed.path).split("/")) + or "%" in unquote(parsed.path) + or "\\" in unquote(parsed.path) + or any(ord(character) < 32 for character in url) + or decoded_parts[-len(expected_path):] != expected_path + or not isinstance(filename, str) + or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,254}", filename) is None + or filename not in expected_filenames + or not isinstance(digest, str) + or _SHA256.fullmatch(digest) is None + or not isinstance(size, int) + or isinstance(size, bool) + or not 0 < size <= 512 * 1024 * 1024 + or value.get("ref") != ref + or value.get("registry_identity") != f"{package}@{version}" + or value.get("git_ref") != f"v{version}" + or value.get("source_commit") != commit + ): + raise ValueError("registry artifact ref, version, bytes, or source binding is inconsistent") + return value diff --git a/tools/release/govoplan_release/source_provenance.py b/tools/release/govoplan_release/source_provenance.py index 536297b..937fc9d 100644 --- a/tools/release/govoplan_release/source_provenance.py +++ b/tools/release/govoplan_release/source_provenance.py @@ -19,10 +19,33 @@ from .git_state import ( scoped_git_command, ) from .repository_tag import RemoteTagResult, ref_commit, remote_tag_commit +from .registry_reference import registry_entry_source from .version_alignment import repository_version_issues from .workspace import load_repository_specs, resolve_repo_path +def registered_source_origin_issues( + *, repo_versions: dict[str, str], workspace: Path, remote: str, +) -> tuple[SourceTagProvenanceIssue, ...]: + """Bind registry candidate attestations to registered source endpoints.""" + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + issues = [] + for repo, version in sorted(repo_versions.items()): + spec = specs.get(repo) + if spec is None: + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "source repository is not registered")) + continue + path = resolve_repo_path(spec, workspace) + if path.absolute() != path.resolve() or not path.resolve().is_relative_to(workspace.resolve()): + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "source checkout leaves the private workspace or traverses a symlink")) + continue + fetch = git_text(path, "remote", "get-url", "--all", remote).splitlines() + push = git_text(path, "remote", "get-url", "--push", "--all", remote).splitlines() + if fetch != [spec.remote] or push != [spec.remote]: + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "source remote does not exactly match the registered origin")) + return tuple(issues) + + _CATALOG_PYTHON_REF = re.compile( r"/(?Pgovoplan-[a-z0-9-]+)\.git@v(?P[^\s;]+)$" ) @@ -61,6 +84,8 @@ def catalog_source_selection(payload: object) -> CatalogSourceSelection: ) versions: dict[str, str] = {} + registry_commits: dict[str, str] = {} + registry_tag_objects: dict[str, str] = {} issues: list[SourceTagProvenanceIssue] = [] entries: list[tuple[str, object]] = [("core_release", payload.get("core_release"))] modules = payload.get("modules") @@ -70,6 +95,21 @@ def catalog_source_selection(payload: object) -> CatalogSourceSelection: for source, raw_entry in entries: if not isinstance(raw_entry, dict): continue + try: + registry_source = registry_entry_source(raw_entry) + except ValueError as exc: + issues.append(SourceTagProvenanceIssue(source, "", str(exc))) + continue + if registry_source is not None: + repo, version = registry_source.repository, registry_source.version + previous = versions.setdefault(repo, version) + previous_commit = registry_commits.setdefault(repo, registry_source.commit) + previous_object = registry_tag_objects.setdefault(repo, registry_source.tag_object) + if (previous, previous_commit, previous_object) != ( + version, registry_source.commit, registry_source.tag_object, + ): + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "registry entries have conflicting immutable source identities")) + continue python_ref = raw_entry.get("python_ref") match = _CATALOG_PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None if match is None: @@ -89,8 +129,8 @@ def catalog_source_selection(payload: object) -> CatalogSourceSelection: release = payload.get("release") selected_units = release.get("selected_units") if isinstance(release, dict) else None selected: dict[str, str] = {} - selected_commits: dict[str, str] = {} - selected_tag_objects: dict[str, str] = {} + selected_commits: dict[str, str] = dict(registry_commits) + selected_tag_objects: dict[str, str] = dict(registry_tag_objects) if not isinstance(selected_units, list) or not selected_units: issues.append( SourceTagProvenanceIssue( @@ -106,16 +146,22 @@ def catalog_source_selection(payload: object) -> CatalogSourceSelection: repo = unit.get("repo") version = unit.get("version") if isinstance(repo, str) and isinstance(version, str): + if repo in selected: + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "duplicate selected repository")) selected[repo] = version.removeprefix("v") commit = unit.get("commit_sha") tag_object = unit.get("tag_object_sha") if isinstance(commit, str) and re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", commit): + if repo in registry_commits and registry_commits[repo] != commit.lower(): + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "selected commit differs from registry artifact source")) selected_commits[repo] = commit.lower() else: issues.append( SourceTagProvenanceIssue(repo, f"v{version.removeprefix('v')}", "selected unit has no valid commit_sha provenance") ) if isinstance(tag_object, str) and re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", tag_object): + if repo in registry_tag_objects and registry_tag_objects[repo] != tag_object.lower(): + issues.append(SourceTagProvenanceIssue(repo, f"v{version}", "selected tag object differs from registry artifact source")) selected_tag_objects[repo] = tag_object.lower() else: issues.append( diff --git a/tools/release/govoplan_release/version_alignment.py b/tools/release/govoplan_release/version_alignment.py index fcb2e2c..f314db3 100644 --- a/tools/release/govoplan_release/version_alignment.py +++ b/tools/release/govoplan_release/version_alignment.py @@ -10,6 +10,7 @@ import subprocess import tomllib from .git_state import collect_versions, sanitized_git_environment +from .registry_reference import registry_artifact_conflicts, registry_entry_source from .workspace import load_repository_specs, resolve_repo_path @@ -407,6 +408,11 @@ def candidate_catalog_version_issues(payload: object) -> tuple[VersionAlignmentI ) release = payload.get("release") + registry_entries = [core_release, *(modules if isinstance(modules, list) else [])] + issues.extend( + _catalog_shape_issue("artifact_integrity", issue) + for issue in registry_artifact_conflicts(registry_entries) + ) if isinstance(release, dict): issues.extend(_catalog_release_issues(release, represented=represented)) return tuple(issues) @@ -419,6 +425,15 @@ def _catalog_entry_issues( represented: dict[str, str], ) -> list[VersionAlignmentIssue]: issues: list[VersionAlignmentIssue] = [] + try: + registry_source = registry_entry_source(entry) + except ValueError as exc: + return [_catalog_shape_issue(source, str(exc))] + if registry_source is not None: + previous = represented.setdefault(registry_source.repository, registry_source.version) + if previous != registry_source.version: + issues.append(_catalog_shape_issue(source, "repository appears with conflicting catalog versions")) + return issues version = entry.get("version") normalized_version = version.removeprefix("v") if isinstance(version, str) and version else None if normalized_version is None: diff --git a/tools/release/release-catalog.py b/tools/release/release-catalog.py index b08483a..87a510d 100644 --- a/tools/release/release-catalog.py +++ b/tools/release/release-catalog.py @@ -21,6 +21,22 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) + full = subparsers.add_parser("full-registry", help="Build a strict full-profile candidate from verified registry artifacts.") + full.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT) + full.add_argument("--package-set", type=Path, required=True) + full.add_argument("--package-lock", type=Path, required=True) + full.add_argument("--wheelhouse", type=Path, required=True) + full.add_argument("--webui-packages", type=Path, required=True) + full.add_argument("--output-dir", type=Path, required=True) + full.add_argument("--selected-repository", action="append", required=True) + full.add_argument("--catalog-signing-key", action="append", required=True) + full.add_argument("--channel", default="stable") + full.add_argument("--source-remote", default="origin") + full.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") + full.add_argument("--expires-days", type=int, default=90) + full.add_argument("--sequence", type=int) + full.add_argument("--json", action="store_true") + selective = subparsers.add_parser( "selective", help="Build a signed selective channel catalog candidate." ) @@ -143,6 +159,21 @@ def main() -> int: ) args = parser.parse_args() + if args.command == "full-registry": + require_release_runtime_trust() + from govoplan_release.full_catalog import build_full_registry_candidate + + result = build_full_registry_candidate( + package_set_path=args.package_set, package_lock_path=args.package_lock, + wheelhouse=args.wheelhouse, webui_packages=args.webui_packages, + output_dir=args.output_dir, selected_repositories=tuple(args.selected_repository), + signing_keys=tuple(args.catalog_signing_key), workspace_root=args.workspace_root, + channel=args.channel, source_remote=args.source_remote, + public_base_url=args.public_base_url, expires_days=args.expires_days, + sequence=args.sequence, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if args.command == "selective": require_release_runtime_trust() result = build_selective_catalog_candidate(