From 09046e6e625ea6db8ddd56795a10d805fd0e9106 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 6 Aug 2026 21:13:33 +0200 Subject: [PATCH] Publish complete signed module catalogs --- .env.example | 2 +- .gitea/workflows/runtime-distribution.yml | 6 + README.md | 2 +- docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md | 5 +- docs/DEPLOYMENT_PROFILES.md | 16 + docs/PACKAGE_REGISTRY_RELEASES.md | 71 ++- ...STEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md | 34 +- packages/govoplan-meta/pyproject.toml | 4 +- tests/test_package_registry_release.py | 43 ++ tests/test_release_entrypoint_gates.py | 4 +- tools/release/generate-release-catalog.py | 519 +++++++----------- tools/release/generate-release-package-set.py | 65 ++- .../catalog_entry_synthesis.py | 126 +++-- .../govoplan_release/selective_catalog.py | 27 + .../install-webui-release-dependencies.sh | 16 +- tools/release/publish-release-catalog.sh | 32 +- tools/release/resolve-package-artifacts.py | 88 ++- 17 files changed, 666 insertions(+), 394 deletions(-) diff --git a/.env.example b/.env.example index ddaf777..e9c80f8 100644 --- a/.env.example +++ b/.env.example @@ -57,4 +57,4 @@ DEV_MAILBOX_API_ENABLED=false GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable diff --git a/.gitea/workflows/runtime-distribution.yml b/.gitea/workflows/runtime-distribution.yml index 4e96aa4..7259de1 100644 --- a/.gitea/workflows/runtime-distribution.yml +++ b/.gitea/workflows/runtime-distribution.yml @@ -101,6 +101,8 @@ jobs: mkdir -p runtime-output git rev-parse "v$VERSION^{commit}" > runtime-output/release-source-commit grep -Eq '^[0-9a-f]{40}$' runtime-output/release-source-commit + git show "v$VERSION:requirements-release.txt" > runtime-output/requirements-release.source.txt + git show "v$VERSION:packages/govoplan-meta/pyproject.toml" > runtime-output/govoplan-meta.source.toml - name: Use HTTPS for GovOPlaN repositories run: | git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan" @@ -119,6 +121,9 @@ jobs: .runtime-build/bin/python -m pip install --upgrade pip cryptography .runtime-build/bin/python tools/release/generate-release-package-set.py \ --version "$VERSION" \ + --profile full \ + --requirements runtime-output/requirements-release.source.txt \ + --meta-package runtime-output/govoplan-meta.source.toml \ --output runtime-output/release-packages.json .runtime-build/bin/python tools/release/resolve-package-artifacts.py \ --package-set runtime-output/release-packages.json \ @@ -130,6 +135,7 @@ jobs: PYTHON="$PWD/.runtime-build/bin/python" \ GOVOPLAN_WEBUI_PACKAGE_LOCK="$PWD/runtime-output/package-artifacts.lock.json" \ GOVOPLAN_WEBUI_PACKAGE_DIR="$PWD/runtime-output/webui-packages" \ + GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES=true \ bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui npm --prefix ../govoplan-core/webui run build .runtime-build/bin/python tools/release/prepare-runtime-context.py \ diff --git a/README.md b/README.md index 02411f7..1e8ef9b 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ sequence are documented in the The selected Campaign-to-Postbox-to-data-to-collaboration implementation path, including stage gates and shared documentation expectations, is in the [Reference Journey Program](docs/REFERENCE_JOURNEY_PROGRAM.md). -The administrator journey from Core-only bootstrap through online module +The administrator journey from a Core-baseline bootstrap through online module installation, scale-out, and reversible environment promotion is defined in [System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md). The corresponding host deployment compiler, managed/external component choices, diff --git a/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md b/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md index 9b535f8..d17dd55 100644 --- a/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md +++ b/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md @@ -267,7 +267,7 @@ inspection is authorized. The complete installation and lifecycle journey is specified in the [System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md): -one-command Core-only bootstrap, signed online module installation and updates, +one-command Core-baseline bootstrap, signed online module installation and updates, stateless scale-out, versioned configuration transfer, undo, and reproducible environment-promotion recipes. @@ -977,7 +977,8 @@ Priorities: 1. Deliver the first slices of the [System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md): - a verified Core-only distribution, first-run control plane, read-only online + a verified full-package distribution with only the Core baseline active, + first-run control plane, read-only online module directory, and durable plan/confirm/install progress. 2. Pin and publish a compatible Core/WebUI/module composition and first reference configuration package. diff --git a/docs/DEPLOYMENT_PROFILES.md b/docs/DEPLOYMENT_PROFILES.md index 1f68e19..21f43d1 100644 --- a/docs/DEPLOYMENT_PROFILES.md +++ b/docs/DEPLOYMENT_PROFILES.md @@ -38,6 +38,22 @@ The K3s VM lab has two modes over the same Kubernetes profile: - `acceptance` requires independently controlled worker failure domains and can contribute target evidence. +## Module composition and availability + +Official immutable API and WebUI images carry the verified `full` package +profile. This is package availability, not runtime activation and not a license +or tenant entitlement. The signed distribution manifest records the complete +package composition; the desired module graph selects which installed modules +are active; tenant module policy applies unavailable/available/forced ceilings; +and Views/Policy control group and user presentation. + +Local and single-host profiles may use the supervised installer to download a +signed catalog artifact into a private digest cache and mutate the local package +environment during maintenance. A multi-host/shared-state profile must never +change one replica in place. Its Admin install plan is a composition request: +publish and roll out a new signed image whose package lock contains the target, +then activate the module graph after all replicas report the same composition. + ## Component Choices The installer may manage a component where its bounded profile is appropriate, diff --git a/docs/PACKAGE_REGISTRY_RELEASES.md b/docs/PACKAGE_REGISTRY_RELEASES.md index 863c49f..8192a17 100644 --- a/docs/PACKAGE_REGISTRY_RELEASES.md +++ b/docs/PACKAGE_REGISTRY_RELEASES.md @@ -136,31 +136,38 @@ allow the same least-privilege credential to serve every module workflow. ## Exact release consumption -`tools/release/generate-release-package-set.py` translates the reviewed Git -source refs in `requirements-release.txt` into an exact registry package set. -It resolves each version tag to its commit and verifies the package metadata in -that tag. +`tools/release/generate-release-package-set.py` supports two explicit package +profiles. `base` translates the reviewed roots in `requirements-release.txt`; +`full` reads the exact `govoplan[full]` dependency set from the developer +meta-package. Both profiles resolve every version tag to its commit and verify +the package metadata from that exact Git tree. The official module directory +and immutable runtime distribution use `full`, so every publicly released +module can be discovered without rebuilding the application image. `tools/release/resolve-package-artifacts.py` then downloads exactly those wheel and WebUI versions from Gitea. It reads the identity embedded in every wheel and npm tarball, rejects missing, duplicate, unexpected, or oversized artifacts, -and writes `package-artifacts.lock.json` with SHA-256 values and npm integrity -values. Credentials are accepted only through environment variables and are +and writes `package-artifacts.lock.json` with credential-free HTTPS download +URLs, SHA-256 values, and npm registry integrity values. The resolver verifies +that the bytes downloaded by `npm pack` match the registry's own integrity +record. Credentials are accepted only through environment variables and are never written to the lock. Python resolution ignores ambient pip configuration and extra indexes for GovOPlaN roots, preventing an internal package name from being selected from an undeclared registry. -The runtime distribution workflow uses the verified wheelhouse directly and -installs module WebUI tarballs only after matching them to the lock. It publishes -the package set, package lock, and hash-locked requirements as release assets. +The runtime distribution workflow uses the verified full-profile wheelhouse +directly and installs every selected module WebUI tarball only after matching +it to the lock. It publishes the package set, package lock, and hash-locked +requirements as release assets. The WebUI installer receives the absolute runtime-build interpreter path so its directory changes cannot escape the isolated release environment. Gitea 1.24 dispatches this workflow from a branch, but that branch is only the workflow implementation. The job fetches and peels the protected `v` -tag explicitly, then binds both the signed distribution source and the Gitea -release assets to that exact commit. A post-tag workflow repair can therefore -retry publication without relabelling the later branch commit as released -source. +tag explicitly and materializes both `requirements-release.txt` and the +developer meta-package from that Git tree. It then binds the signed distribution +source and Gitea release assets to the same exact commit. A post-tag workflow +repair can therefore retry publication without changing the released package +composition or relabelling the later branch commit as released source. The package-lock SHA-256 is part of the signed distribution manifest. Runtime finalization also requires the lock's package versions and hashes to match the wheel composition embedded in the images. OCI assembly remains network-free @@ -170,6 +177,44 @@ The source refs remain in the module catalog for source provenance and release planning. Production installation consumes the signed runtime images rather than invoking `pip`, `npm`, or Git on the target host. +## Public module directory + +`tools/release/publish-release-catalog.sh` resolves the selected package set and +registry lock before it creates a catalog. Catalog entries are synthesized from +the exact tagged module manifests, never from a hand-maintained module list or +the current workspace. Each entry binds its Python wheel and optional WebUI +tarball to the registry URL, filename, size, SHA-256, package identity, source +tag, and source commit before the complete catalog is signed. + +Official GovOPlaN modules are open-source directory entries and do not require +license entitlements. The generic `license_features` contract remains available +for third-party package directories, support/configuration packages, or future +deployment-specific presets. A catalog entry is gated only when that entry +explicitly declares such features. + +Core carries the public stable catalog URL and its independently pinned trust +anchor. In the absence of an operator-configured catalog, Admin discovers the +official directory automatically. Selecting an entry creates a reviewed +install/update plan; the trusted installer downloads the exact signed artifacts +into a private digest cache, verifies size and hash, and installs only from that +cache. A saved plan is rejected if any package ref, artifact identity, catalog +channel, sequence, or signing-key identity differs from the currently validated +catalog. + +Package lifecycle and availability are intentionally separate: + +- install, update, and uninstall change the instance-wide package composition; +- enable and disable change the active instance runtime graph; +- tenant module entitlements define unavailable, available, and forced modules; +- group/user presentation is governed through Views and Policy; and +- enabling a capability module does not opt data into that capability. + +Single-process or single-host installations may execute a supervised package +plan locally. Shared-state and Kubernetes profiles reject node-local package +mutation: operators compose and roll out a new signed full-profile runtime image +instead. This prevents replicas from drifting while retaining the same Admin +catalog and preflight experience. + ## Developer meta-package `packages/govoplan-meta` builds the optional `govoplan` package. Its default diff --git a/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md b/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md index 714804e..aad5b17 100644 --- a/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md +++ b/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md @@ -4,7 +4,8 @@ > As a system administrator, I can execute one shell command that downloads a > verified GovOPlaN distribution and starts a completely configured Core control -> plane without optional modules. In the WebUI I can browse compatible signed +> plane with the official package directory available but only the protected +> baseline active. In the WebUI I can browse compatible signed > module releases, select the modules for this installation, and follow every > download, validation, migration, installation, activation, and health-check > step. When an update is available, I can review its impact and confirm it. @@ -25,7 +26,8 @@ The canonical backlog item is - **Core control plane:** the smallest bootable distribution: Core API, Core WebUI, PostgreSQL, Redis, installer worker, migration runner, and durable - storage configuration. No optional GovOPlaN module package is installed. + storage configuration. An immutable image may carry the full verified package + profile, but optional modules are not active or tenant-entitled by implication. - **Bootstrap administrator:** a single-use, time-limited installation identity that may access only first-run and module-lifecycle functions. It is retired when the selected identity/access configuration becomes healthy. @@ -55,7 +57,9 @@ The canonical backlog item is 5. It prints the local URL and one-time bootstrap credential. Re-running the command is idempotent and shows or repairs the existing installation rather than creating another identity or database. -6. No optional module is installed or enabled at this point. +6. Only the protected baseline is enabled. Installed package availability does + not grant permissions, tenant entitlement, View visibility, or capability + opt-in. ### Module selection, installation, and update @@ -160,19 +164,22 @@ Implementation status as of the current source tree: ledger and deployment operation journal. Automatic database backup and broad adoption by module-owned external effects remain open work. -1. **Reproducible Core-only distribution.** Publish pinned multi-architecture - images, signed distribution manifest, Core-only Compose profile, bootstrap +1. **Reproducible Core-baseline distribution.** Publish pinned multi-architecture + full-package images, signed distribution manifest, Core-baseline Compose profile, bootstrap preflight, generated secrets, readiness, and idempotent rerun/repair. 2. **First-run control plane.** Add the restricted bootstrap administrator, one-time enrollment, initial catalog/keyring configuration, and retirement after durable administrator access is established. -3. **Read-only online module directory.** Move the existing catalog and module - directory contracts into the installed Core WebUI with compatibility, - provenance, release-note, and update-state presentation. -4. **Durable module plan and install.** Reuse the existing installer queue, - locks, signed-package validator, rollback drill, and run evidence behind a - plan/confirm/progress UI. Add initial catalog-entry synthesis and artifact - acquisition where the current release console still assumes local sources. +3. **Read-only online module directory (implemented foundation).** Admin falls + back to the signed public stable directory, presents installed/update state, + compatibility and provenance, and retains operator-configured catalogs as an + explicit override. +4. **Durable module plan and install (implemented local boundary).** Catalog + selection creates a reviewed plan; the installer queue, lock, preflight, + maintenance gate, digest-verified artifact cache, rollback drill, and run + evidence remain separate from the API process. Shared deployments convert + the same intent into a new immutable release composition instead of mutating + one replica. 5. **Safe module update.** Add drain/maintenance coordination, backup gate, migration compatibility window, reconnectable progress, health verification, retry/recovery, and update notification. @@ -194,7 +201,8 @@ Implementation status as of the current source tree: ## Explicit non-goals for the first distribution slice -- Shipping optional modules in the Core image. +- Activating, tenant-entitling, or exposing optional modules merely because the + immutable image carries their verified packages. - Exporting secrets or production business data with configuration. - Pretending every schema migration can be reversed automatically. - Building a proprietary orchestrator instead of supporting Compose and a diff --git a/packages/govoplan-meta/pyproject.toml b/packages/govoplan-meta/pyproject.toml index 6face97..fe3cd39 100644 --- a/packages/govoplan-meta/pyproject.toml +++ b/packages/govoplan-meta/pyproject.toml @@ -64,7 +64,7 @@ full = [ "govoplan-procurement==0.1.18", "govoplan-projects==0.1.18", "govoplan-quick-access==0.1.18", - "govoplan-records==0.1.18", + "govoplan-records==0.1.19", "govoplan-reporting==0.1.18", "govoplan-resources==0.1.18", "govoplan-rest==0.1.18", @@ -73,7 +73,7 @@ full = [ "govoplan-search==0.1.18", "govoplan-services==0.1.18", "govoplan-soap==0.1.18", - "govoplan-tasks==0.1.18", + "govoplan-tasks==0.1.19", "govoplan-templates==0.1.18", "govoplan-tickets==0.1.18", "govoplan-transparency==0.1.18", diff --git a/tests/test_package_registry_release.py b/tests/test_package_registry_release.py index 2f8b7fb..0ca7b2a 100644 --- a/tests/test_package_registry_release.py +++ b/tests/test_package_registry_release.py @@ -46,6 +46,7 @@ class PackageRegistryReleaseTests(unittest.TestCase): ) self.assertEqual("1", payload["schema_version"]) + self.assertEqual("base", payload["profile"]) self.assertEqual("govoplan-core", payload["python"][0]["name"]) self.assertIn( "@govoplan/core-webui", @@ -55,6 +56,43 @@ class PackageRegistryReleaseTests(unittest.TestCase): digest = unsigned.pop("package_set_sha256") self.assertEqual(ARTIFACTS._canonical_sha256(unsigned), digest) + def test_full_profile_is_derived_from_the_developer_meta_package(self) -> None: + selected = PACKAGE_SET.parse_meta_package( + ROOT / "packages/govoplan-meta/pyproject.toml", + core_version="0.1.18", + ) + + by_name = {item["name"]: item for item in selected} + self.assertIn("govoplan-core", by_name) + self.assertIn("govoplan-records", by_name) + self.assertEqual("0.1.19", by_name["govoplan-tasks"]["version"]) + + payload = PACKAGE_SET.generate_package_set( + core_version="0.1.18", + requirements=ROOT / "requirements-release.txt", + workspace=ROOT.parent, + profile="full", + meta_package=ROOT / "packages/govoplan-meta/pyproject.toml", + ) + self.assertEqual("full", payload["profile"]) + self.assertEqual(len(selected), len(payload["python"])) + self.assertIn( + "@govoplan/records-webui", + {item["name"] for item in payload["webui"]}, + ) + + def test_python_registry_artifact_url_is_immutable_and_credential_free(self) -> None: + url = ARTIFACTS._python_artifact_url( + "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple", + package={"name": "govoplan-files", "version": "0.1.18"}, + filename="govoplan_files-0.1.18-py3-none-any.whl", + ) + + self.assertEqual( + "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/files/govoplan-files/0.1.18/govoplan_files-0.1.18-py3-none-any.whl", + url, + ) + def test_wheel_and_webui_artifacts_are_verified_by_embedded_identity(self) -> None: with tempfile.TemporaryDirectory(prefix="govoplan-package-artifacts-") as value: root = Path(value) @@ -138,6 +176,11 @@ class PackageRegistryReleaseTests(unittest.TestCase): ) self.assertIn("resolve-package-artifacts.py", workflow) + self.assertIn("--profile full", workflow) + self.assertIn('git show "v$VERSION:requirements-release.txt"', workflow) + self.assertIn('git show "v$VERSION:packages/govoplan-meta/pyproject.toml"', workflow) + self.assertIn("--meta-package runtime-output/govoplan-meta.source.toml", workflow) + self.assertIn("GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES=true", workflow) self.assertIn("package-artifacts.lock.json", workflow) self.assertIn( "--package-lock runtime-output/package-artifacts.lock.json", diff --git a/tests/test_release_entrypoint_gates.py b/tests/test_release_entrypoint_gates.py index f1a316c..7a94126 100644 --- a/tests/test_release_entrypoint_gates.py +++ b/tests/test_release_entrypoint_gates.py @@ -95,10 +95,10 @@ class ReleaseEntrypointGateTests(unittest.TestCase): ) self.assertIn('cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"', script) - def test_source_catalog_generator_enforces_explicit_repo_versions(self) -> None: + def test_catalog_generator_validates_registry_package_set_before_writing(self) -> None: script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text() - gate = script.index("selected_repository_version_issues(") + gate = script.index("_validate_release_inputs(package_set, package_lock") write = script.index("output.write_text(") self.assertLess(gate, write) diff --git a/tools/release/generate-release-catalog.py b/tools/release/generate-release-catalog.py index 54bf676..2f646a6 100644 --- a/tools/release/generate-release-catalog.py +++ b/tools/release/generate-release-catalog.py @@ -1,187 +1,44 @@ #!/usr/bin/env python3 -"""Generate and sign a GovOPlaN module package release catalog.""" +"""Generate a signed registry-backed GovOPlaN module package catalog.""" from __future__ import annotations import argparse import base64 -from dataclasses import dataclass from datetime import UTC, datetime, timedelta +import hashlib import json import os from pathlib import Path +import re import sys from typing import Any +from urllib.parse import urlsplit from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + META_ROOT = Path(__file__).resolve().parents[2] CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve() sys.path.insert(0, str(CORE_ROOT / "src")) sys.path.insert(0, str(META_ROOT / "tools" / "release")) -from govoplan_core.core.modules import ModuleManifest # noqa: E402 -from govoplan_core.server.registry import available_module_manifests # noqa: E402 -from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402 - - -GITEA_BASE = "git+ssh://git@git.add-ideas.de/GovOPlaN" - - -@dataclass(frozen=True, slots=True) -class CatalogModule: - module_id: str - repo: str - python_package: str - name: str - description: str - tags: tuple[str, ...] - webui_package: str | None = None - provides_interfaces: tuple[dict[str, object], ...] = () - requires_interfaces: tuple[dict[str, object], ...] = () - - -CATALOG_MODULES = ( - CatalogModule( - module_id="tenancy", - repo="govoplan-tenancy", - python_package="govoplan-tenancy", - name="Tenancy", - description="Tenant registry, tenant settings, and tenant resolution platform module.", - tags=("official", "platform-module"), - webui_package="@govoplan/tenancy-webui", - ), - CatalogModule( - module_id="organizations", - repo="govoplan-organizations", - python_package="govoplan-organizations", - name="Organizations", - description="Organization units, functions, and account-held function assignments.", - tags=("official", "platform-module"), - ), - CatalogModule( - module_id="identity", - repo="govoplan-identity", - python_package="govoplan-identity", - name="Identity", - description="Canonical identities and links between identities and platform accounts.", - tags=("official", "platform-module"), - ), - CatalogModule( - module_id="access", - repo="govoplan-access", - python_package="govoplan-access", - name="Access", - description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.", - tags=("official", "platform-module"), - webui_package="@govoplan/access-webui", - ), - CatalogModule( - module_id="admin", - repo="govoplan-admin", - python_package="govoplan-admin", - name="Admin", - description="System settings, governance templates, module management, and admin shell contributions.", - tags=("official", "platform-module"), - webui_package="@govoplan/admin-webui", - ), - CatalogModule( - module_id="policy", - repo="govoplan-policy", - python_package="govoplan-policy", - name="Policy", - description="Policy and governance capability module.", - tags=("official", "platform-module"), - webui_package="@govoplan/policy-webui", - ), - CatalogModule( - module_id="audit", - repo="govoplan-audit", - python_package="govoplan-audit", - name="Audit", - description="Audit-log storage and audit administration routes.", - tags=("official", "platform-module"), - webui_package="@govoplan/audit-webui", - ), - CatalogModule( - module_id="dashboard", - repo="govoplan-dashboard", - python_package="govoplan-dashboard", - name="Dashboard", - description="Configurable user home assembled from module-provided dashboard widgets.", - tags=("official", "platform-module"), - webui_package="@govoplan/dashboard-webui", - ), - CatalogModule( - module_id="addresses", - repo="govoplan-addresses", - python_package="govoplan-addresses", - name="Addresses", - description="Reusable address directories, recipient sources, consent metadata, and address quality workflows.", - tags=("official", "business-module"), - webui_package="@govoplan/addresses-webui", - ), - CatalogModule( - module_id="files", - repo="govoplan-files", - python_package="govoplan-files", - name="Files", - description="Managed file spaces and campaign attachment integration.", - tags=("official", "service-module"), - webui_package="@govoplan/files-webui", - ), - CatalogModule( - module_id="mail", - repo="govoplan-mail", - python_package="govoplan-mail", - name="Mail", - description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.", - tags=("official", "service-module"), - webui_package="@govoplan/mail-webui", - ), - CatalogModule( - module_id="campaigns", - repo="govoplan-campaign", - python_package="govoplan-campaign", - name="Campaigns", - description="Campaign authoring, validation, queueing, delivery control, and reports.", - tags=("official", "business-module"), - webui_package="@govoplan/campaign-webui", - ), - CatalogModule( - module_id="calendar", - repo="govoplan-calendar", - python_package="govoplan-calendar", - name="Calendar", - description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.", - tags=("official", "service-module"), - webui_package="@govoplan/calendar-webui", - ), - CatalogModule( - module_id="docs", - repo="govoplan-docs", - python_package="govoplan-docs", - name="Docs", - description="Configured-system documentation and evidence-aware help surfaces.", - tags=("official", "platform-module"), - webui_package="@govoplan/docs-webui", - ), - CatalogModule( - module_id="ops", - repo="govoplan-ops", - python_package="govoplan-ops", - name="Ops", - description="Runtime health, deployment profile, worker split, and sizing visibility.", - tags=("official", "platform-module"), - webui_package="@govoplan/ops-webui", - ), +from govoplan_release.catalog_entry_synthesis import ( # noqa: E402 + synthesize_repository_catalog_entries, + validate_initial_entry_closure, ) +SHA256 = re.compile(r"^[0-9a-f]{64}$") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.") + parser.add_argument("--version", required=True, help="Core release version, without leading v.") + parser.add_argument("--package-set", type=Path, required=True) + parser.add_argument("--package-lock", type=Path, required=True) + parser.add_argument("--workspace", type=Path, default=META_ROOT.parent) parser.add_argument("--channel", default="stable") parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.") parser.add_argument("--expires-days", type=int, default=90) @@ -195,41 +52,33 @@ def main() -> int: help="Ed25519 private key used to sign the catalog; may be repeated for rotation.", ) parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") - parser.add_argument("--repository-base", default=GITEA_BASE) args = parser.parse_args() - version = args.version.removeprefix("v") - version_issues = selected_repository_version_issues( - repo_versions={ - "govoplan-core": version, - **{module.repo: version for module in CATALOG_MODULES}, - }, - workspace=CORE_ROOT.parent, - ) - if version_issues: - details = "; ".join( - f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})" - for issue in version_issues + try: + version = args.version.removeprefix("v") + package_set = _read_hashed_json(args.package_set, hash_field="package_set_sha256") + package_lock = _read_hashed_json(args.package_lock, hash_field="lock_sha256") + _validate_release_inputs(package_set, package_lock, core_version=version) + signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key] + generated_at = datetime.now(tz=UTC) + sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M")) + catalog = _catalog_payload( + package_set=package_set, + package_lock=package_lock, + channel=args.channel, + sequence=sequence, + generated_at=generated_at, + expires_at=generated_at + timedelta(days=args.expires_days), + workspace=args.workspace.expanduser().resolve(), + public_base_url=args.public_base_url.rstrip("/"), ) - parser.error(f"version alignment gate failed: {details}") - tag = f"v{version}" - generated_at = datetime.now(tz=UTC) - sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M")) - expires_at = generated_at + timedelta(days=args.expires_days) - signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key] - - catalog = _catalog_payload( - version=version, - tag=tag, - channel=args.channel, - sequence=sequence, - generated_at=generated_at, - expires_at=expires_at, - repository_base=args.repository_base.rstrip("/"), - public_base_url=args.public_base_url.rstrip("/"), - ) - if signing_keys: - catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys] + if signing_keys: + catalog["signatures"] = [ + _signature(catalog, key_id=key_id, private_key=private_key) + for key_id, private_key in signing_keys + ] + except (KeyError, OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) output = args.catalog_output.expanduser() output.parent.mkdir(parents=True, exist_ok=True) @@ -249,48 +98,98 @@ def main() -> int: print(f"channel={args.channel}") print(f"sequence={sequence}") print(f"version={version}") + print(f"profile={package_set.get('profile', 'base')}") return 0 def _catalog_payload( *, - version: str, - tag: str, + package_set: dict[str, Any], + package_lock: dict[str, Any], channel: str, sequence: int, generated_at: datetime, expires_at: datetime, - repository_base: str, + workspace: Path, public_base_url: str, ) -> dict[str, Any]: - manifests = _discovered_catalog_manifests() - modules: list[dict[str, Any]] = [] - for module in CATALOG_MODULES: - manifest = manifests.get(module.module_id) - module_version = manifest.version if manifest is not None else version - module_tag = f"v{module_version.removeprefix('v')}" - entry: dict[str, Any] = { - "module_id": module.module_id, - "name": module.name, - "description": module.description, - "version": module_version, - "action": "install", - "python_package": module.python_package, - "python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}", - "license_features": [f"module.{module.module_id}"], - "tags": list(module.tags), - } - if module.webui_package: - entry["webui_package"] = module.webui_package - entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}" - manifest_metadata = _manifest_catalog_metadata(manifest) - entry.update(manifest_metadata) - if module.provides_interfaces: - entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces] - if module.requires_interfaces: - entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces] - modules.append(entry) + python_lock = _rows_by_name(package_lock, "python") + webui_lock = _rows_by_repository(package_lock, "webui") + modules: list[dict[str, object]] = [] + core_release: dict[str, object] | None = None + selected_units: list[dict[str, str]] = [] + for package in package_set["python"]: + name = str(package["name"]) + version = str(package["version"]) + repository = str(package["repository"]) + selected_units.append( + { + "repo": repository, + "version": version, + "tag": str(package["tag"]), + "commit": str(package["commit"]), + } + ) + python_artifact = python_lock[name] + webui_artifact = webui_lock.get(repository) + if name == "govoplan-core": + core_release = { + "name": "GovOPlaN Core", + "version": version, + "python_package": name, + "python_ref": _python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())), + "artifact_integrity": { + "python": _artifact_integrity(python_artifact, ref=_python_ref(name, python_artifact, extras=tuple(package.get("extras") or ()))) + }, + } + if webui_artifact is not None: + webui_ref = _artifact_url(webui_artifact) + core_release.update( + { + "webui_package": webui_artifact["name"], + "webui_ref": webui_ref, + } + ) + core_release["artifact_integrity"]["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref) + continue + + entries = synthesize_repository_catalog_entries( + repo=repository, + version=version, + workspace=workspace, + repository_base="git+https://git.add-ideas.de/GovOPlaN", + source_ref=str(package["tag"]), + ) + for entry in entries: + python_ref = _python_ref(name, python_artifact) + entry["python_ref"] = python_ref + entry["source"] = { + "repository": repository, + "tag": package["tag"], + "commit": package["commit"], + } + integrity: dict[str, object] = { + "python": _artifact_integrity(python_artifact, ref=python_ref), + } + if entry.get("webui_package"): + if webui_artifact is None or webui_artifact.get("name") != entry["webui_package"]: + raise ValueError(f"Package lock has no matching WebUI artifact for {repository}.") + webui_ref = _artifact_url(webui_artifact) + entry["webui_ref"] = webui_ref + integrity["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref) + else: + entry.pop("webui_ref", None) + entry["artifact_integrity"] = integrity + modules.append(entry) + + if core_release is None: + raise ValueError("Package set does not contain govoplan-core.") + validate_initial_entry_closure( + catalog_modules=modules, + initial_module_ids={str(item["module_id"]) for item in modules}, + ) + release_version = str(package_set["release_version"]) return { "catalog_version": "1", "channel": channel, @@ -298,97 +197,103 @@ def _catalog_payload( "generated_at": _json_datetime(generated_at), "expires_at": _json_datetime(expires_at), "release": { - "version": version, - "tag": tag, + "version": release_version, + "tag": f"v{release_version}", + "profile": package_set.get("profile", "base"), "catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json", "keyring_url": f"{public_base_url}/catalogs/v1/keyring.json", + "package_set_sha256": package_set["package_set_sha256"], + "package_lock_sha256": package_lock["lock_sha256"], + "selected_units": sorted(selected_units, key=lambda item: item["repo"]), }, - "core_release": { - "name": "GovOPlaN Core", - "version": version, - "python_package": "govoplan-core", - "python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}", - "webui_package": "@govoplan/core-webui", - "webui_ref": f"{repository_base}/govoplan-core.git#{tag}", - }, - "modules": modules, + "core_release": core_release, + "modules": sorted(modules, key=lambda item: str(item["module_id"])), } -def _discovered_catalog_manifests() -> dict[str, ModuleManifest]: - try: - return available_module_manifests(ignore_load_errors=True) - except Exception: - return {} - - -def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]: - if manifest is None: - return {} - payload: dict[str, object] = {} - if manifest.dependencies: - payload["dependencies"] = list(manifest.dependencies) - if manifest.optional_dependencies: - payload["optional_dependencies"] = list(manifest.optional_dependencies) - if manifest.architecture is not None: - payload["architecture"] = manifest.architecture.to_dict() - payload["information_governance"] = manifest.information_governance.to_dict() - if manifest.external_providers: - payload["external_providers"] = [ - declaration.to_dict() - for declaration in manifest.external_providers - ] - if manifest.migration_spec is not None: - payload["migration_safety"] = "requires_review" - payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation." - if manifest.migration_spec.migration_after: - payload["migration_after"] = list(manifest.migration_spec.migration_after) - if manifest.migration_spec.migration_before: - payload["migration_before"] = list(manifest.migration_spec.migration_before) - if manifest.migration_spec.migration_tasks: - tasks: list[dict[str, object]] = [] - for task in manifest.migration_spec.migration_tasks: - task_payload: dict[str, object] = { - "task_id": task.task_id, - "phase": task.phase, - "summary": task.summary, - "task_version": task.task_version, - "safety": task.safety, - "idempotent": task.idempotent, - } - if task.timeout_seconds is not None: - task_payload["timeout_seconds"] = task.timeout_seconds - tasks.append(task_payload) - payload["migration_tasks"] = tasks - if manifest.provides_interfaces: - payload["provides_interfaces"] = [ - {"name": item.name, "version": item.version} - for item in manifest.provides_interfaces - ] - if manifest.requires_interfaces: - requirements: list[dict[str, object]] = [] - for item in manifest.requires_interfaces: - requirement: dict[str, object] = { - "name": item.name, - "optional": item.optional, - } - if item.version_min is not None: - requirement["version_min"] = item.version_min - if item.version_max_exclusive is not None: - requirement["version_max_exclusive"] = item.version_max_exclusive - requirements.append(requirement) - payload["requires_interfaces"] = requirements +def _read_hashed_json(path: Path, *, hash_field: str) -> dict[str, Any]: + payload = json.loads(path.expanduser().read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path} must contain a JSON object.") + expected = payload.get(hash_field) + unsigned = dict(payload) + unsigned.pop(hash_field, None) + if not isinstance(expected, str) or expected != _canonical_sha256(unsigned): + raise ValueError(f"{path} {hash_field} does not match its contents.") return payload +def _validate_release_inputs(package_set: dict[str, Any], package_lock: dict[str, Any], *, core_version: str) -> None: + if package_set.get("schema_version") != "1" or package_lock.get("schema_version") != "1": + raise ValueError("Package set and lock must use schema version 1.") + if package_set.get("release_version") != core_version or package_lock.get("release_version") != core_version: + raise ValueError("Package set and lock release versions must match --version.") + if package_lock.get("package_set_sha256") != package_set.get("package_set_sha256"): + raise ValueError("Package lock does not belong to the selected package set.") + if package_lock.get("profile", "base") != package_set.get("profile", "base"): + raise ValueError("Package set and lock profiles do not match.") + for group in ("python", "webui"): + selected = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_set.get(group, ()) if isinstance(item, dict)} + locked = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_lock.get(group, ()) if isinstance(item, dict)} + if not selected or selected != locked: + raise ValueError(f"Package lock does not contain the exact {group} package set.") + for item in package_lock[group]: + _artifact_url(item) + if SHA256.fullmatch(str(item.get("sha256") or "")) is None: + raise ValueError(f"Package lock has an invalid {group} artifact digest.") + + +def _rows_by_name(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]: + return {str(item["name"]): item for item in payload[group]} + + +def _rows_by_repository(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]: + result: dict[str, dict[str, object]] = {} + for item in payload[group]: + repository = str(item["repository"]) + if repository in result: + raise ValueError(f"Package lock contains multiple {group} artifacts for {repository}.") + result[repository] = item + return result + + +def _artifact_url(artifact: dict[str, object]) -> str: + value = str(artifact.get("url") or "") + parsed = urlsplit(value) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.fragment: + raise ValueError(f"Package artifact has an unsafe download URL: {value!r}.") + return value + + +def _python_ref(name: str, artifact: dict[str, object], *, extras: tuple[object, ...] = ()) -> str: + extra = f"[{','.join(str(item) for item in extras)}]" if extras else "" + return f"{name}{extra} @ {_artifact_url(artifact)}#sha256={artifact['sha256']}" + + +def _artifact_integrity(artifact: dict[str, object], *, ref: str) -> dict[str, object]: + result: dict[str, object] = { + "ref": ref, + "url": _artifact_url(artifact), + "filename": artifact["filename"], + "sha256": artifact["sha256"], + "size": artifact["size"], + "registry_identity": f"{artifact['name']}@{artifact['version']}", + "git_ref": artifact["tag"], + "source_commit": artifact["commit"], + } + if artifact.get("integrity"): + result["integrity"] = artifact["integrity"] + return result + + def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]: key_id, separator, path_text = value.partition("=") if not separator or not key_id.strip() or not path_text.strip(): - raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem") + raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem") path = Path(path_text).expanduser() private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) if not isinstance(private_key, Ed25519PrivateKey): - raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}") + raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}") return key_id.strip(), private_key @@ -396,11 +301,10 @@ def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519Priv signature_payload = dict(payload) signature_payload.pop("signature", None) signature_payload.pop("signatures", None) - signature = private_key.sign(_canonical_bytes(signature_payload)) return { "algorithm": "ed25519", "key_id": key_id, - "value": base64.b64encode(signature).decode("ascii"), + "value": base64.b64encode(private_key.sign(_canonical_bytes(signature_payload))).decode("ascii"), } @@ -413,7 +317,12 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: { "key_id": key_id, "status": "active", - "public_key": _public_key_base64(private_key), + "public_key": base64.b64encode( + private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + ).decode("ascii"), "not_before": generated_at.date().isoformat() + "T00:00:00Z", } for key_id, private_key in signing_keys @@ -421,18 +330,14 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: } -def _public_key_base64(private_key: Ed25519PrivateKey) -> str: - public_bytes = private_key.public_key().public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw, - ) - return base64.b64encode(public_bytes).decode("ascii") - - def _canonical_bytes(payload: object) -> bytes: return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_bytes(payload)).hexdigest() + + def _json_datetime(value: datetime) -> str: return value.astimezone(UTC).isoformat().replace("+00:00", "Z") diff --git a/tools/release/generate-release-package-set.py b/tools/release/generate-release-package-set.py index 3fd894d..3d8db15 100644 --- a/tools/release/generate-release-package-set.py +++ b/tools/release/generate-release-package-set.py @@ -21,6 +21,10 @@ GIT_REQUIREMENT = re.compile( r"(?P[A-Za-z0-9._+!-]+))$" ) LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P[^]]+)\])?$") +EXACT_PACKAGE = re.compile( + r"^(?Pgovoplan(?:-[a-z0-9-]+)?)(?:\[(?P[^]]+)\])?==" + r"(?P[A-Za-z0-9._+!-]+)$" +) class PackageSetError(ValueError): @@ -38,6 +42,17 @@ def build_parser() -> argparse.ArgumentParser: type=Path, default=META_ROOT / "requirements-release.txt", ) + parser.add_argument( + "--profile", + choices=("base", "full"), + default="base", + help="Base runtime roots or every package selected by govoplan[full].", + ) + parser.add_argument( + "--meta-package", + type=Path, + default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml", + ) parser.add_argument("--workspace", type=Path, default=META_ROOT.parent) parser.add_argument("--output", type=Path, required=True) return parser @@ -79,11 +94,46 @@ def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[s return tuple(values) +def parse_meta_package(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]: + project = tomllib.loads(path.read_text(encoding="utf-8")).get("project") + if not isinstance(project, dict): + raise PackageSetError("developer meta-package has no [project] table") + if str(project.get("name") or "") != "govoplan": + raise PackageSetError("developer meta-package has an unexpected project name") + if str(project.get("version") or "").removeprefix("v") != core_version.removeprefix("v"): + raise PackageSetError("developer meta-package version does not match Core") + optional = project.get("optional-dependencies") + full = optional.get("full") if isinstance(optional, dict) else None + dependencies = project.get("dependencies") + if not isinstance(dependencies, list) or not isinstance(full, list): + raise PackageSetError("developer meta-package must declare dependencies and the full extra") + values: list[dict[str, object]] = [] + for raw in (*dependencies, *full): + if not isinstance(raw, str) or (match := EXACT_PACKAGE.fullmatch(raw.strip())) is None: + raise PackageSetError(f"developer meta-package requirement is not exact: {raw!r}") + package = match.group("package") + repository = "govoplan-core" if package == "govoplan-core" else package + values.append( + { + "name": package, + "version": match.group("version"), + "repository": repository, + "extras": _extras(match.group("extras")), + } + ) + names = [str(item["name"]) for item in values] + if names.count("govoplan-core") != 1 or len(names) != len(set(names)): + raise PackageSetError("developer meta-package must contain one Core and unique packages") + return tuple(values) + + def generate_package_set( *, core_version: str, requirements: Path, workspace: Path, + profile: str = "base", + meta_package: Path | None = None, ) -> dict[str, object]: core_version = core_version.removeprefix("v") if VERSION.fullmatch(core_version) is None: @@ -91,7 +141,17 @@ def generate_package_set( python_packages: list[dict[str, object]] = [] webui_packages: list[dict[str, object]] = [] seen_webui: set[str] = set() - for requirement in parse_release_requirements(requirements, core_version=core_version): + if profile not in {"base", "full"}: + raise PackageSetError(f"unsupported release profile: {profile}") + selected = ( + parse_meta_package( + meta_package or META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml", + core_version=core_version, + ) + if profile == "full" + else parse_release_requirements(requirements, core_version=core_version) + ) + for requirement in selected: repository = workspace / str(requirement["repository"]) tag = f"v{requirement['version']}" if not (repository / ".git").is_dir(): @@ -134,6 +194,7 @@ def generate_package_set( payload: dict[str, object] = { "schema_version": "1", "release_version": core_version, + "profile": profile, "registries": { "python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple", "npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/", @@ -181,6 +242,8 @@ def main() -> int: core_version=version, requirements=args.requirements.expanduser().resolve(), workspace=workspace, + profile=args.profile, + meta_package=args.meta_package.expanduser().resolve(), ) except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc: print(f"error: {exc}") diff --git a/tools/release/govoplan_release/catalog_entry_synthesis.py b/tools/release/govoplan_release/catalog_entry_synthesis.py index cbc21b7..13fd9f2 100644 --- a/tools/release/govoplan_release/catalog_entry_synthesis.py +++ b/tools/release/govoplan_release/catalog_entry_synthesis.py @@ -10,6 +10,8 @@ import os from pathlib import Path import subprocess import sys +import tarfile +import tempfile import tomllib from types import ModuleType from typing import Iterator @@ -30,13 +32,14 @@ def synthesize_repository_catalog_entries( version: str, workspace: Path, repository_base: str, + source_ref: str | None = None, ) -> tuple[dict[str, object], ...]: """Build install entries from tagged, preflighted local source metadata. - The caller owns source-tag and worktree provenance checks. This function - accepts no hand-maintained module catalog registry: distribution metadata - identifies the runtime entry point and the runtime ``ModuleManifest`` is - the authoritative dependency/interface/frontend description. + Distribution metadata identifies the runtime entry point and the runtime + ``ModuleManifest`` is the authoritative dependency/interface/frontend + description. When ``source_ref`` is supplied, metadata is read from that + immutable Git tree rather than from the current checkout. """ if os.getenv(_INSPECTION_CHILD) == "1": @@ -45,6 +48,7 @@ def synthesize_repository_catalog_entries( version=version, workspace=workspace, repository_base=repository_base, + source_ref=source_ref, ) command = ( sys.executable, @@ -59,6 +63,8 @@ def synthesize_repository_catalog_entries( "--repository-base", repository_base, ) + if source_ref: + command = (*command, "--source-ref", source_ref) environment = os.environ.copy() environment[_INSPECTION_CHILD] = "1" release_root = str(Path(__file__).resolve().parents[1]) @@ -98,44 +104,46 @@ def synthesize_repository_catalog_entries_in_process( version: str, workspace: Path, repository_base: str, + source_ref: str | None = None, ) -> tuple[dict[str, object], ...]: specs = {item.name: item for item in load_repository_specs(include_website=False)} spec = specs.get(repo) if spec is None: raise ValueError(f"Cannot synthesize {repo}: repository is not registered.") - root = resolve_repo_path(spec, workspace) - project = read_project_metadata(root / "pyproject.toml") - package = required_text(project, "name", source=f"{repo}/pyproject.toml") - project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v") - expected_version = version.removeprefix("v") - if project_version != expected_version: - raise ValueError( - f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}." - ) - description = optional_text(project.get("description")) - entry_points = module_entry_points(project, repo=repo) - entries: list[dict[str, object]] = [] - for declared_module_id, target in sorted(entry_points.items()): - manifest = load_manifest(root=root, target=target, repo=repo) - if manifest.id != declared_module_id: + checkout_root = resolve_repo_path(spec, workspace) + with materialized_source_tree(checkout_root, source_ref=source_ref) as root: + project = read_project_metadata(root / "pyproject.toml") + package = required_text(project, "name", source=f"{repo}/pyproject.toml") + project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v") + expected_version = version.removeprefix("v") + if project_version != expected_version: raise ValueError( - f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}." + f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}." ) - if manifest.version.removeprefix("v") != expected_version: - raise ValueError( - f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}." + description = optional_text(project.get("description")) + entry_points = module_entry_points(project, repo=repo) + entries: list[dict[str, object]] = [] + for declared_module_id, target in sorted(entry_points.items()): + manifest = load_manifest(root=root, target=target, repo=repo) + if manifest.id != declared_module_id: + raise ValueError( + f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}." + ) + if manifest.version.removeprefix("v") != expected_version: + raise ValueError( + f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}." + ) + entry = manifest_catalog_entry( + manifest=manifest, + repo=repo, + package=package, + version=expected_version, + description=description, + root=root, + repository_base=repository_base.rstrip("/"), ) - entry = manifest_catalog_entry( - manifest=manifest, - repo=repo, - package=package, - version=expected_version, - description=description, - root=root, - repository_base=repository_base.rstrip("/"), - ) - entries.append(entry) - return tuple(entries) + entries.append(entry) + return tuple(entries) def manifest_catalog_entry( @@ -156,8 +164,7 @@ def manifest_catalog_entry( "action": "install", "python_package": package, "python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}", - "license_features": [f"module.{manifest.id}"], - "tags": ["official"], + "tags": ["official", "open-source"], } if description: entry["description"] = description @@ -218,6 +225,51 @@ def manifest_catalog_entry( return entry +@contextmanager +def materialized_source_tree(root: Path, *, source_ref: str | None) -> Iterator[Path]: + if not source_ref: + yield root + return + if not (root / ".git").exists(): + raise ValueError(f"Cannot inspect {source_ref!r}: {root} is not a Git checkout.") + with tempfile.TemporaryDirectory(prefix="govoplan-catalog-source-") as value: + temporary = Path(value) + archive_path = temporary / "source.tar" + source_root = temporary / "source" + source_root.mkdir() + result = subprocess.run( + [ + "git", + "-C", + str(root), + "archive", + "--format=tar", + f"--output={archive_path}", + source_ref, + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + timeout=30, + ) + if result.returncode != 0: + detail = result.stderr.strip() or "Git archive failed" + raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: {detail}") + if archive_path.stat().st_size > 256 * 1024 * 1024: + raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive exceeds 256 MiB.") + with tarfile.open(archive_path, mode="r:") as archive: + members = archive.getmembers() + if len(members) > 50_000: + raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive has too many entries.") + for member in members: + path = Path(member.name) + if path.is_absolute() or ".." in path.parts or member.issym() or member.islnk() or member.isdev(): + raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive contains an unsafe entry.") + archive.extractall(source_root, members=members, filter="data") + yield source_root + + def validate_initial_entry_closure( *, catalog_modules: list[object], @@ -367,6 +419,7 @@ def main() -> int: parser.add_argument("--version", required=True) parser.add_argument("--workspace", type=Path, required=True) parser.add_argument("--repository-base", required=True) + parser.add_argument("--source-ref") args = parser.parse_args() try: entries = synthesize_repository_catalog_entries( @@ -374,6 +427,7 @@ def main() -> int: version=args.version, workspace=args.workspace.resolve(), repository_base=args.repository_base, + source_ref=args.source_ref, ) except ValueError as exc: print(str(exc), file=sys.stderr) diff --git a/tools/release/govoplan_release/selective_catalog.py b/tools/release/govoplan_release/selective_catalog.py index 286b964..38294e9 100644 --- a/tools/release/govoplan_release/selective_catalog.py +++ b/tools/release/govoplan_release/selective_catalog.py @@ -158,6 +158,7 @@ def build_selective_catalog_candidate( repository_base=repository_base.rstrip("/"), workspace=workspace, ) + changes.extend(remove_official_license_requirements(candidate)) changes.extend( apply_python_artifact_identities( candidate, @@ -728,6 +729,32 @@ def module_entry_repo(entry: dict[str, Any]) -> str | None: return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None +def remove_official_license_requirements(payload: dict[str, Any]) -> list[CatalogEntryChange]: + """Official open-source modules never require commercial entitlements.""" + + modules = payload.get("modules") + if not isinstance(modules, list): + return [] + changes: list[CatalogEntryChange] = [] + for entry in modules: + if not isinstance(entry, dict): + continue + tags = entry.get("tags") + if not isinstance(tags, list) or "official" not in tags or "license_features" not in entry: + continue + before = entry.pop("license_features") + changes.append( + CatalogEntryChange( + repo=module_entry_repo(entry) or "unknown", + module_id=str(entry.get("module_id") or "") or None, + field="license_features", + before=json.dumps(before, sort_keys=True), + after=None, + ) + ) + return changes + + def apply_python_artifact_identities( payload: dict[str, Any], *, diff --git a/tools/release/install-webui-release-dependencies.sh b/tools/release/install-webui-release-dependencies.sh index 3b7a20a..eb3052c 100644 --- a/tools/release/install-webui-release-dependencies.sh +++ b/tools/release/install-webui-release-dependencies.sh @@ -68,6 +68,7 @@ from __future__ import annotations import hashlib import json +import os from pathlib import Path import sys @@ -91,11 +92,18 @@ for item in lock["webui"]: if item["name"] in rows: raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}") rows[item["name"]] = item +install_all = os.environ.get("GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES", "").strip().lower() in {"1", "true", "yes", "on"} +names = ( + sorted(name for name in rows if name != "@govoplan/core-webui") + if install_all + else [ + line.split("\t", 1)[0] + for line in output.read_text(encoding="utf-8").splitlines() + if line + ] +) requested = [] -for line in output.read_text(encoding="utf-8").splitlines(): - if not line: - continue - name, _source_ref = line.split("\t", 1) +for name in names: row = rows.get(name) if not isinstance(row, dict): raise SystemExit(f"WebUI package lock has no artifact for {name}") diff --git a/tools/release/publish-release-catalog.sh b/tools/release/publish-release-catalog.sh index 223b1a4..8c90ec6 100644 --- a/tools/release/publish-release-catalog.sh +++ b/tools/release/publish-release-catalog.sh @@ -15,6 +15,7 @@ Options: --channel Catalog channel. Defaults to stable. --sequence Monotonic channel sequence. Defaults to UTC timestamp. --expires-days Catalog expiry window. Defaults to 90. + --profile Package composition to publish. Defaults to full. --catalog-signing-key Ed25519 private key. May be repeated for rotation. --core-root govoplan-core checkout. Defaults to ../govoplan-core. @@ -46,6 +47,7 @@ VERSION="" CHANNEL="stable" SEQUENCE="" EXPIRES_DAYS="90" +PROFILE="full" PUBLIC_BASE_URL="https://govoplan.add-ideas.de" REMOTE="origin" BRANCH="" @@ -95,6 +97,11 @@ while [[ $# -gt 0 ]]; do EXPIRES_DAYS="$2" shift 2 ;; + --profile) + [[ $# -ge 2 ]] || fail "missing value for $1" + PROFILE="$2" + shift 2 + ;; --catalog-signing-key) [[ $# -ge 2 ]] || fail "missing value for $1" SIGNING_KEYS+=("$2") @@ -166,14 +173,12 @@ done [[ -n "$VERSION" ]] || fail "--version is required" [[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION" +[[ "$PROFILE" == "base" || "$PROFILE" == "full" ]] || fail "profile must be base or full: $PROFILE" [[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required" [[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT" [[ -d "$WEB_ROOT/.git" ]] || fail "not an addideas-govoplan-website git repo: $WEB_ROOT" command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON" - -if [[ "$BUILD_WEB" -eq 1 ]]; then - command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN" -fi +command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN" if [[ -z "$BRANCH" ]]; then BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)" @@ -183,6 +188,10 @@ fi CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json" KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json" TAG_NAME="catalog-v$VERSION" +TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-catalog.XXXXXXXX")" +PACKAGE_SET="$TEMP_ROOT/release-packages.json" +PACKAGE_LOCK="$TEMP_ROOT/package-artifacts.lock.json" +trap 'rm -rf "$TEMP_ROOT"' EXIT run() { printf '+' @@ -197,6 +206,9 @@ GEN_ARGS=( env "GOVOPLAN_CORE_ROOT=$CORE_ROOT" "$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py" --version "$VERSION" + --package-set "$PACKAGE_SET" + --package-lock "$PACKAGE_LOCK" + --workspace "$PARENT" --channel "$CHANNEL" --expires-days "$EXPIRES_DAYS" --catalog-output "$CATALOG_PATH" @@ -210,6 +222,18 @@ for signing_key in "${SIGNING_KEYS[@]}"; do GEN_ARGS+=(--catalog-signing-key "$signing_key") done +run "$PYTHON" "$META_ROOT/tools/release/generate-release-package-set.py" \ + --version "$VERSION" \ + --profile "$PROFILE" \ + --workspace "$PARENT" \ + --output "$PACKAGE_SET" +run "$PYTHON" "$META_ROOT/tools/release/resolve-package-artifacts.py" \ + --package-set "$PACKAGE_SET" \ + --wheelhouse "$TEMP_ROOT/wheelhouse" \ + --webui-packages "$TEMP_ROOT/webui-packages" \ + --lock-output "$PACKAGE_LOCK" \ + --python "$PYTHON" \ + --npm "$NPM_BIN" run "${GEN_ARGS[@]}" if [[ "$DRY_RUN" -eq 0 ]]; then diff --git a/tools/release/resolve-package-artifacts.py b/tools/release/resolve-package-artifacts.py index 6470e44..17ed30b 100644 --- a/tools/release/resolve-package-artifacts.py +++ b/tools/release/resolve-package-artifacts.py @@ -64,17 +64,26 @@ def resolve(args: argparse.Namespace) -> dict[str, object]: python=args.python, index_url=str(package_set["registries"]["python"]), ) - _download_webui( + webui_registry_metadata = _download_webui( packages=tuple(package_set["webui"]), destination=webui, npm=args.npm, registry=str(package_set["registries"]["npm"]), ) - python_rows = _verify_wheels(tuple(package_set["python"]), wheels) - webui_rows = _verify_webui(tuple(package_set["webui"]), webui) + python_rows = _verify_wheels( + tuple(package_set["python"]), + wheels, + registry=str(package_set["registries"]["python"]), + ) + webui_rows = _verify_webui( + tuple(package_set["webui"]), + webui, + registry_metadata=webui_registry_metadata, + ) lock: dict[str, object] = { "schema_version": "1", "release_version": package_set["release_version"], + "profile": package_set.get("profile", "base"), "package_set_sha256": package_set["package_set_sha256"], "registries": package_set["registries"], "python": python_rows, @@ -169,7 +178,7 @@ def _download_wheels( def _download_webui( *, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str -) -> None: +) -> dict[str, dict[str, str]]: environment = dict(os.environ) npmrc: tempfile.NamedTemporaryFile[bytes] | None = None token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "") @@ -182,7 +191,31 @@ def _download_webui( os.chmod(npmrc.name, 0o600) environment["NPM_CONFIG_USERCONFIG"] = npmrc.name try: + metadata: dict[str, dict[str, str]] = {} for item in packages: + view = subprocess.run( + [npm, "view", f"{item['name']}@{item['version']}", "dist", "--json", "--registry", registry], + check=True, + env=environment, + text=True, + stdout=subprocess.PIPE, + ) + dist = json.loads(view.stdout) + if not isinstance(dist, dict): + raise PackageArtifactError(f"npm registry returned no distribution metadata for {item['name']}") + tarball = dist.get("tarball") + integrity = dist.get("integrity") + parsed = urlsplit(str(tarball or "")) + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.username + or parsed.password + or not isinstance(integrity, str) + or not integrity.startswith("sha512-") + ): + raise PackageArtifactError(f"npm registry returned unsafe distribution metadata for {item['name']}") + metadata[str(item["name"])] = {"url": str(tarball), "integrity": integrity} subprocess.run( [npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry], check=True, @@ -191,9 +224,15 @@ def _download_webui( finally: if npmrc is not None: Path(npmrc.name).unlink(missing_ok=True) + return metadata -def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: +def _verify_wheels( + packages: tuple[dict[str, object], ...], + root: Path, + *, + registry: str | None = None, +) -> list[dict[str, object]]: expected = {_normalize(str(item["name"])): item for item in packages} rows: list[dict[str, object]] = [] seen: set[str] = set() @@ -204,13 +243,21 @@ def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[ if package is None or identity["version"] != package["version"] or name in seen: raise PackageArtifactError(f"unexpected wheel artifact: {path.name}") seen.add(name) - rows.append(_artifact_row(path, package)) + row = _artifact_row(path, package) + if registry: + row["url"] = _python_artifact_url(registry, package=package, filename=path.name) + rows.append(row) if seen != set(expected): raise PackageArtifactError("registry did not return every selected Python wheel") return sorted(rows, key=lambda item: str(item["name"])) -def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: +def _verify_webui( + packages: tuple[dict[str, object], ...], + root: Path, + *, + registry_metadata: dict[str, dict[str, str]] | None = None, +) -> list[dict[str, object]]: expected = {str(item["name"]): item for item in packages} rows: list[dict[str, object]] = [] seen: set[str] = set() @@ -222,7 +269,13 @@ def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[d raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}") seen.add(name) row = _artifact_row(path, package) - row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii") + integrity = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii") + row["integrity"] = integrity + metadata = (registry_metadata or {}).get(name) + if metadata: + if metadata.get("integrity") != integrity: + raise PackageArtifactError(f"npm registry integrity does not match downloaded package: {name}") + row["url"] = metadata["url"] rows.append(row) if seen != set(expected): raise PackageArtifactError("registry did not return every selected WebUI package") @@ -283,6 +336,25 @@ def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]: return row +def _python_artifact_url( + registry: str, + *, + package: dict[str, object], + filename: str, +) -> str: + parsed = urlsplit(registry.rstrip("/")) + path = parsed.path.rstrip("/") + if not path.endswith("/simple"): + raise PackageArtifactError("Python registry URL must end in /simple to derive immutable artifacts") + artifact_path = ( + f"{path.removesuffix('/simple')}/files/" + f"{quote(str(package['name']), safe='')}/" + f"{quote(str(package['version']), safe='')}/" + f"{quote(filename, safe='')}" + ) + return urlunsplit((parsed.scheme, parsed.netloc, artifact_path, "", "")) + + def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None: lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"] for row in rows: