Publish complete signed module catalogs
Dependency Audit / dependency-audit (push) Successful in 1m46s
Deployment Installer / deployment-installer (push) Successful in 5s
Security Audit / security-audit (push) Successful in 11m49s

This commit is contained in:
2026-08-06 21:13:33 +02:00
parent f3cfd1bccc
commit 09046e6e62
17 changed files with 666 additions and 394 deletions
+1 -1
View File
@@ -57,4 +57,4 @@ DEV_MAILBOX_API_ENABLED=false
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable
@@ -101,6 +101,8 @@ jobs:
mkdir -p runtime-output mkdir -p runtime-output
git rev-parse "v$VERSION^{commit}" > runtime-output/release-source-commit git rev-parse "v$VERSION^{commit}" > runtime-output/release-source-commit
grep -Eq '^[0-9a-f]{40}$' runtime-output/release-source-commit grep -Eq '^[0-9a-f]{40}$' runtime-output/release-source-commit
git show "v$VERSION:requirements-release.txt" > runtime-output/requirements-release.source.txt
git show "v$VERSION:packages/govoplan-meta/pyproject.toml" > runtime-output/govoplan-meta.source.toml
- name: Use HTTPS for GovOPlaN repositories - name: Use HTTPS for GovOPlaN repositories
run: | run: |
git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan" git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan"
@@ -119,6 +121,9 @@ jobs:
.runtime-build/bin/python -m pip install --upgrade pip cryptography .runtime-build/bin/python -m pip install --upgrade pip cryptography
.runtime-build/bin/python tools/release/generate-release-package-set.py \ .runtime-build/bin/python tools/release/generate-release-package-set.py \
--version "$VERSION" \ --version "$VERSION" \
--profile full \
--requirements runtime-output/requirements-release.source.txt \
--meta-package runtime-output/govoplan-meta.source.toml \
--output runtime-output/release-packages.json --output runtime-output/release-packages.json
.runtime-build/bin/python tools/release/resolve-package-artifacts.py \ .runtime-build/bin/python tools/release/resolve-package-artifacts.py \
--package-set runtime-output/release-packages.json \ --package-set runtime-output/release-packages.json \
@@ -130,6 +135,7 @@ jobs:
PYTHON="$PWD/.runtime-build/bin/python" \ PYTHON="$PWD/.runtime-build/bin/python" \
GOVOPLAN_WEBUI_PACKAGE_LOCK="$PWD/runtime-output/package-artifacts.lock.json" \ GOVOPLAN_WEBUI_PACKAGE_LOCK="$PWD/runtime-output/package-artifacts.lock.json" \
GOVOPLAN_WEBUI_PACKAGE_DIR="$PWD/runtime-output/webui-packages" \ GOVOPLAN_WEBUI_PACKAGE_DIR="$PWD/runtime-output/webui-packages" \
GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES=true \
bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
npm --prefix ../govoplan-core/webui run build npm --prefix ../govoplan-core/webui run build
.runtime-build/bin/python tools/release/prepare-runtime-context.py \ .runtime-build/bin/python tools/release/prepare-runtime-context.py \
+1 -1
View File
@@ -217,7 +217,7 @@ sequence are documented in the
The selected Campaign-to-Postbox-to-data-to-collaboration implementation path, The selected Campaign-to-Postbox-to-data-to-collaboration implementation path,
including stage gates and shared documentation expectations, is in the including stage gates and shared documentation expectations, is in the
[Reference Journey Program](docs/REFERENCE_JOURNEY_PROGRAM.md). [Reference Journey Program](docs/REFERENCE_JOURNEY_PROGRAM.md).
The administrator journey from Core-only bootstrap through online module The administrator journey from a Core-baseline bootstrap through online module
installation, scale-out, and reversible environment promotion is defined in installation, scale-out, and reversible environment promotion is defined in
[System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md). [System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md).
The corresponding host deployment compiler, managed/external component choices, The corresponding host deployment compiler, managed/external component choices,
@@ -267,7 +267,7 @@ inspection is authorized.
The complete installation and lifecycle journey is specified in the The complete installation and lifecycle journey is specified in the
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md): [System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
one-command Core-only bootstrap, signed online module installation and updates, one-command Core-baseline bootstrap, signed online module installation and updates,
stateless scale-out, versioned configuration transfer, undo, and reproducible stateless scale-out, versioned configuration transfer, undo, and reproducible
environment-promotion recipes. environment-promotion recipes.
@@ -977,7 +977,8 @@ Priorities:
1. Deliver the first slices of the 1. Deliver the first slices of the
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md): [System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
a verified Core-only distribution, first-run control plane, read-only online a verified full-package distribution with only the Core baseline active,
first-run control plane, read-only online
module directory, and durable plan/confirm/install progress. module directory, and durable plan/confirm/install progress.
2. Pin and publish a compatible Core/WebUI/module composition and first 2. Pin and publish a compatible Core/WebUI/module composition and first
reference configuration package. reference configuration package.
+16
View File
@@ -38,6 +38,22 @@ The K3s VM lab has two modes over the same Kubernetes profile:
- `acceptance` requires independently controlled worker failure domains and can - `acceptance` requires independently controlled worker failure domains and can
contribute target evidence. contribute target evidence.
## Module composition and availability
Official immutable API and WebUI images carry the verified `full` package
profile. This is package availability, not runtime activation and not a license
or tenant entitlement. The signed distribution manifest records the complete
package composition; the desired module graph selects which installed modules
are active; tenant module policy applies unavailable/available/forced ceilings;
and Views/Policy control group and user presentation.
Local and single-host profiles may use the supervised installer to download a
signed catalog artifact into a private digest cache and mutate the local package
environment during maintenance. A multi-host/shared-state profile must never
change one replica in place. Its Admin install plan is a composition request:
publish and roll out a new signed image whose package lock contains the target,
then activate the module graph after all replicas report the same composition.
## Component Choices ## Component Choices
The installer may manage a component where its bounded profile is appropriate, The installer may manage a component where its bounded profile is appropriate,
+58 -13
View File
@@ -136,31 +136,38 @@ allow the same least-privilege credential to serve every module workflow.
## Exact release consumption ## Exact release consumption
`tools/release/generate-release-package-set.py` translates the reviewed Git `tools/release/generate-release-package-set.py` supports two explicit package
source refs in `requirements-release.txt` into an exact registry package set. profiles. `base` translates the reviewed roots in `requirements-release.txt`;
It resolves each version tag to its commit and verifies the package metadata in `full` reads the exact `govoplan[full]` dependency set from the developer
that tag. meta-package. Both profiles resolve every version tag to its commit and verify
the package metadata from that exact Git tree. The official module directory
and immutable runtime distribution use `full`, so every publicly released
module can be discovered without rebuilding the application image.
`tools/release/resolve-package-artifacts.py` then downloads exactly those wheel `tools/release/resolve-package-artifacts.py` then downloads exactly those wheel
and WebUI versions from Gitea. It reads the identity embedded in every wheel and and WebUI versions from Gitea. It reads the identity embedded in every wheel and
npm tarball, rejects missing, duplicate, unexpected, or oversized artifacts, npm tarball, rejects missing, duplicate, unexpected, or oversized artifacts,
and writes `package-artifacts.lock.json` with SHA-256 values and npm integrity and writes `package-artifacts.lock.json` with credential-free HTTPS download
values. Credentials are accepted only through environment variables and are URLs, SHA-256 values, and npm registry integrity values. The resolver verifies
that the bytes downloaded by `npm pack` match the registry's own integrity
record. Credentials are accepted only through environment variables and are
never written to the lock. Python resolution ignores ambient pip configuration never written to the lock. Python resolution ignores ambient pip configuration
and extra indexes for GovOPlaN roots, preventing an internal package name from and extra indexes for GovOPlaN roots, preventing an internal package name from
being selected from an undeclared registry. being selected from an undeclared registry.
The runtime distribution workflow uses the verified wheelhouse directly and The runtime distribution workflow uses the verified full-profile wheelhouse
installs module WebUI tarballs only after matching them to the lock. It publishes directly and installs every selected module WebUI tarball only after matching
the package set, package lock, and hash-locked requirements as release assets. it to the lock. It publishes the package set, package lock, and hash-locked
requirements as release assets.
The WebUI installer receives the absolute runtime-build interpreter path so its The WebUI installer receives the absolute runtime-build interpreter path so its
directory changes cannot escape the isolated release environment. directory changes cannot escape the isolated release environment.
Gitea 1.24 dispatches this workflow from a branch, but that branch is only the Gitea 1.24 dispatches this workflow from a branch, but that branch is only the
workflow implementation. The job fetches and peels the protected `v<version>` workflow implementation. The job fetches and peels the protected `v<version>`
tag explicitly, then binds both the signed distribution source and the Gitea tag explicitly and materializes both `requirements-release.txt` and the
release assets to that exact commit. A post-tag workflow repair can therefore developer meta-package from that Git tree. It then binds the signed distribution
retry publication without relabelling the later branch commit as released source and Gitea release assets to the same exact commit. A post-tag workflow
source. repair can therefore retry publication without changing the released package
composition or relabelling the later branch commit as released source.
The package-lock SHA-256 is part of the signed distribution manifest. Runtime The package-lock SHA-256 is part of the signed distribution manifest. Runtime
finalization also requires the lock's package versions and hashes to match the finalization also requires the lock's package versions and hashes to match the
wheel composition embedded in the images. OCI assembly remains network-free wheel composition embedded in the images. OCI assembly remains network-free
@@ -170,6 +177,44 @@ The source refs remain in the module catalog for source provenance and release
planning. Production installation consumes the signed runtime images rather planning. Production installation consumes the signed runtime images rather
than invoking `pip`, `npm`, or Git on the target host. than invoking `pip`, `npm`, or Git on the target host.
## Public module directory
`tools/release/publish-release-catalog.sh` resolves the selected package set and
registry lock before it creates a catalog. Catalog entries are synthesized from
the exact tagged module manifests, never from a hand-maintained module list or
the current workspace. Each entry binds its Python wheel and optional WebUI
tarball to the registry URL, filename, size, SHA-256, package identity, source
tag, and source commit before the complete catalog is signed.
Official GovOPlaN modules are open-source directory entries and do not require
license entitlements. The generic `license_features` contract remains available
for third-party package directories, support/configuration packages, or future
deployment-specific presets. A catalog entry is gated only when that entry
explicitly declares such features.
Core carries the public stable catalog URL and its independently pinned trust
anchor. In the absence of an operator-configured catalog, Admin discovers the
official directory automatically. Selecting an entry creates a reviewed
install/update plan; the trusted installer downloads the exact signed artifacts
into a private digest cache, verifies size and hash, and installs only from that
cache. A saved plan is rejected if any package ref, artifact identity, catalog
channel, sequence, or signing-key identity differs from the currently validated
catalog.
Package lifecycle and availability are intentionally separate:
- install, update, and uninstall change the instance-wide package composition;
- enable and disable change the active instance runtime graph;
- tenant module entitlements define unavailable, available, and forced modules;
- group/user presentation is governed through Views and Policy; and
- enabling a capability module does not opt data into that capability.
Single-process or single-host installations may execute a supervised package
plan locally. Shared-state and Kubernetes profiles reject node-local package
mutation: operators compose and roll out a new signed full-profile runtime image
instead. This prevents replicas from drifting while retaining the same Admin
catalog and preflight experience.
## Developer meta-package ## Developer meta-package
`packages/govoplan-meta` builds the optional `govoplan` package. Its default `packages/govoplan-meta` builds the optional `govoplan` package. Its default
@@ -4,7 +4,8 @@
> As a system administrator, I can execute one shell command that downloads a > As a system administrator, I can execute one shell command that downloads a
> verified GovOPlaN distribution and starts a completely configured Core control > verified GovOPlaN distribution and starts a completely configured Core control
> plane without optional modules. In the WebUI I can browse compatible signed > plane with the official package directory available but only the protected
> baseline active. In the WebUI I can browse compatible signed
> module releases, select the modules for this installation, and follow every > module releases, select the modules for this installation, and follow every
> download, validation, migration, installation, activation, and health-check > download, validation, migration, installation, activation, and health-check
> step. When an update is available, I can review its impact and confirm it. > step. When an update is available, I can review its impact and confirm it.
@@ -25,7 +26,8 @@ The canonical backlog item is
- **Core control plane:** the smallest bootable distribution: Core API, Core - **Core control plane:** the smallest bootable distribution: Core API, Core
WebUI, PostgreSQL, Redis, installer worker, migration runner, and durable WebUI, PostgreSQL, Redis, installer worker, migration runner, and durable
storage configuration. No optional GovOPlaN module package is installed. storage configuration. An immutable image may carry the full verified package
profile, but optional modules are not active or tenant-entitled by implication.
- **Bootstrap administrator:** a single-use, time-limited installation identity - **Bootstrap administrator:** a single-use, time-limited installation identity
that may access only first-run and module-lifecycle functions. It is retired that may access only first-run and module-lifecycle functions. It is retired
when the selected identity/access configuration becomes healthy. when the selected identity/access configuration becomes healthy.
@@ -55,7 +57,9 @@ The canonical backlog item is
5. It prints the local URL and one-time bootstrap credential. Re-running the 5. It prints the local URL and one-time bootstrap credential. Re-running the
command is idempotent and shows or repairs the existing installation rather command is idempotent and shows or repairs the existing installation rather
than creating another identity or database. than creating another identity or database.
6. No optional module is installed or enabled at this point. 6. Only the protected baseline is enabled. Installed package availability does
not grant permissions, tenant entitlement, View visibility, or capability
opt-in.
### Module selection, installation, and update ### Module selection, installation, and update
@@ -160,19 +164,22 @@ Implementation status as of the current source tree:
ledger and deployment operation journal. Automatic database backup and broad ledger and deployment operation journal. Automatic database backup and broad
adoption by module-owned external effects remain open work. adoption by module-owned external effects remain open work.
1. **Reproducible Core-only distribution.** Publish pinned multi-architecture 1. **Reproducible Core-baseline distribution.** Publish pinned multi-architecture
images, signed distribution manifest, Core-only Compose profile, bootstrap full-package images, signed distribution manifest, Core-baseline Compose profile, bootstrap
preflight, generated secrets, readiness, and idempotent rerun/repair. preflight, generated secrets, readiness, and idempotent rerun/repair.
2. **First-run control plane.** Add the restricted bootstrap administrator, 2. **First-run control plane.** Add the restricted bootstrap administrator,
one-time enrollment, initial catalog/keyring configuration, and retirement one-time enrollment, initial catalog/keyring configuration, and retirement
after durable administrator access is established. after durable administrator access is established.
3. **Read-only online module directory.** Move the existing catalog and module 3. **Read-only online module directory (implemented foundation).** Admin falls
directory contracts into the installed Core WebUI with compatibility, back to the signed public stable directory, presents installed/update state,
provenance, release-note, and update-state presentation. compatibility and provenance, and retains operator-configured catalogs as an
4. **Durable module plan and install.** Reuse the existing installer queue, explicit override.
locks, signed-package validator, rollback drill, and run evidence behind a 4. **Durable module plan and install (implemented local boundary).** Catalog
plan/confirm/progress UI. Add initial catalog-entry synthesis and artifact selection creates a reviewed plan; the installer queue, lock, preflight,
acquisition where the current release console still assumes local sources. maintenance gate, digest-verified artifact cache, rollback drill, and run
evidence remain separate from the API process. Shared deployments convert
the same intent into a new immutable release composition instead of mutating
one replica.
5. **Safe module update.** Add drain/maintenance coordination, backup gate, 5. **Safe module update.** Add drain/maintenance coordination, backup gate,
migration compatibility window, reconnectable progress, health verification, migration compatibility window, reconnectable progress, health verification,
retry/recovery, and update notification. retry/recovery, and update notification.
@@ -194,7 +201,8 @@ Implementation status as of the current source tree:
## Explicit non-goals for the first distribution slice ## Explicit non-goals for the first distribution slice
- Shipping optional modules in the Core image. - Activating, tenant-entitling, or exposing optional modules merely because the
immutable image carries their verified packages.
- Exporting secrets or production business data with configuration. - Exporting secrets or production business data with configuration.
- Pretending every schema migration can be reversed automatically. - Pretending every schema migration can be reversed automatically.
- Building a proprietary orchestrator instead of supporting Compose and a - Building a proprietary orchestrator instead of supporting Compose and a
+2 -2
View File
@@ -64,7 +64,7 @@ full = [
"govoplan-procurement==0.1.18", "govoplan-procurement==0.1.18",
"govoplan-projects==0.1.18", "govoplan-projects==0.1.18",
"govoplan-quick-access==0.1.18", "govoplan-quick-access==0.1.18",
"govoplan-records==0.1.18", "govoplan-records==0.1.19",
"govoplan-reporting==0.1.18", "govoplan-reporting==0.1.18",
"govoplan-resources==0.1.18", "govoplan-resources==0.1.18",
"govoplan-rest==0.1.18", "govoplan-rest==0.1.18",
@@ -73,7 +73,7 @@ full = [
"govoplan-search==0.1.18", "govoplan-search==0.1.18",
"govoplan-services==0.1.18", "govoplan-services==0.1.18",
"govoplan-soap==0.1.18", "govoplan-soap==0.1.18",
"govoplan-tasks==0.1.18", "govoplan-tasks==0.1.19",
"govoplan-templates==0.1.18", "govoplan-templates==0.1.18",
"govoplan-tickets==0.1.18", "govoplan-tickets==0.1.18",
"govoplan-transparency==0.1.18", "govoplan-transparency==0.1.18",
+43
View File
@@ -46,6 +46,7 @@ class PackageRegistryReleaseTests(unittest.TestCase):
) )
self.assertEqual("1", payload["schema_version"]) self.assertEqual("1", payload["schema_version"])
self.assertEqual("base", payload["profile"])
self.assertEqual("govoplan-core", payload["python"][0]["name"]) self.assertEqual("govoplan-core", payload["python"][0]["name"])
self.assertIn( self.assertIn(
"@govoplan/core-webui", "@govoplan/core-webui",
@@ -55,6 +56,43 @@ class PackageRegistryReleaseTests(unittest.TestCase):
digest = unsigned.pop("package_set_sha256") digest = unsigned.pop("package_set_sha256")
self.assertEqual(ARTIFACTS._canonical_sha256(unsigned), digest) self.assertEqual(ARTIFACTS._canonical_sha256(unsigned), digest)
def test_full_profile_is_derived_from_the_developer_meta_package(self) -> None:
selected = PACKAGE_SET.parse_meta_package(
ROOT / "packages/govoplan-meta/pyproject.toml",
core_version="0.1.18",
)
by_name = {item["name"]: item for item in selected}
self.assertIn("govoplan-core", by_name)
self.assertIn("govoplan-records", by_name)
self.assertEqual("0.1.19", by_name["govoplan-tasks"]["version"])
payload = PACKAGE_SET.generate_package_set(
core_version="0.1.18",
requirements=ROOT / "requirements-release.txt",
workspace=ROOT.parent,
profile="full",
meta_package=ROOT / "packages/govoplan-meta/pyproject.toml",
)
self.assertEqual("full", payload["profile"])
self.assertEqual(len(selected), len(payload["python"]))
self.assertIn(
"@govoplan/records-webui",
{item["name"] for item in payload["webui"]},
)
def test_python_registry_artifact_url_is_immutable_and_credential_free(self) -> None:
url = ARTIFACTS._python_artifact_url(
"https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
package={"name": "govoplan-files", "version": "0.1.18"},
filename="govoplan_files-0.1.18-py3-none-any.whl",
)
self.assertEqual(
"https://git.add-ideas.de/api/packages/GovOPlaN/pypi/files/govoplan-files/0.1.18/govoplan_files-0.1.18-py3-none-any.whl",
url,
)
def test_wheel_and_webui_artifacts_are_verified_by_embedded_identity(self) -> None: def test_wheel_and_webui_artifacts_are_verified_by_embedded_identity(self) -> None:
with tempfile.TemporaryDirectory(prefix="govoplan-package-artifacts-") as value: with tempfile.TemporaryDirectory(prefix="govoplan-package-artifacts-") as value:
root = Path(value) root = Path(value)
@@ -138,6 +176,11 @@ class PackageRegistryReleaseTests(unittest.TestCase):
) )
self.assertIn("resolve-package-artifacts.py", workflow) self.assertIn("resolve-package-artifacts.py", workflow)
self.assertIn("--profile full", workflow)
self.assertIn('git show "v$VERSION:requirements-release.txt"', workflow)
self.assertIn('git show "v$VERSION:packages/govoplan-meta/pyproject.toml"', workflow)
self.assertIn("--meta-package runtime-output/govoplan-meta.source.toml", workflow)
self.assertIn("GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES=true", workflow)
self.assertIn("package-artifacts.lock.json", workflow) self.assertIn("package-artifacts.lock.json", workflow)
self.assertIn( self.assertIn(
"--package-lock runtime-output/package-artifacts.lock.json", "--package-lock runtime-output/package-artifacts.lock.json",
+2 -2
View File
@@ -95,10 +95,10 @@ class ReleaseEntrypointGateTests(unittest.TestCase):
) )
self.assertIn('cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"', script) self.assertIn('cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"', script)
def test_source_catalog_generator_enforces_explicit_repo_versions(self) -> None: def test_catalog_generator_validates_registry_package_set_before_writing(self) -> None:
script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text() script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text()
gate = script.index("selected_repository_version_issues(") gate = script.index("_validate_release_inputs(package_set, package_lock")
write = script.index("output.write_text(") write = script.index("output.write_text(")
self.assertLess(gate, write) self.assertLess(gate, write)
+212 -307
View File
@@ -1,187 +1,44 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Generate and sign a GovOPlaN module package release catalog.""" """Generate a signed registry-backed GovOPlaN module package catalog."""
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import base64 import base64
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
import hashlib
import json import json
import os import os
from pathlib import Path from pathlib import Path
import re
import sys import sys
from typing import Any from typing import Any
from urllib.parse import urlsplit
from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
META_ROOT = Path(__file__).resolve().parents[2] META_ROOT = Path(__file__).resolve().parents[2]
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve() CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
sys.path.insert(0, str(CORE_ROOT / "src")) sys.path.insert(0, str(CORE_ROOT / "src"))
sys.path.insert(0, str(META_ROOT / "tools" / "release")) sys.path.insert(0, str(META_ROOT / "tools" / "release"))
from govoplan_core.core.modules import ModuleManifest # noqa: E402 from govoplan_release.catalog_entry_synthesis import ( # noqa: E402
from govoplan_core.server.registry import available_module_manifests # noqa: E402 synthesize_repository_catalog_entries,
from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402 validate_initial_entry_closure,
GITEA_BASE = "git+ssh://git@git.add-ideas.de/GovOPlaN"
@dataclass(frozen=True, slots=True)
class CatalogModule:
module_id: str
repo: str
python_package: str
name: str
description: str
tags: tuple[str, ...]
webui_package: str | None = None
provides_interfaces: tuple[dict[str, object], ...] = ()
requires_interfaces: tuple[dict[str, object], ...] = ()
CATALOG_MODULES = (
CatalogModule(
module_id="tenancy",
repo="govoplan-tenancy",
python_package="govoplan-tenancy",
name="Tenancy",
description="Tenant registry, tenant settings, and tenant resolution platform module.",
tags=("official", "platform-module"),
webui_package="@govoplan/tenancy-webui",
),
CatalogModule(
module_id="organizations",
repo="govoplan-organizations",
python_package="govoplan-organizations",
name="Organizations",
description="Organization units, functions, and account-held function assignments.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="identity",
repo="govoplan-identity",
python_package="govoplan-identity",
name="Identity",
description="Canonical identities and links between identities and platform accounts.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="access",
repo="govoplan-access",
python_package="govoplan-access",
name="Access",
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
tags=("official", "platform-module"),
webui_package="@govoplan/access-webui",
),
CatalogModule(
module_id="admin",
repo="govoplan-admin",
python_package="govoplan-admin",
name="Admin",
description="System settings, governance templates, module management, and admin shell contributions.",
tags=("official", "platform-module"),
webui_package="@govoplan/admin-webui",
),
CatalogModule(
module_id="policy",
repo="govoplan-policy",
python_package="govoplan-policy",
name="Policy",
description="Policy and governance capability module.",
tags=("official", "platform-module"),
webui_package="@govoplan/policy-webui",
),
CatalogModule(
module_id="audit",
repo="govoplan-audit",
python_package="govoplan-audit",
name="Audit",
description="Audit-log storage and audit administration routes.",
tags=("official", "platform-module"),
webui_package="@govoplan/audit-webui",
),
CatalogModule(
module_id="dashboard",
repo="govoplan-dashboard",
python_package="govoplan-dashboard",
name="Dashboard",
description="Configurable user home assembled from module-provided dashboard widgets.",
tags=("official", "platform-module"),
webui_package="@govoplan/dashboard-webui",
),
CatalogModule(
module_id="addresses",
repo="govoplan-addresses",
python_package="govoplan-addresses",
name="Addresses",
description="Reusable address directories, recipient sources, consent metadata, and address quality workflows.",
tags=("official", "business-module"),
webui_package="@govoplan/addresses-webui",
),
CatalogModule(
module_id="files",
repo="govoplan-files",
python_package="govoplan-files",
name="Files",
description="Managed file spaces and campaign attachment integration.",
tags=("official", "service-module"),
webui_package="@govoplan/files-webui",
),
CatalogModule(
module_id="mail",
repo="govoplan-mail",
python_package="govoplan-mail",
name="Mail",
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
tags=("official", "service-module"),
webui_package="@govoplan/mail-webui",
),
CatalogModule(
module_id="campaigns",
repo="govoplan-campaign",
python_package="govoplan-campaign",
name="Campaigns",
description="Campaign authoring, validation, queueing, delivery control, and reports.",
tags=("official", "business-module"),
webui_package="@govoplan/campaign-webui",
),
CatalogModule(
module_id="calendar",
repo="govoplan-calendar",
python_package="govoplan-calendar",
name="Calendar",
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
tags=("official", "service-module"),
webui_package="@govoplan/calendar-webui",
),
CatalogModule(
module_id="docs",
repo="govoplan-docs",
python_package="govoplan-docs",
name="Docs",
description="Configured-system documentation and evidence-aware help surfaces.",
tags=("official", "platform-module"),
webui_package="@govoplan/docs-webui",
),
CatalogModule(
module_id="ops",
repo="govoplan-ops",
python_package="govoplan-ops",
name="Ops",
description="Runtime health, deployment profile, worker split, and sizing visibility.",
tags=("official", "platform-module"),
webui_package="@govoplan/ops-webui",
),
) )
SHA256 = re.compile(r"^[0-9a-f]{64}$")
def main() -> int: def main() -> int:
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.") parser.add_argument("--version", required=True, help="Core release version, without leading v.")
parser.add_argument("--package-set", type=Path, required=True)
parser.add_argument("--package-lock", type=Path, required=True)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument("--channel", default="stable") parser.add_argument("--channel", default="stable")
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.") parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
parser.add_argument("--expires-days", type=int, default=90) parser.add_argument("--expires-days", type=int, default=90)
@@ -195,41 +52,33 @@ def main() -> int:
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.", help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
) )
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
parser.add_argument("--repository-base", default=GITEA_BASE)
args = parser.parse_args() args = parser.parse_args()
version = args.version.removeprefix("v") try:
version_issues = selected_repository_version_issues( version = args.version.removeprefix("v")
repo_versions={ package_set = _read_hashed_json(args.package_set, hash_field="package_set_sha256")
"govoplan-core": version, package_lock = _read_hashed_json(args.package_lock, hash_field="lock_sha256")
**{module.repo: version for module in CATALOG_MODULES}, _validate_release_inputs(package_set, package_lock, core_version=version)
}, signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
workspace=CORE_ROOT.parent, generated_at = datetime.now(tz=UTC)
) sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
if version_issues: catalog = _catalog_payload(
details = "; ".join( package_set=package_set,
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})" package_lock=package_lock,
for issue in version_issues channel=args.channel,
sequence=sequence,
generated_at=generated_at,
expires_at=generated_at + timedelta(days=args.expires_days),
workspace=args.workspace.expanduser().resolve(),
public_base_url=args.public_base_url.rstrip("/"),
) )
parser.error(f"version alignment gate failed: {details}") if signing_keys:
tag = f"v{version}" catalog["signatures"] = [
generated_at = datetime.now(tz=UTC) _signature(catalog, key_id=key_id, private_key=private_key)
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M")) for key_id, private_key in signing_keys
expires_at = generated_at + timedelta(days=args.expires_days) ]
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key] except (KeyError, OSError, ValueError, json.JSONDecodeError) as exc:
parser.error(str(exc))
catalog = _catalog_payload(
version=version,
tag=tag,
channel=args.channel,
sequence=sequence,
generated_at=generated_at,
expires_at=expires_at,
repository_base=args.repository_base.rstrip("/"),
public_base_url=args.public_base_url.rstrip("/"),
)
if signing_keys:
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
output = args.catalog_output.expanduser() output = args.catalog_output.expanduser()
output.parent.mkdir(parents=True, exist_ok=True) output.parent.mkdir(parents=True, exist_ok=True)
@@ -249,48 +98,98 @@ def main() -> int:
print(f"channel={args.channel}") print(f"channel={args.channel}")
print(f"sequence={sequence}") print(f"sequence={sequence}")
print(f"version={version}") print(f"version={version}")
print(f"profile={package_set.get('profile', 'base')}")
return 0 return 0
def _catalog_payload( def _catalog_payload(
*, *,
version: str, package_set: dict[str, Any],
tag: str, package_lock: dict[str, Any],
channel: str, channel: str,
sequence: int, sequence: int,
generated_at: datetime, generated_at: datetime,
expires_at: datetime, expires_at: datetime,
repository_base: str, workspace: Path,
public_base_url: str, public_base_url: str,
) -> dict[str, Any]: ) -> dict[str, Any]:
manifests = _discovered_catalog_manifests() python_lock = _rows_by_name(package_lock, "python")
modules: list[dict[str, Any]] = [] webui_lock = _rows_by_repository(package_lock, "webui")
for module in CATALOG_MODULES: modules: list[dict[str, object]] = []
manifest = manifests.get(module.module_id) core_release: dict[str, object] | None = None
module_version = manifest.version if manifest is not None else version selected_units: list[dict[str, str]] = []
module_tag = f"v{module_version.removeprefix('v')}"
entry: dict[str, Any] = {
"module_id": module.module_id,
"name": module.name,
"description": module.description,
"version": module_version,
"action": "install",
"python_package": module.python_package,
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
"license_features": [f"module.{module.module_id}"],
"tags": list(module.tags),
}
if module.webui_package:
entry["webui_package"] = module.webui_package
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
manifest_metadata = _manifest_catalog_metadata(manifest)
entry.update(manifest_metadata)
if module.provides_interfaces:
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
if module.requires_interfaces:
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
modules.append(entry)
for package in package_set["python"]:
name = str(package["name"])
version = str(package["version"])
repository = str(package["repository"])
selected_units.append(
{
"repo": repository,
"version": version,
"tag": str(package["tag"]),
"commit": str(package["commit"]),
}
)
python_artifact = python_lock[name]
webui_artifact = webui_lock.get(repository)
if name == "govoplan-core":
core_release = {
"name": "GovOPlaN Core",
"version": version,
"python_package": name,
"python_ref": _python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())),
"artifact_integrity": {
"python": _artifact_integrity(python_artifact, ref=_python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())))
},
}
if webui_artifact is not None:
webui_ref = _artifact_url(webui_artifact)
core_release.update(
{
"webui_package": webui_artifact["name"],
"webui_ref": webui_ref,
}
)
core_release["artifact_integrity"]["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
continue
entries = synthesize_repository_catalog_entries(
repo=repository,
version=version,
workspace=workspace,
repository_base="git+https://git.add-ideas.de/GovOPlaN",
source_ref=str(package["tag"]),
)
for entry in entries:
python_ref = _python_ref(name, python_artifact)
entry["python_ref"] = python_ref
entry["source"] = {
"repository": repository,
"tag": package["tag"],
"commit": package["commit"],
}
integrity: dict[str, object] = {
"python": _artifact_integrity(python_artifact, ref=python_ref),
}
if entry.get("webui_package"):
if webui_artifact is None or webui_artifact.get("name") != entry["webui_package"]:
raise ValueError(f"Package lock has no matching WebUI artifact for {repository}.")
webui_ref = _artifact_url(webui_artifact)
entry["webui_ref"] = webui_ref
integrity["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
else:
entry.pop("webui_ref", None)
entry["artifact_integrity"] = integrity
modules.append(entry)
if core_release is None:
raise ValueError("Package set does not contain govoplan-core.")
validate_initial_entry_closure(
catalog_modules=modules,
initial_module_ids={str(item["module_id"]) for item in modules},
)
release_version = str(package_set["release_version"])
return { return {
"catalog_version": "1", "catalog_version": "1",
"channel": channel, "channel": channel,
@@ -298,97 +197,103 @@ def _catalog_payload(
"generated_at": _json_datetime(generated_at), "generated_at": _json_datetime(generated_at),
"expires_at": _json_datetime(expires_at), "expires_at": _json_datetime(expires_at),
"release": { "release": {
"version": version, "version": release_version,
"tag": tag, "tag": f"v{release_version}",
"profile": package_set.get("profile", "base"),
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json", "catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json", "keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
"package_set_sha256": package_set["package_set_sha256"],
"package_lock_sha256": package_lock["lock_sha256"],
"selected_units": sorted(selected_units, key=lambda item: item["repo"]),
}, },
"core_release": { "core_release": core_release,
"name": "GovOPlaN Core", "modules": sorted(modules, key=lambda item: str(item["module_id"])),
"version": version,
"python_package": "govoplan-core",
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
"webui_package": "@govoplan/core-webui",
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
},
"modules": modules,
} }
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]: def _read_hashed_json(path: Path, *, hash_field: str) -> dict[str, Any]:
try: payload = json.loads(path.expanduser().read_text(encoding="utf-8"))
return available_module_manifests(ignore_load_errors=True) if not isinstance(payload, dict):
except Exception: raise ValueError(f"{path} must contain a JSON object.")
return {} expected = payload.get(hash_field)
unsigned = dict(payload)
unsigned.pop(hash_field, None)
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]: if not isinstance(expected, str) or expected != _canonical_sha256(unsigned):
if manifest is None: raise ValueError(f"{path} {hash_field} does not match its contents.")
return {}
payload: dict[str, object] = {}
if manifest.dependencies:
payload["dependencies"] = list(manifest.dependencies)
if manifest.optional_dependencies:
payload["optional_dependencies"] = list(manifest.optional_dependencies)
if manifest.architecture is not None:
payload["architecture"] = manifest.architecture.to_dict()
payload["information_governance"] = manifest.information_governance.to_dict()
if manifest.external_providers:
payload["external_providers"] = [
declaration.to_dict()
for declaration in manifest.external_providers
]
if manifest.migration_spec is not None:
payload["migration_safety"] = "requires_review"
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
if manifest.migration_spec.migration_after:
payload["migration_after"] = list(manifest.migration_spec.migration_after)
if manifest.migration_spec.migration_before:
payload["migration_before"] = list(manifest.migration_spec.migration_before)
if manifest.migration_spec.migration_tasks:
tasks: list[dict[str, object]] = []
for task in manifest.migration_spec.migration_tasks:
task_payload: dict[str, object] = {
"task_id": task.task_id,
"phase": task.phase,
"summary": task.summary,
"task_version": task.task_version,
"safety": task.safety,
"idempotent": task.idempotent,
}
if task.timeout_seconds is not None:
task_payload["timeout_seconds"] = task.timeout_seconds
tasks.append(task_payload)
payload["migration_tasks"] = tasks
if manifest.provides_interfaces:
payload["provides_interfaces"] = [
{"name": item.name, "version": item.version}
for item in manifest.provides_interfaces
]
if manifest.requires_interfaces:
requirements: list[dict[str, object]] = []
for item in manifest.requires_interfaces:
requirement: dict[str, object] = {
"name": item.name,
"optional": item.optional,
}
if item.version_min is not None:
requirement["version_min"] = item.version_min
if item.version_max_exclusive is not None:
requirement["version_max_exclusive"] = item.version_max_exclusive
requirements.append(requirement)
payload["requires_interfaces"] = requirements
return payload return payload
def _validate_release_inputs(package_set: dict[str, Any], package_lock: dict[str, Any], *, core_version: str) -> None:
if package_set.get("schema_version") != "1" or package_lock.get("schema_version") != "1":
raise ValueError("Package set and lock must use schema version 1.")
if package_set.get("release_version") != core_version or package_lock.get("release_version") != core_version:
raise ValueError("Package set and lock release versions must match --version.")
if package_lock.get("package_set_sha256") != package_set.get("package_set_sha256"):
raise ValueError("Package lock does not belong to the selected package set.")
if package_lock.get("profile", "base") != package_set.get("profile", "base"):
raise ValueError("Package set and lock profiles do not match.")
for group in ("python", "webui"):
selected = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_set.get(group, ()) if isinstance(item, dict)}
locked = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_lock.get(group, ()) if isinstance(item, dict)}
if not selected or selected != locked:
raise ValueError(f"Package lock does not contain the exact {group} package set.")
for item in package_lock[group]:
_artifact_url(item)
if SHA256.fullmatch(str(item.get("sha256") or "")) is None:
raise ValueError(f"Package lock has an invalid {group} artifact digest.")
def _rows_by_name(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
return {str(item["name"]): item for item in payload[group]}
def _rows_by_repository(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
result: dict[str, dict[str, object]] = {}
for item in payload[group]:
repository = str(item["repository"])
if repository in result:
raise ValueError(f"Package lock contains multiple {group} artifacts for {repository}.")
result[repository] = item
return result
def _artifact_url(artifact: dict[str, object]) -> str:
value = str(artifact.get("url") or "")
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.fragment:
raise ValueError(f"Package artifact has an unsafe download URL: {value!r}.")
return value
def _python_ref(name: str, artifact: dict[str, object], *, extras: tuple[object, ...] = ()) -> str:
extra = f"[{','.join(str(item) for item in extras)}]" if extras else ""
return f"{name}{extra} @ {_artifact_url(artifact)}#sha256={artifact['sha256']}"
def _artifact_integrity(artifact: dict[str, object], *, ref: str) -> dict[str, object]:
result: dict[str, object] = {
"ref": ref,
"url": _artifact_url(artifact),
"filename": artifact["filename"],
"sha256": artifact["sha256"],
"size": artifact["size"],
"registry_identity": f"{artifact['name']}@{artifact['version']}",
"git_ref": artifact["tag"],
"source_commit": artifact["commit"],
}
if artifact.get("integrity"):
result["integrity"] = artifact["integrity"]
return result
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]: def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
key_id, separator, path_text = value.partition("=") key_id, separator, path_text = value.partition("=")
if not separator or not key_id.strip() or not path_text.strip(): if not separator or not key_id.strip() or not path_text.strip():
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem") raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
path = Path(path_text).expanduser() path = Path(path_text).expanduser()
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
if not isinstance(private_key, Ed25519PrivateKey): if not isinstance(private_key, Ed25519PrivateKey):
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}") raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}")
return key_id.strip(), private_key return key_id.strip(), private_key
@@ -396,11 +301,10 @@ def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519Priv
signature_payload = dict(payload) signature_payload = dict(payload)
signature_payload.pop("signature", None) signature_payload.pop("signature", None)
signature_payload.pop("signatures", None) signature_payload.pop("signatures", None)
signature = private_key.sign(_canonical_bytes(signature_payload))
return { return {
"algorithm": "ed25519", "algorithm": "ed25519",
"key_id": key_id, "key_id": key_id,
"value": base64.b64encode(signature).decode("ascii"), "value": base64.b64encode(private_key.sign(_canonical_bytes(signature_payload))).decode("ascii"),
} }
@@ -413,7 +317,12 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
{ {
"key_id": key_id, "key_id": key_id,
"status": "active", "status": "active",
"public_key": _public_key_base64(private_key), "public_key": base64.b64encode(
private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
).decode("ascii"),
"not_before": generated_at.date().isoformat() + "T00:00:00Z", "not_before": generated_at.date().isoformat() + "T00:00:00Z",
} }
for key_id, private_key in signing_keys for key_id, private_key in signing_keys
@@ -421,18 +330,14 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
} }
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
public_bytes = private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
return base64.b64encode(public_bytes).decode("ascii")
def _canonical_bytes(payload: object) -> bytes: def _canonical_bytes(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def _canonical_sha256(payload: object) -> str:
return hashlib.sha256(_canonical_bytes(payload)).hexdigest()
def _json_datetime(value: datetime) -> str: def _json_datetime(value: datetime) -> str:
return value.astimezone(UTC).isoformat().replace("+00:00", "Z") return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
+64 -1
View File
@@ -21,6 +21,10 @@ GIT_REQUIREMENT = re.compile(
r"(?P<version>[A-Za-z0-9._+!-]+))$" r"(?P<version>[A-Za-z0-9._+!-]+))$"
) )
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$") LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
EXACT_PACKAGE = re.compile(
r"^(?P<package>govoplan(?:-[a-z0-9-]+)?)(?:\[(?P<extras>[^]]+)\])?=="
r"(?P<version>[A-Za-z0-9._+!-]+)$"
)
class PackageSetError(ValueError): class PackageSetError(ValueError):
@@ -38,6 +42,17 @@ def build_parser() -> argparse.ArgumentParser:
type=Path, type=Path,
default=META_ROOT / "requirements-release.txt", default=META_ROOT / "requirements-release.txt",
) )
parser.add_argument(
"--profile",
choices=("base", "full"),
default="base",
help="Base runtime roots or every package selected by govoplan[full].",
)
parser.add_argument(
"--meta-package",
type=Path,
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent) parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument("--output", type=Path, required=True) parser.add_argument("--output", type=Path, required=True)
return parser return parser
@@ -79,11 +94,46 @@ def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[s
return tuple(values) return tuple(values)
def parse_meta_package(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
project = tomllib.loads(path.read_text(encoding="utf-8")).get("project")
if not isinstance(project, dict):
raise PackageSetError("developer meta-package has no [project] table")
if str(project.get("name") or "") != "govoplan":
raise PackageSetError("developer meta-package has an unexpected project name")
if str(project.get("version") or "").removeprefix("v") != core_version.removeprefix("v"):
raise PackageSetError("developer meta-package version does not match Core")
optional = project.get("optional-dependencies")
full = optional.get("full") if isinstance(optional, dict) else None
dependencies = project.get("dependencies")
if not isinstance(dependencies, list) or not isinstance(full, list):
raise PackageSetError("developer meta-package must declare dependencies and the full extra")
values: list[dict[str, object]] = []
for raw in (*dependencies, *full):
if not isinstance(raw, str) or (match := EXACT_PACKAGE.fullmatch(raw.strip())) is None:
raise PackageSetError(f"developer meta-package requirement is not exact: {raw!r}")
package = match.group("package")
repository = "govoplan-core" if package == "govoplan-core" else package
values.append(
{
"name": package,
"version": match.group("version"),
"repository": repository,
"extras": _extras(match.group("extras")),
}
)
names = [str(item["name"]) for item in values]
if names.count("govoplan-core") != 1 or len(names) != len(set(names)):
raise PackageSetError("developer meta-package must contain one Core and unique packages")
return tuple(values)
def generate_package_set( def generate_package_set(
*, *,
core_version: str, core_version: str,
requirements: Path, requirements: Path,
workspace: Path, workspace: Path,
profile: str = "base",
meta_package: Path | None = None,
) -> dict[str, object]: ) -> dict[str, object]:
core_version = core_version.removeprefix("v") core_version = core_version.removeprefix("v")
if VERSION.fullmatch(core_version) is None: if VERSION.fullmatch(core_version) is None:
@@ -91,7 +141,17 @@ def generate_package_set(
python_packages: list[dict[str, object]] = [] python_packages: list[dict[str, object]] = []
webui_packages: list[dict[str, object]] = [] webui_packages: list[dict[str, object]] = []
seen_webui: set[str] = set() seen_webui: set[str] = set()
for requirement in parse_release_requirements(requirements, core_version=core_version): if profile not in {"base", "full"}:
raise PackageSetError(f"unsupported release profile: {profile}")
selected = (
parse_meta_package(
meta_package or META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
core_version=core_version,
)
if profile == "full"
else parse_release_requirements(requirements, core_version=core_version)
)
for requirement in selected:
repository = workspace / str(requirement["repository"]) repository = workspace / str(requirement["repository"])
tag = f"v{requirement['version']}" tag = f"v{requirement['version']}"
if not (repository / ".git").is_dir(): if not (repository / ".git").is_dir():
@@ -134,6 +194,7 @@ def generate_package_set(
payload: dict[str, object] = { payload: dict[str, object] = {
"schema_version": "1", "schema_version": "1",
"release_version": core_version, "release_version": core_version,
"profile": profile,
"registries": { "registries": {
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple", "python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/", "npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
@@ -181,6 +242,8 @@ def main() -> int:
core_version=version, core_version=version,
requirements=args.requirements.expanduser().resolve(), requirements=args.requirements.expanduser().resolve(),
workspace=workspace, workspace=workspace,
profile=args.profile,
meta_package=args.meta_package.expanduser().resolve(),
) )
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc: except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
print(f"error: {exc}") print(f"error: {exc}")
@@ -10,6 +10,8 @@ import os
from pathlib import Path from pathlib import Path
import subprocess import subprocess
import sys import sys
import tarfile
import tempfile
import tomllib import tomllib
from types import ModuleType from types import ModuleType
from typing import Iterator from typing import Iterator
@@ -30,13 +32,14 @@ def synthesize_repository_catalog_entries(
version: str, version: str,
workspace: Path, workspace: Path,
repository_base: str, repository_base: str,
source_ref: str | None = None,
) -> tuple[dict[str, object], ...]: ) -> tuple[dict[str, object], ...]:
"""Build install entries from tagged, preflighted local source metadata. """Build install entries from tagged, preflighted local source metadata.
The caller owns source-tag and worktree provenance checks. This function Distribution metadata identifies the runtime entry point and the runtime
accepts no hand-maintained module catalog registry: distribution metadata ``ModuleManifest`` is the authoritative dependency/interface/frontend
identifies the runtime entry point and the runtime ``ModuleManifest`` is description. When ``source_ref`` is supplied, metadata is read from that
the authoritative dependency/interface/frontend description. immutable Git tree rather than from the current checkout.
""" """
if os.getenv(_INSPECTION_CHILD) == "1": if os.getenv(_INSPECTION_CHILD) == "1":
@@ -45,6 +48,7 @@ def synthesize_repository_catalog_entries(
version=version, version=version,
workspace=workspace, workspace=workspace,
repository_base=repository_base, repository_base=repository_base,
source_ref=source_ref,
) )
command = ( command = (
sys.executable, sys.executable,
@@ -59,6 +63,8 @@ def synthesize_repository_catalog_entries(
"--repository-base", "--repository-base",
repository_base, repository_base,
) )
if source_ref:
command = (*command, "--source-ref", source_ref)
environment = os.environ.copy() environment = os.environ.copy()
environment[_INSPECTION_CHILD] = "1" environment[_INSPECTION_CHILD] = "1"
release_root = str(Path(__file__).resolve().parents[1]) release_root = str(Path(__file__).resolve().parents[1])
@@ -98,44 +104,46 @@ def synthesize_repository_catalog_entries_in_process(
version: str, version: str,
workspace: Path, workspace: Path,
repository_base: str, repository_base: str,
source_ref: str | None = None,
) -> tuple[dict[str, object], ...]: ) -> tuple[dict[str, object], ...]:
specs = {item.name: item for item in load_repository_specs(include_website=False)} specs = {item.name: item for item in load_repository_specs(include_website=False)}
spec = specs.get(repo) spec = specs.get(repo)
if spec is None: if spec is None:
raise ValueError(f"Cannot synthesize {repo}: repository is not registered.") raise ValueError(f"Cannot synthesize {repo}: repository is not registered.")
root = resolve_repo_path(spec, workspace) checkout_root = resolve_repo_path(spec, workspace)
project = read_project_metadata(root / "pyproject.toml") with materialized_source_tree(checkout_root, source_ref=source_ref) as root:
package = required_text(project, "name", source=f"{repo}/pyproject.toml") project = read_project_metadata(root / "pyproject.toml")
project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v") package = required_text(project, "name", source=f"{repo}/pyproject.toml")
expected_version = version.removeprefix("v") project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v")
if project_version != expected_version: expected_version = version.removeprefix("v")
raise ValueError( if project_version != expected_version:
f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
)
description = optional_text(project.get("description"))
entry_points = module_entry_points(project, repo=repo)
entries: list[dict[str, object]] = []
for declared_module_id, target in sorted(entry_points.items()):
manifest = load_manifest(root=root, target=target, repo=repo)
if manifest.id != declared_module_id:
raise ValueError( raise ValueError(
f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}." f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
) )
if manifest.version.removeprefix("v") != expected_version: description = optional_text(project.get("description"))
raise ValueError( entry_points = module_entry_points(project, repo=repo)
f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}." entries: list[dict[str, object]] = []
for declared_module_id, target in sorted(entry_points.items()):
manifest = load_manifest(root=root, target=target, repo=repo)
if manifest.id != declared_module_id:
raise ValueError(
f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}."
)
if manifest.version.removeprefix("v") != expected_version:
raise ValueError(
f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}."
)
entry = manifest_catalog_entry(
manifest=manifest,
repo=repo,
package=package,
version=expected_version,
description=description,
root=root,
repository_base=repository_base.rstrip("/"),
) )
entry = manifest_catalog_entry( entries.append(entry)
manifest=manifest, return tuple(entries)
repo=repo,
package=package,
version=expected_version,
description=description,
root=root,
repository_base=repository_base.rstrip("/"),
)
entries.append(entry)
return tuple(entries)
def manifest_catalog_entry( def manifest_catalog_entry(
@@ -156,8 +164,7 @@ def manifest_catalog_entry(
"action": "install", "action": "install",
"python_package": package, "python_package": package,
"python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}", "python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}",
"license_features": [f"module.{manifest.id}"], "tags": ["official", "open-source"],
"tags": ["official"],
} }
if description: if description:
entry["description"] = description entry["description"] = description
@@ -218,6 +225,51 @@ def manifest_catalog_entry(
return entry return entry
@contextmanager
def materialized_source_tree(root: Path, *, source_ref: str | None) -> Iterator[Path]:
if not source_ref:
yield root
return
if not (root / ".git").exists():
raise ValueError(f"Cannot inspect {source_ref!r}: {root} is not a Git checkout.")
with tempfile.TemporaryDirectory(prefix="govoplan-catalog-source-") as value:
temporary = Path(value)
archive_path = temporary / "source.tar"
source_root = temporary / "source"
source_root.mkdir()
result = subprocess.run(
[
"git",
"-C",
str(root),
"archive",
"--format=tar",
f"--output={archive_path}",
source_ref,
],
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
timeout=30,
)
if result.returncode != 0:
detail = result.stderr.strip() or "Git archive failed"
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: {detail}")
if archive_path.stat().st_size > 256 * 1024 * 1024:
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive exceeds 256 MiB.")
with tarfile.open(archive_path, mode="r:") as archive:
members = archive.getmembers()
if len(members) > 50_000:
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive has too many entries.")
for member in members:
path = Path(member.name)
if path.is_absolute() or ".." in path.parts or member.issym() or member.islnk() or member.isdev():
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive contains an unsafe entry.")
archive.extractall(source_root, members=members, filter="data")
yield source_root
def validate_initial_entry_closure( def validate_initial_entry_closure(
*, *,
catalog_modules: list[object], catalog_modules: list[object],
@@ -367,6 +419,7 @@ def main() -> int:
parser.add_argument("--version", required=True) parser.add_argument("--version", required=True)
parser.add_argument("--workspace", type=Path, required=True) parser.add_argument("--workspace", type=Path, required=True)
parser.add_argument("--repository-base", required=True) parser.add_argument("--repository-base", required=True)
parser.add_argument("--source-ref")
args = parser.parse_args() args = parser.parse_args()
try: try:
entries = synthesize_repository_catalog_entries( entries = synthesize_repository_catalog_entries(
@@ -374,6 +427,7 @@ def main() -> int:
version=args.version, version=args.version,
workspace=args.workspace.resolve(), workspace=args.workspace.resolve(),
repository_base=args.repository_base, repository_base=args.repository_base,
source_ref=args.source_ref,
) )
except ValueError as exc: except ValueError as exc:
print(str(exc), file=sys.stderr) print(str(exc), file=sys.stderr)
@@ -158,6 +158,7 @@ def build_selective_catalog_candidate(
repository_base=repository_base.rstrip("/"), repository_base=repository_base.rstrip("/"),
workspace=workspace, workspace=workspace,
) )
changes.extend(remove_official_license_requirements(candidate))
changes.extend( changes.extend(
apply_python_artifact_identities( apply_python_artifact_identities(
candidate, candidate,
@@ -728,6 +729,32 @@ def module_entry_repo(entry: dict[str, Any]) -> str | None:
return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None
def remove_official_license_requirements(payload: dict[str, Any]) -> list[CatalogEntryChange]:
"""Official open-source modules never require commercial entitlements."""
modules = payload.get("modules")
if not isinstance(modules, list):
return []
changes: list[CatalogEntryChange] = []
for entry in modules:
if not isinstance(entry, dict):
continue
tags = entry.get("tags")
if not isinstance(tags, list) or "official" not in tags or "license_features" not in entry:
continue
before = entry.pop("license_features")
changes.append(
CatalogEntryChange(
repo=module_entry_repo(entry) or "unknown",
module_id=str(entry.get("module_id") or "") or None,
field="license_features",
before=json.dumps(before, sort_keys=True),
after=None,
)
)
return changes
def apply_python_artifact_identities( def apply_python_artifact_identities(
payload: dict[str, Any], payload: dict[str, Any],
*, *,
@@ -68,6 +68,7 @@ from __future__ import annotations
import hashlib import hashlib
import json import json
import os
from pathlib import Path from pathlib import Path
import sys import sys
@@ -91,11 +92,18 @@ for item in lock["webui"]:
if item["name"] in rows: if item["name"] in rows:
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}") raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
rows[item["name"]] = item rows[item["name"]] = item
install_all = os.environ.get("GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES", "").strip().lower() in {"1", "true", "yes", "on"}
names = (
sorted(name for name in rows if name != "@govoplan/core-webui")
if install_all
else [
line.split("\t", 1)[0]
for line in output.read_text(encoding="utf-8").splitlines()
if line
]
)
requested = [] requested = []
for line in output.read_text(encoding="utf-8").splitlines(): for name in names:
if not line:
continue
name, _source_ref = line.split("\t", 1)
row = rows.get(name) row = rows.get(name)
if not isinstance(row, dict): if not isinstance(row, dict):
raise SystemExit(f"WebUI package lock has no artifact for {name}") raise SystemExit(f"WebUI package lock has no artifact for {name}")
+28 -4
View File
@@ -15,6 +15,7 @@ Options:
--channel <name> Catalog channel. Defaults to stable. --channel <name> Catalog channel. Defaults to stable.
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp. --sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
--expires-days <days> Catalog expiry window. Defaults to 90. --expires-days <days> Catalog expiry window. Defaults to 90.
--profile <base|full> Package composition to publish. Defaults to full.
--catalog-signing-key <key-id=/path/private.pem> --catalog-signing-key <key-id=/path/private.pem>
Ed25519 private key. May be repeated for rotation. Ed25519 private key. May be repeated for rotation.
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core. --core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
@@ -46,6 +47,7 @@ VERSION=""
CHANNEL="stable" CHANNEL="stable"
SEQUENCE="" SEQUENCE=""
EXPIRES_DAYS="90" EXPIRES_DAYS="90"
PROFILE="full"
PUBLIC_BASE_URL="https://govoplan.add-ideas.de" PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
REMOTE="origin" REMOTE="origin"
BRANCH="" BRANCH=""
@@ -95,6 +97,11 @@ while [[ $# -gt 0 ]]; do
EXPIRES_DAYS="$2" EXPIRES_DAYS="$2"
shift 2 shift 2
;; ;;
--profile)
[[ $# -ge 2 ]] || fail "missing value for $1"
PROFILE="$2"
shift 2
;;
--catalog-signing-key) --catalog-signing-key)
[[ $# -ge 2 ]] || fail "missing value for $1" [[ $# -ge 2 ]] || fail "missing value for $1"
SIGNING_KEYS+=("$2") SIGNING_KEYS+=("$2")
@@ -166,14 +173,12 @@ done
[[ -n "$VERSION" ]] || fail "--version is required" [[ -n "$VERSION" ]] || fail "--version is required"
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION" [[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
[[ "$PROFILE" == "base" || "$PROFILE" == "full" ]] || fail "profile must be base or full: $PROFILE"
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required" [[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT" [[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
[[ -d "$WEB_ROOT/.git" ]] || fail "not an addideas-govoplan-website git repo: $WEB_ROOT" [[ -d "$WEB_ROOT/.git" ]] || fail "not an addideas-govoplan-website git repo: $WEB_ROOT"
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON" command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
if [[ "$BUILD_WEB" -eq 1 ]]; then
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
fi
if [[ -z "$BRANCH" ]]; then if [[ -z "$BRANCH" ]]; then
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)" BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
@@ -183,6 +188,10 @@ fi
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json" CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json" KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
TAG_NAME="catalog-v$VERSION" TAG_NAME="catalog-v$VERSION"
TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-catalog.XXXXXXXX")"
PACKAGE_SET="$TEMP_ROOT/release-packages.json"
PACKAGE_LOCK="$TEMP_ROOT/package-artifacts.lock.json"
trap 'rm -rf "$TEMP_ROOT"' EXIT
run() { run() {
printf '+' printf '+'
@@ -197,6 +206,9 @@ GEN_ARGS=(
env "GOVOPLAN_CORE_ROOT=$CORE_ROOT" env "GOVOPLAN_CORE_ROOT=$CORE_ROOT"
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py" "$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
--version "$VERSION" --version "$VERSION"
--package-set "$PACKAGE_SET"
--package-lock "$PACKAGE_LOCK"
--workspace "$PARENT"
--channel "$CHANNEL" --channel "$CHANNEL"
--expires-days "$EXPIRES_DAYS" --expires-days "$EXPIRES_DAYS"
--catalog-output "$CATALOG_PATH" --catalog-output "$CATALOG_PATH"
@@ -210,6 +222,18 @@ for signing_key in "${SIGNING_KEYS[@]}"; do
GEN_ARGS+=(--catalog-signing-key "$signing_key") GEN_ARGS+=(--catalog-signing-key "$signing_key")
done done
run "$PYTHON" "$META_ROOT/tools/release/generate-release-package-set.py" \
--version "$VERSION" \
--profile "$PROFILE" \
--workspace "$PARENT" \
--output "$PACKAGE_SET"
run "$PYTHON" "$META_ROOT/tools/release/resolve-package-artifacts.py" \
--package-set "$PACKAGE_SET" \
--wheelhouse "$TEMP_ROOT/wheelhouse" \
--webui-packages "$TEMP_ROOT/webui-packages" \
--lock-output "$PACKAGE_LOCK" \
--python "$PYTHON" \
--npm "$NPM_BIN"
run "${GEN_ARGS[@]}" run "${GEN_ARGS[@]}"
if [[ "$DRY_RUN" -eq 0 ]]; then if [[ "$DRY_RUN" -eq 0 ]]; then
+80 -8
View File
@@ -64,17 +64,26 @@ def resolve(args: argparse.Namespace) -> dict[str, object]:
python=args.python, python=args.python,
index_url=str(package_set["registries"]["python"]), index_url=str(package_set["registries"]["python"]),
) )
_download_webui( webui_registry_metadata = _download_webui(
packages=tuple(package_set["webui"]), packages=tuple(package_set["webui"]),
destination=webui, destination=webui,
npm=args.npm, npm=args.npm,
registry=str(package_set["registries"]["npm"]), registry=str(package_set["registries"]["npm"]),
) )
python_rows = _verify_wheels(tuple(package_set["python"]), wheels) python_rows = _verify_wheels(
webui_rows = _verify_webui(tuple(package_set["webui"]), webui) tuple(package_set["python"]),
wheels,
registry=str(package_set["registries"]["python"]),
)
webui_rows = _verify_webui(
tuple(package_set["webui"]),
webui,
registry_metadata=webui_registry_metadata,
)
lock: dict[str, object] = { lock: dict[str, object] = {
"schema_version": "1", "schema_version": "1",
"release_version": package_set["release_version"], "release_version": package_set["release_version"],
"profile": package_set.get("profile", "base"),
"package_set_sha256": package_set["package_set_sha256"], "package_set_sha256": package_set["package_set_sha256"],
"registries": package_set["registries"], "registries": package_set["registries"],
"python": python_rows, "python": python_rows,
@@ -169,7 +178,7 @@ def _download_wheels(
def _download_webui( def _download_webui(
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str *, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
) -> None: ) -> dict[str, dict[str, str]]:
environment = dict(os.environ) environment = dict(os.environ)
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "") token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
@@ -182,7 +191,31 @@ def _download_webui(
os.chmod(npmrc.name, 0o600) os.chmod(npmrc.name, 0o600)
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
try: try:
metadata: dict[str, dict[str, str]] = {}
for item in packages: for item in packages:
view = subprocess.run(
[npm, "view", f"{item['name']}@{item['version']}", "dist", "--json", "--registry", registry],
check=True,
env=environment,
text=True,
stdout=subprocess.PIPE,
)
dist = json.loads(view.stdout)
if not isinstance(dist, dict):
raise PackageArtifactError(f"npm registry returned no distribution metadata for {item['name']}")
tarball = dist.get("tarball")
integrity = dist.get("integrity")
parsed = urlsplit(str(tarball or ""))
if (
parsed.scheme != "https"
or not parsed.netloc
or parsed.username
or parsed.password
or not isinstance(integrity, str)
or not integrity.startswith("sha512-")
):
raise PackageArtifactError(f"npm registry returned unsafe distribution metadata for {item['name']}")
metadata[str(item["name"])] = {"url": str(tarball), "integrity": integrity}
subprocess.run( subprocess.run(
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry], [npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
check=True, check=True,
@@ -191,9 +224,15 @@ def _download_webui(
finally: finally:
if npmrc is not None: if npmrc is not None:
Path(npmrc.name).unlink(missing_ok=True) Path(npmrc.name).unlink(missing_ok=True)
return metadata
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: def _verify_wheels(
packages: tuple[dict[str, object], ...],
root: Path,
*,
registry: str | None = None,
) -> list[dict[str, object]]:
expected = {_normalize(str(item["name"])): item for item in packages} expected = {_normalize(str(item["name"])): item for item in packages}
rows: list[dict[str, object]] = [] rows: list[dict[str, object]] = []
seen: set[str] = set() seen: set[str] = set()
@@ -204,13 +243,21 @@ def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[
if package is None or identity["version"] != package["version"] or name in seen: if package is None or identity["version"] != package["version"] or name in seen:
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}") raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
seen.add(name) seen.add(name)
rows.append(_artifact_row(path, package)) row = _artifact_row(path, package)
if registry:
row["url"] = _python_artifact_url(registry, package=package, filename=path.name)
rows.append(row)
if seen != set(expected): if seen != set(expected):
raise PackageArtifactError("registry did not return every selected Python wheel") raise PackageArtifactError("registry did not return every selected Python wheel")
return sorted(rows, key=lambda item: str(item["name"])) return sorted(rows, key=lambda item: str(item["name"]))
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: def _verify_webui(
packages: tuple[dict[str, object], ...],
root: Path,
*,
registry_metadata: dict[str, dict[str, str]] | None = None,
) -> list[dict[str, object]]:
expected = {str(item["name"]): item for item in packages} expected = {str(item["name"]): item for item in packages}
rows: list[dict[str, object]] = [] rows: list[dict[str, object]] = []
seen: set[str] = set() seen: set[str] = set()
@@ -222,7 +269,13 @@ def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[d
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}") raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
seen.add(name) seen.add(name)
row = _artifact_row(path, package) row = _artifact_row(path, package)
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii") integrity = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
row["integrity"] = integrity
metadata = (registry_metadata or {}).get(name)
if metadata:
if metadata.get("integrity") != integrity:
raise PackageArtifactError(f"npm registry integrity does not match downloaded package: {name}")
row["url"] = metadata["url"]
rows.append(row) rows.append(row)
if seen != set(expected): if seen != set(expected):
raise PackageArtifactError("registry did not return every selected WebUI package") raise PackageArtifactError("registry did not return every selected WebUI package")
@@ -283,6 +336,25 @@ def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
return row return row
def _python_artifact_url(
registry: str,
*,
package: dict[str, object],
filename: str,
) -> str:
parsed = urlsplit(registry.rstrip("/"))
path = parsed.path.rstrip("/")
if not path.endswith("/simple"):
raise PackageArtifactError("Python registry URL must end in /simple to derive immutable artifacts")
artifact_path = (
f"{path.removesuffix('/simple')}/files/"
f"{quote(str(package['name']), safe='')}/"
f"{quote(str(package['version']), safe='')}/"
f"{quote(filename, safe='')}"
)
return urlunsplit((parsed.scheme, parsed.netloc, artifact_path, "", ""))
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None: def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"] lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
for row in rows: for row in rows: