GovOPlaN/govoplan
2
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Publish complete signed module catalogs
Dependency Audit / dependency-audit (push) Successful in 1m46s
Details
Deployment Installer / deployment-installer (push) Successful in 5s
Details
Security Audit / security-audit (push) Successful in 11m49s
Details

Browse Source
This commit is contained in:
zemion
2026-08-06 21:13:33 +02:00
parent f3cfd1bccc
commit 09046e6e62
17 changed files with 666 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/release/generate-release-catalog.py
+212 -307
View File
@@ -1,187 +1,44 @@
#!/usr/bin/env python3
"""Generate and sign a GovOPlaN module package release catalog."""
"""Generate a signed registry-backed GovOPlaN module package catalog."""
from __future__ import annotations
import argparse
import base64
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import hashlib
import json
import os
from pathlib import Path
import re
import sys
from typing import Any
from urllib.parse import urlsplit
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
META_ROOT = Path(__file__).resolve().parents[2]
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
sys.path.insert(0, str(CORE_ROOT / "src"))
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
from govoplan_core.core.modules import ModuleManifest # noqa: E402
from govoplan_core.server.registry import available_module_manifests # noqa: E402
from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402
GITEA_BASE = "git+ssh://git@git.add-ideas.de/GovOPlaN"
@dataclass(frozen=True, slots=True)
class CatalogModule:
module_id: str
repo: str
python_package: str
name: str
description: str
tags: tuple[str, ...]
webui_package: str | None = None
provides_interfaces: tuple[dict[str, object], ...] = ()
requires_interfaces: tuple[dict[str, object], ...] = ()
CATALOG_MODULES = (
CatalogModule(
module_id="tenancy",
repo="govoplan-tenancy",
python_package="govoplan-tenancy",
name="Tenancy",
description="Tenant registry, tenant settings, and tenant resolution platform module.",
tags=("official", "platform-module"),
webui_package="@govoplan/tenancy-webui",
),
CatalogModule(
module_id="organizations",
repo="govoplan-organizations",
python_package="govoplan-organizations",
name="Organizations",
description="Organization units, functions, and account-held function assignments.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="identity",
repo="govoplan-identity",
python_package="govoplan-identity",
name="Identity",
description="Canonical identities and links between identities and platform accounts.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="access",
repo="govoplan-access",
python_package="govoplan-access",
name="Access",
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
tags=("official", "platform-module"),
webui_package="@govoplan/access-webui",
),
CatalogModule(
module_id="admin",
repo="govoplan-admin",
python_package="govoplan-admin",
name="Admin",
description="System settings, governance templates, module management, and admin shell contributions.",
tags=("official", "platform-module"),
webui_package="@govoplan/admin-webui",
),
CatalogModule(
module_id="policy",
repo="govoplan-policy",
python_package="govoplan-policy",
name="Policy",
description="Policy and governance capability module.",
tags=("official", "platform-module"),
webui_package="@govoplan/policy-webui",
),
CatalogModule(
module_id="audit",
repo="govoplan-audit",
python_package="govoplan-audit",
name="Audit",
description="Audit-log storage and audit administration routes.",
tags=("official", "platform-module"),
webui_package="@govoplan/audit-webui",
),
CatalogModule(
module_id="dashboard",
repo="govoplan-dashboard",
python_package="govoplan-dashboard",
name="Dashboard",
description="Configurable user home assembled from module-provided dashboard widgets.",
tags=("official", "platform-module"),
webui_package="@govoplan/dashboard-webui",
),
CatalogModule(
module_id="addresses",
repo="govoplan-addresses",
python_package="govoplan-addresses",
name="Addresses",
description="Reusable address directories, recipient sources, consent metadata, and address quality workflows.",
tags=("official", "business-module"),
webui_package="@govoplan/addresses-webui",
),
CatalogModule(
module_id="files",
repo="govoplan-files",
python_package="govoplan-files",
name="Files",
description="Managed file spaces and campaign attachment integration.",
tags=("official", "service-module"),
webui_package="@govoplan/files-webui",
),
CatalogModule(
module_id="mail",
repo="govoplan-mail",
python_package="govoplan-mail",
name="Mail",
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
tags=("official", "service-module"),
webui_package="@govoplan/mail-webui",
),
CatalogModule(
module_id="campaigns",
repo="govoplan-campaign",
python_package="govoplan-campaign",
name="Campaigns",
description="Campaign authoring, validation, queueing, delivery control, and reports.",
tags=("official", "business-module"),
webui_package="@govoplan/campaign-webui",
),
CatalogModule(
module_id="calendar",
repo="govoplan-calendar",
python_package="govoplan-calendar",
name="Calendar",
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
tags=("official", "service-module"),
webui_package="@govoplan/calendar-webui",
),
CatalogModule(
module_id="docs",
repo="govoplan-docs",
python_package="govoplan-docs",
name="Docs",
description="Configured-system documentation and evidence-aware help surfaces.",
tags=("official", "platform-module"),
webui_package="@govoplan/docs-webui",
),
CatalogModule(
module_id="ops",
repo="govoplan-ops",
python_package="govoplan-ops",
name="Ops",
description="Runtime health, deployment profile, worker split, and sizing visibility.",
tags=("official", "platform-module"),
webui_package="@govoplan/ops-webui",
),
from govoplan_release.catalog_entry_synthesis import ( # noqa: E402
synthesize_repository_catalog_entries,
validate_initial_entry_closure,
)
SHA256 = re.compile(r"^[0-9a-f]{64}$")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
parser.add_argument("--version", required=True, help="Core release version, without leading v.")
parser.add_argument("--package-set", type=Path, required=True)
parser.add_argument("--package-lock", type=Path, required=True)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument("--channel", default="stable")
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
parser.add_argument("--expires-days", type=int, default=90)
@@ -195,41 +52,33 @@ def main() -> int:
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
)
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
parser.add_argument("--repository-base", default=GITEA_BASE)
args = parser.parse_args()
version = args.version.removeprefix("v")
version_issues = selected_repository_version_issues(
repo_versions={
"govoplan-core": version,
**{module.repo: version for module in CATALOG_MODULES},
},
workspace=CORE_ROOT.parent,
)
if version_issues:
details = "; ".join(
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
for issue in version_issues
try:
version = args.version.removeprefix("v")
package_set = _read_hashed_json(args.package_set, hash_field="package_set_sha256")
package_lock = _read_hashed_json(args.package_lock, hash_field="lock_sha256")
_validate_release_inputs(package_set, package_lock, core_version=version)
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
generated_at = datetime.now(tz=UTC)
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
catalog = _catalog_payload(
package_set=package_set,
package_lock=package_lock,
channel=args.channel,
sequence=sequence,
generated_at=generated_at,
expires_at=generated_at + timedelta(days=args.expires_days),
workspace=args.workspace.expanduser().resolve(),
public_base_url=args.public_base_url.rstrip("/"),
)
parser.error(f"version alignment gate failed: {details}")
tag = f"v{version}"
generated_at = datetime.now(tz=UTC)
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
expires_at = generated_at + timedelta(days=args.expires_days)
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
catalog = _catalog_payload(
version=version,
tag=tag,
channel=args.channel,
sequence=sequence,
generated_at=generated_at,
expires_at=expires_at,
repository_base=args.repository_base.rstrip("/"),
public_base_url=args.public_base_url.rstrip("/"),
)
if signing_keys:
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
if signing_keys:
catalog["signatures"] = [
_signature(catalog, key_id=key_id, private_key=private_key)
for key_id, private_key in signing_keys
]
except (KeyError, OSError, ValueError, json.JSONDecodeError) as exc:
parser.error(str(exc))
output = args.catalog_output.expanduser()
output.parent.mkdir(parents=True, exist_ok=True)
@@ -249,48 +98,98 @@ def main() -> int:
print(f"channel={args.channel}")
print(f"sequence={sequence}")
print(f"version={version}")
print(f"profile={package_set.get('profile', 'base')}")
return 0
def _catalog_payload(
*,
version: str,
tag: str,
package_set: dict[str, Any],
package_lock: dict[str, Any],
channel: str,
sequence: int,
generated_at: datetime,
expires_at: datetime,
repository_base: str,
workspace: Path,
public_base_url: str,
) -> dict[str, Any]:
manifests = _discovered_catalog_manifests()
modules: list[dict[str, Any]] = []
for module in CATALOG_MODULES:
manifest = manifests.get(module.module_id)
module_version = manifest.version if manifest is not None else version
module_tag = f"v{module_version.removeprefix('v')}"
entry: dict[str, Any] = {
"module_id": module.module_id,
"name": module.name,
"description": module.description,
"version": module_version,
"action": "install",
"python_package": module.python_package,
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
"license_features": [f"module.{module.module_id}"],
"tags": list(module.tags),
}
if module.webui_package:
entry["webui_package"] = module.webui_package
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
manifest_metadata = _manifest_catalog_metadata(manifest)
entry.update(manifest_metadata)
if module.provides_interfaces:
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
if module.requires_interfaces:
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
modules.append(entry)
python_lock = _rows_by_name(package_lock, "python")
webui_lock = _rows_by_repository(package_lock, "webui")
modules: list[dict[str, object]] = []
core_release: dict[str, object] | None = None
selected_units: list[dict[str, str]] = []
for package in package_set["python"]:
name = str(package["name"])
version = str(package["version"])
repository = str(package["repository"])
selected_units.append(
{
"repo": repository,
"version": version,
"tag": str(package["tag"]),
"commit": str(package["commit"]),
}
)
python_artifact = python_lock[name]
webui_artifact = webui_lock.get(repository)
if name == "govoplan-core":
core_release = {
"name": "GovOPlaN Core",
"version": version,
"python_package": name,
"python_ref": _python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())),
"artifact_integrity": {
"python": _artifact_integrity(python_artifact, ref=_python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())))
},
}
if webui_artifact is not None:
webui_ref = _artifact_url(webui_artifact)
core_release.update(
{
"webui_package": webui_artifact["name"],
"webui_ref": webui_ref,
}
)
core_release["artifact_integrity"]["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
continue
entries = synthesize_repository_catalog_entries(
repo=repository,
version=version,
workspace=workspace,
repository_base="git+https://git.add-ideas.de/GovOPlaN",
source_ref=str(package["tag"]),
)
for entry in entries:
python_ref = _python_ref(name, python_artifact)
entry["python_ref"] = python_ref
entry["source"] = {
"repository": repository,
"tag": package["tag"],
"commit": package["commit"],
}
integrity: dict[str, object] = {
"python": _artifact_integrity(python_artifact, ref=python_ref),
}
if entry.get("webui_package"):
if webui_artifact is None or webui_artifact.get("name") != entry["webui_package"]:
raise ValueError(f"Package lock has no matching WebUI artifact for {repository}.")
webui_ref = _artifact_url(webui_artifact)
entry["webui_ref"] = webui_ref
integrity["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
else:
entry.pop("webui_ref", None)
entry["artifact_integrity"] = integrity
modules.append(entry)
if core_release is None:
raise ValueError("Package set does not contain govoplan-core.")
validate_initial_entry_closure(
catalog_modules=modules,
initial_module_ids={str(item["module_id"]) for item in modules},
)
release_version = str(package_set["release_version"])
return {
"catalog_version": "1",
"channel": channel,
@@ -298,97 +197,103 @@ def _catalog_payload(
"generated_at": _json_datetime(generated_at),
"expires_at": _json_datetime(expires_at),
"release": {
"version": version,
"tag": tag,
"version": release_version,
"tag": f"v{release_version}",
"profile": package_set.get("profile", "base"),
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
"package_set_sha256": package_set["package_set_sha256"],
"package_lock_sha256": package_lock["lock_sha256"],
"selected_units": sorted(selected_units, key=lambda item: item["repo"]),
},
"core_release": {
"name": "GovOPlaN Core",
"version": version,
"python_package": "govoplan-core",
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
"webui_package": "@govoplan/core-webui",
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
},
"modules": modules,
"core_release": core_release,
"modules": sorted(modules, key=lambda item: str(item["module_id"])),
}
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
try:
return available_module_manifests(ignore_load_errors=True)
except Exception:
return {}
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
if manifest is None:
return {}
payload: dict[str, object] = {}
if manifest.dependencies:
payload["dependencies"] = list(manifest.dependencies)
if manifest.optional_dependencies:
payload["optional_dependencies"] = list(manifest.optional_dependencies)
if manifest.architecture is not None:
payload["architecture"] = manifest.architecture.to_dict()
payload["information_governance"] = manifest.information_governance.to_dict()
if manifest.external_providers:
payload["external_providers"] = [
declaration.to_dict()
for declaration in manifest.external_providers
]
if manifest.migration_spec is not None:
payload["migration_safety"] = "requires_review"
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
if manifest.migration_spec.migration_after:
payload["migration_after"] = list(manifest.migration_spec.migration_after)
if manifest.migration_spec.migration_before:
payload["migration_before"] = list(manifest.migration_spec.migration_before)
if manifest.migration_spec.migration_tasks:
tasks: list[dict[str, object]] = []
for task in manifest.migration_spec.migration_tasks:
task_payload: dict[str, object] = {
"task_id": task.task_id,
"phase": task.phase,
"summary": task.summary,
"task_version": task.task_version,
"safety": task.safety,
"idempotent": task.idempotent,
}
if task.timeout_seconds is not None:
task_payload["timeout_seconds"] = task.timeout_seconds
tasks.append(task_payload)
payload["migration_tasks"] = tasks
if manifest.provides_interfaces:
payload["provides_interfaces"] = [
{"name": item.name, "version": item.version}
for item in manifest.provides_interfaces
]
if manifest.requires_interfaces:
requirements: list[dict[str, object]] = []
for item in manifest.requires_interfaces:
requirement: dict[str, object] = {
"name": item.name,
"optional": item.optional,
}
if item.version_min is not None:
requirement["version_min"] = item.version_min
if item.version_max_exclusive is not None:
requirement["version_max_exclusive"] = item.version_max_exclusive
requirements.append(requirement)
payload["requires_interfaces"] = requirements
def _read_hashed_json(path: Path, *, hash_field: str) -> dict[str, Any]:
payload = json.loads(path.expanduser().read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"{path} must contain a JSON object.")
expected = payload.get(hash_field)
unsigned = dict(payload)
unsigned.pop(hash_field, None)
if not isinstance(expected, str) or expected != _canonical_sha256(unsigned):
raise ValueError(f"{path} {hash_field} does not match its contents.")
return payload
def _validate_release_inputs(package_set: dict[str, Any], package_lock: dict[str, Any], *, core_version: str) -> None:
if package_set.get("schema_version") != "1" or package_lock.get("schema_version") != "1":
raise ValueError("Package set and lock must use schema version 1.")
if package_set.get("release_version") != core_version or package_lock.get("release_version") != core_version:
raise ValueError("Package set and lock release versions must match --version.")
if package_lock.get("package_set_sha256") != package_set.get("package_set_sha256"):
raise ValueError("Package lock does not belong to the selected package set.")
if package_lock.get("profile", "base") != package_set.get("profile", "base"):
raise ValueError("Package set and lock profiles do not match.")
for group in ("python", "webui"):
selected = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_set.get(group, ()) if isinstance(item, dict)}
locked = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_lock.get(group, ()) if isinstance(item, dict)}
if not selected or selected != locked:
raise ValueError(f"Package lock does not contain the exact {group} package set.")
for item in package_lock[group]:
_artifact_url(item)
if SHA256.fullmatch(str(item.get("sha256") or "")) is None:
raise ValueError(f"Package lock has an invalid {group} artifact digest.")
def _rows_by_name(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
return {str(item["name"]): item for item in payload[group]}
def _rows_by_repository(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
result: dict[str, dict[str, object]] = {}
for item in payload[group]:
repository = str(item["repository"])
if repository in result:
raise ValueError(f"Package lock contains multiple {group} artifacts for {repository}.")
result[repository] = item
return result
def _artifact_url(artifact: dict[str, object]) -> str:
value = str(artifact.get("url") or "")
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.fragment:
raise ValueError(f"Package artifact has an unsafe download URL: {value!r}.")
return value
def _python_ref(name: str, artifact: dict[str, object], *, extras: tuple[object, ...] = ()) -> str:
extra = f"[{','.join(str(item) for item in extras)}]" if extras else ""
return f"{name}{extra} @ {_artifact_url(artifact)}#sha256={artifact['sha256']}"
def _artifact_integrity(artifact: dict[str, object], *, ref: str) -> dict[str, object]:
result: dict[str, object] = {
"ref": ref,
"url": _artifact_url(artifact),
"filename": artifact["filename"],
"sha256": artifact["sha256"],
"size": artifact["size"],
"registry_identity": f"{artifact['name']}@{artifact['version']}",
"git_ref": artifact["tag"],
"source_commit": artifact["commit"],
}
if artifact.get("integrity"):
result["integrity"] = artifact["integrity"]
return result
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
key_id, separator, path_text = value.partition("=")
if not separator or not key_id.strip() or not path_text.strip():
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
path = Path(path_text).expanduser()
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
if not isinstance(private_key, Ed25519PrivateKey):
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}")
return key_id.strip(), private_key
@@ -396,11 +301,10 @@ def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519Priv
signature_payload = dict(payload)
signature_payload.pop("signature", None)
signature_payload.pop("signatures", None)
signature = private_key.sign(_canonical_bytes(signature_payload))
return {
"algorithm": "ed25519",
"key_id": key_id,
"value": base64.b64encode(signature).decode("ascii"),
"value": base64.b64encode(private_key.sign(_canonical_bytes(signature_payload))).decode("ascii"),
}
@@ -413,7 +317,12 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
{
"key_id": key_id,
"status": "active",
"public_key": _public_key_base64(private_key),
"public_key": base64.b64encode(
private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
).decode("ascii"),
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
}
for key_id, private_key in signing_keys
@@ -421,18 +330,14 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
}
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
public_bytes = private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
return base64.b64encode(public_bytes).decode("ascii")
def _canonical_bytes(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def _canonical_sha256(payload: object) -> str:
return hashlib.sha256(_canonical_bytes(payload)).hexdigest()
def _json_datetime(value: datetime) -> str:
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")