Publish complete signed module catalogs
This commit is contained in:
@@ -1,187 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate and sign a GovOPlaN module package release catalog."""
|
||||
"""Generate a signed registry-backed GovOPlaN module package catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
|
||||
sys.path.insert(0, str(CORE_ROOT / "src"))
|
||||
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||
from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402
|
||||
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/GovOPlaN"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CatalogModule:
|
||||
module_id: str
|
||||
repo: str
|
||||
python_package: str
|
||||
name: str
|
||||
description: str
|
||||
tags: tuple[str, ...]
|
||||
webui_package: str | None = None
|
||||
provides_interfaces: tuple[dict[str, object], ...] = ()
|
||||
requires_interfaces: tuple[dict[str, object], ...] = ()
|
||||
|
||||
|
||||
CATALOG_MODULES = (
|
||||
CatalogModule(
|
||||
module_id="tenancy",
|
||||
repo="govoplan-tenancy",
|
||||
python_package="govoplan-tenancy",
|
||||
name="Tenancy",
|
||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/tenancy-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="organizations",
|
||||
repo="govoplan-organizations",
|
||||
python_package="govoplan-organizations",
|
||||
name="Organizations",
|
||||
description="Organization units, functions, and account-held function assignments.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="identity",
|
||||
repo="govoplan-identity",
|
||||
python_package="govoplan-identity",
|
||||
name="Identity",
|
||||
description="Canonical identities and links between identities and platform accounts.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="access",
|
||||
repo="govoplan-access",
|
||||
python_package="govoplan-access",
|
||||
name="Access",
|
||||
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/access-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="admin",
|
||||
repo="govoplan-admin",
|
||||
python_package="govoplan-admin",
|
||||
name="Admin",
|
||||
description="System settings, governance templates, module management, and admin shell contributions.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/admin-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="policy",
|
||||
repo="govoplan-policy",
|
||||
python_package="govoplan-policy",
|
||||
name="Policy",
|
||||
description="Policy and governance capability module.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/policy-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="audit",
|
||||
repo="govoplan-audit",
|
||||
python_package="govoplan-audit",
|
||||
name="Audit",
|
||||
description="Audit-log storage and audit administration routes.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/audit-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="dashboard",
|
||||
repo="govoplan-dashboard",
|
||||
python_package="govoplan-dashboard",
|
||||
name="Dashboard",
|
||||
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/dashboard-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="addresses",
|
||||
repo="govoplan-addresses",
|
||||
python_package="govoplan-addresses",
|
||||
name="Addresses",
|
||||
description="Reusable address directories, recipient sources, consent metadata, and address quality workflows.",
|
||||
tags=("official", "business-module"),
|
||||
webui_package="@govoplan/addresses-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="files",
|
||||
repo="govoplan-files",
|
||||
python_package="govoplan-files",
|
||||
name="Files",
|
||||
description="Managed file spaces and campaign attachment integration.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/files-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="mail",
|
||||
repo="govoplan-mail",
|
||||
python_package="govoplan-mail",
|
||||
name="Mail",
|
||||
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/mail-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="campaigns",
|
||||
repo="govoplan-campaign",
|
||||
python_package="govoplan-campaign",
|
||||
name="Campaigns",
|
||||
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
||||
tags=("official", "business-module"),
|
||||
webui_package="@govoplan/campaign-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="calendar",
|
||||
repo="govoplan-calendar",
|
||||
python_package="govoplan-calendar",
|
||||
name="Calendar",
|
||||
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/calendar-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="docs",
|
||||
repo="govoplan-docs",
|
||||
python_package="govoplan-docs",
|
||||
name="Docs",
|
||||
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/docs-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="ops",
|
||||
repo="govoplan-ops",
|
||||
python_package="govoplan-ops",
|
||||
name="Ops",
|
||||
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/ops-webui",
|
||||
),
|
||||
from govoplan_release.catalog_entry_synthesis import ( # noqa: E402
|
||||
synthesize_repository_catalog_entries,
|
||||
validate_initial_entry_closure,
|
||||
)
|
||||
|
||||
|
||||
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
||||
parser.add_argument("--version", required=True, help="Core release version, without leading v.")
|
||||
parser.add_argument("--package-set", type=Path, required=True)
|
||||
parser.add_argument("--package-lock", type=Path, required=True)
|
||||
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||
parser.add_argument("--channel", default="stable")
|
||||
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
||||
parser.add_argument("--expires-days", type=int, default=90)
|
||||
@@ -195,41 +52,33 @@ def main() -> int:
|
||||
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
||||
)
|
||||
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
||||
parser.add_argument("--repository-base", default=GITEA_BASE)
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version.removeprefix("v")
|
||||
version_issues = selected_repository_version_issues(
|
||||
repo_versions={
|
||||
"govoplan-core": version,
|
||||
**{module.repo: version for module in CATALOG_MODULES},
|
||||
},
|
||||
workspace=CORE_ROOT.parent,
|
||||
)
|
||||
if version_issues:
|
||||
details = "; ".join(
|
||||
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
|
||||
for issue in version_issues
|
||||
try:
|
||||
version = args.version.removeprefix("v")
|
||||
package_set = _read_hashed_json(args.package_set, hash_field="package_set_sha256")
|
||||
package_lock = _read_hashed_json(args.package_lock, hash_field="lock_sha256")
|
||||
_validate_release_inputs(package_set, package_lock, core_version=version)
|
||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
catalog = _catalog_payload(
|
||||
package_set=package_set,
|
||||
package_lock=package_lock,
|
||||
channel=args.channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
expires_at=generated_at + timedelta(days=args.expires_days),
|
||||
workspace=args.workspace.expanduser().resolve(),
|
||||
public_base_url=args.public_base_url.rstrip("/"),
|
||||
)
|
||||
parser.error(f"version alignment gate failed: {details}")
|
||||
tag = f"v{version}"
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
expires_at = generated_at + timedelta(days=args.expires_days)
|
||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||
|
||||
catalog = _catalog_payload(
|
||||
version=version,
|
||||
tag=tag,
|
||||
channel=args.channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
expires_at=expires_at,
|
||||
repository_base=args.repository_base.rstrip("/"),
|
||||
public_base_url=args.public_base_url.rstrip("/"),
|
||||
)
|
||||
if signing_keys:
|
||||
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
||||
if signing_keys:
|
||||
catalog["signatures"] = [
|
||||
_signature(catalog, key_id=key_id, private_key=private_key)
|
||||
for key_id, private_key in signing_keys
|
||||
]
|
||||
except (KeyError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
output = args.catalog_output.expanduser()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -249,48 +98,98 @@ def main() -> int:
|
||||
print(f"channel={args.channel}")
|
||||
print(f"sequence={sequence}")
|
||||
print(f"version={version}")
|
||||
print(f"profile={package_set.get('profile', 'base')}")
|
||||
return 0
|
||||
|
||||
|
||||
def _catalog_payload(
|
||||
*,
|
||||
version: str,
|
||||
tag: str,
|
||||
package_set: dict[str, Any],
|
||||
package_lock: dict[str, Any],
|
||||
channel: str,
|
||||
sequence: int,
|
||||
generated_at: datetime,
|
||||
expires_at: datetime,
|
||||
repository_base: str,
|
||||
workspace: Path,
|
||||
public_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
manifests = _discovered_catalog_manifests()
|
||||
modules: list[dict[str, Any]] = []
|
||||
for module in CATALOG_MODULES:
|
||||
manifest = manifests.get(module.module_id)
|
||||
module_version = manifest.version if manifest is not None else version
|
||||
module_tag = f"v{module_version.removeprefix('v')}"
|
||||
entry: dict[str, Any] = {
|
||||
"module_id": module.module_id,
|
||||
"name": module.name,
|
||||
"description": module.description,
|
||||
"version": module_version,
|
||||
"action": "install",
|
||||
"python_package": module.python_package,
|
||||
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
|
||||
"license_features": [f"module.{module.module_id}"],
|
||||
"tags": list(module.tags),
|
||||
}
|
||||
if module.webui_package:
|
||||
entry["webui_package"] = module.webui_package
|
||||
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
|
||||
manifest_metadata = _manifest_catalog_metadata(manifest)
|
||||
entry.update(manifest_metadata)
|
||||
if module.provides_interfaces:
|
||||
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
|
||||
if module.requires_interfaces:
|
||||
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
|
||||
modules.append(entry)
|
||||
python_lock = _rows_by_name(package_lock, "python")
|
||||
webui_lock = _rows_by_repository(package_lock, "webui")
|
||||
modules: list[dict[str, object]] = []
|
||||
core_release: dict[str, object] | None = None
|
||||
selected_units: list[dict[str, str]] = []
|
||||
|
||||
for package in package_set["python"]:
|
||||
name = str(package["name"])
|
||||
version = str(package["version"])
|
||||
repository = str(package["repository"])
|
||||
selected_units.append(
|
||||
{
|
||||
"repo": repository,
|
||||
"version": version,
|
||||
"tag": str(package["tag"]),
|
||||
"commit": str(package["commit"]),
|
||||
}
|
||||
)
|
||||
python_artifact = python_lock[name]
|
||||
webui_artifact = webui_lock.get(repository)
|
||||
if name == "govoplan-core":
|
||||
core_release = {
|
||||
"name": "GovOPlaN Core",
|
||||
"version": version,
|
||||
"python_package": name,
|
||||
"python_ref": _python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())),
|
||||
"artifact_integrity": {
|
||||
"python": _artifact_integrity(python_artifact, ref=_python_ref(name, python_artifact, extras=tuple(package.get("extras") or ())))
|
||||
},
|
||||
}
|
||||
if webui_artifact is not None:
|
||||
webui_ref = _artifact_url(webui_artifact)
|
||||
core_release.update(
|
||||
{
|
||||
"webui_package": webui_artifact["name"],
|
||||
"webui_ref": webui_ref,
|
||||
}
|
||||
)
|
||||
core_release["artifact_integrity"]["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
|
||||
continue
|
||||
|
||||
entries = synthesize_repository_catalog_entries(
|
||||
repo=repository,
|
||||
version=version,
|
||||
workspace=workspace,
|
||||
repository_base="git+https://git.add-ideas.de/GovOPlaN",
|
||||
source_ref=str(package["tag"]),
|
||||
)
|
||||
for entry in entries:
|
||||
python_ref = _python_ref(name, python_artifact)
|
||||
entry["python_ref"] = python_ref
|
||||
entry["source"] = {
|
||||
"repository": repository,
|
||||
"tag": package["tag"],
|
||||
"commit": package["commit"],
|
||||
}
|
||||
integrity: dict[str, object] = {
|
||||
"python": _artifact_integrity(python_artifact, ref=python_ref),
|
||||
}
|
||||
if entry.get("webui_package"):
|
||||
if webui_artifact is None or webui_artifact.get("name") != entry["webui_package"]:
|
||||
raise ValueError(f"Package lock has no matching WebUI artifact for {repository}.")
|
||||
webui_ref = _artifact_url(webui_artifact)
|
||||
entry["webui_ref"] = webui_ref
|
||||
integrity["webui"] = _artifact_integrity(webui_artifact, ref=webui_ref)
|
||||
else:
|
||||
entry.pop("webui_ref", None)
|
||||
entry["artifact_integrity"] = integrity
|
||||
modules.append(entry)
|
||||
|
||||
if core_release is None:
|
||||
raise ValueError("Package set does not contain govoplan-core.")
|
||||
validate_initial_entry_closure(
|
||||
catalog_modules=modules,
|
||||
initial_module_ids={str(item["module_id"]) for item in modules},
|
||||
)
|
||||
release_version = str(package_set["release_version"])
|
||||
return {
|
||||
"catalog_version": "1",
|
||||
"channel": channel,
|
||||
@@ -298,97 +197,103 @@ def _catalog_payload(
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"expires_at": _json_datetime(expires_at),
|
||||
"release": {
|
||||
"version": version,
|
||||
"tag": tag,
|
||||
"version": release_version,
|
||||
"tag": f"v{release_version}",
|
||||
"profile": package_set.get("profile", "base"),
|
||||
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
||||
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
||||
"package_set_sha256": package_set["package_set_sha256"],
|
||||
"package_lock_sha256": package_lock["lock_sha256"],
|
||||
"selected_units": sorted(selected_units, key=lambda item: item["repo"]),
|
||||
},
|
||||
"core_release": {
|
||||
"name": "GovOPlaN Core",
|
||||
"version": version,
|
||||
"python_package": "govoplan-core",
|
||||
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
||||
"webui_package": "@govoplan/core-webui",
|
||||
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
||||
},
|
||||
"modules": modules,
|
||||
"core_release": core_release,
|
||||
"modules": sorted(modules, key=lambda item: str(item["module_id"])),
|
||||
}
|
||||
|
||||
|
||||
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
|
||||
try:
|
||||
return available_module_manifests(ignore_load_errors=True)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
|
||||
if manifest is None:
|
||||
return {}
|
||||
payload: dict[str, object] = {}
|
||||
if manifest.dependencies:
|
||||
payload["dependencies"] = list(manifest.dependencies)
|
||||
if manifest.optional_dependencies:
|
||||
payload["optional_dependencies"] = list(manifest.optional_dependencies)
|
||||
if manifest.architecture is not None:
|
||||
payload["architecture"] = manifest.architecture.to_dict()
|
||||
payload["information_governance"] = manifest.information_governance.to_dict()
|
||||
if manifest.external_providers:
|
||||
payload["external_providers"] = [
|
||||
declaration.to_dict()
|
||||
for declaration in manifest.external_providers
|
||||
]
|
||||
if manifest.migration_spec is not None:
|
||||
payload["migration_safety"] = "requires_review"
|
||||
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
|
||||
if manifest.migration_spec.migration_after:
|
||||
payload["migration_after"] = list(manifest.migration_spec.migration_after)
|
||||
if manifest.migration_spec.migration_before:
|
||||
payload["migration_before"] = list(manifest.migration_spec.migration_before)
|
||||
if manifest.migration_spec.migration_tasks:
|
||||
tasks: list[dict[str, object]] = []
|
||||
for task in manifest.migration_spec.migration_tasks:
|
||||
task_payload: dict[str, object] = {
|
||||
"task_id": task.task_id,
|
||||
"phase": task.phase,
|
||||
"summary": task.summary,
|
||||
"task_version": task.task_version,
|
||||
"safety": task.safety,
|
||||
"idempotent": task.idempotent,
|
||||
}
|
||||
if task.timeout_seconds is not None:
|
||||
task_payload["timeout_seconds"] = task.timeout_seconds
|
||||
tasks.append(task_payload)
|
||||
payload["migration_tasks"] = tasks
|
||||
if manifest.provides_interfaces:
|
||||
payload["provides_interfaces"] = [
|
||||
{"name": item.name, "version": item.version}
|
||||
for item in manifest.provides_interfaces
|
||||
]
|
||||
if manifest.requires_interfaces:
|
||||
requirements: list[dict[str, object]] = []
|
||||
for item in manifest.requires_interfaces:
|
||||
requirement: dict[str, object] = {
|
||||
"name": item.name,
|
||||
"optional": item.optional,
|
||||
}
|
||||
if item.version_min is not None:
|
||||
requirement["version_min"] = item.version_min
|
||||
if item.version_max_exclusive is not None:
|
||||
requirement["version_max_exclusive"] = item.version_max_exclusive
|
||||
requirements.append(requirement)
|
||||
payload["requires_interfaces"] = requirements
|
||||
def _read_hashed_json(path: Path, *, hash_field: str) -> dict[str, Any]:
|
||||
payload = json.loads(path.expanduser().read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object.")
|
||||
expected = payload.get(hash_field)
|
||||
unsigned = dict(payload)
|
||||
unsigned.pop(hash_field, None)
|
||||
if not isinstance(expected, str) or expected != _canonical_sha256(unsigned):
|
||||
raise ValueError(f"{path} {hash_field} does not match its contents.")
|
||||
return payload
|
||||
|
||||
|
||||
def _validate_release_inputs(package_set: dict[str, Any], package_lock: dict[str, Any], *, core_version: str) -> None:
|
||||
if package_set.get("schema_version") != "1" or package_lock.get("schema_version") != "1":
|
||||
raise ValueError("Package set and lock must use schema version 1.")
|
||||
if package_set.get("release_version") != core_version or package_lock.get("release_version") != core_version:
|
||||
raise ValueError("Package set and lock release versions must match --version.")
|
||||
if package_lock.get("package_set_sha256") != package_set.get("package_set_sha256"):
|
||||
raise ValueError("Package lock does not belong to the selected package set.")
|
||||
if package_lock.get("profile", "base") != package_set.get("profile", "base"):
|
||||
raise ValueError("Package set and lock profiles do not match.")
|
||||
for group in ("python", "webui"):
|
||||
selected = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_set.get(group, ()) if isinstance(item, dict)}
|
||||
locked = {(item.get("name"), item.get("version"), item.get("repository")) for item in package_lock.get(group, ()) if isinstance(item, dict)}
|
||||
if not selected or selected != locked:
|
||||
raise ValueError(f"Package lock does not contain the exact {group} package set.")
|
||||
for item in package_lock[group]:
|
||||
_artifact_url(item)
|
||||
if SHA256.fullmatch(str(item.get("sha256") or "")) is None:
|
||||
raise ValueError(f"Package lock has an invalid {group} artifact digest.")
|
||||
|
||||
|
||||
def _rows_by_name(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
|
||||
return {str(item["name"]): item for item in payload[group]}
|
||||
|
||||
|
||||
def _rows_by_repository(payload: dict[str, Any], group: str) -> dict[str, dict[str, object]]:
|
||||
result: dict[str, dict[str, object]] = {}
|
||||
for item in payload[group]:
|
||||
repository = str(item["repository"])
|
||||
if repository in result:
|
||||
raise ValueError(f"Package lock contains multiple {group} artifacts for {repository}.")
|
||||
result[repository] = item
|
||||
return result
|
||||
|
||||
|
||||
def _artifact_url(artifact: dict[str, object]) -> str:
|
||||
value = str(artifact.get("url") or "")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.fragment:
|
||||
raise ValueError(f"Package artifact has an unsafe download URL: {value!r}.")
|
||||
return value
|
||||
|
||||
|
||||
def _python_ref(name: str, artifact: dict[str, object], *, extras: tuple[object, ...] = ()) -> str:
|
||||
extra = f"[{','.join(str(item) for item in extras)}]" if extras else ""
|
||||
return f"{name}{extra} @ {_artifact_url(artifact)}#sha256={artifact['sha256']}"
|
||||
|
||||
|
||||
def _artifact_integrity(artifact: dict[str, object], *, ref: str) -> dict[str, object]:
|
||||
result: dict[str, object] = {
|
||||
"ref": ref,
|
||||
"url": _artifact_url(artifact),
|
||||
"filename": artifact["filename"],
|
||||
"sha256": artifact["sha256"],
|
||||
"size": artifact["size"],
|
||||
"registry_identity": f"{artifact['name']}@{artifact['version']}",
|
||||
"git_ref": artifact["tag"],
|
||||
"source_commit": artifact["commit"],
|
||||
}
|
||||
if artifact.get("integrity"):
|
||||
result["integrity"] = artifact["integrity"]
|
||||
return result
|
||||
|
||||
|
||||
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
||||
key_id, separator, path_text = value.partition("=")
|
||||
if not separator or not key_id.strip() or not path_text.strip():
|
||||
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||
raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||
path = Path(path_text).expanduser()
|
||||
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(private_key, Ed25519PrivateKey):
|
||||
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||
raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||
return key_id.strip(), private_key
|
||||
|
||||
|
||||
@@ -396,11 +301,10 @@ def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519Priv
|
||||
signature_payload = dict(payload)
|
||||
signature_payload.pop("signature", None)
|
||||
signature_payload.pop("signatures", None)
|
||||
signature = private_key.sign(_canonical_bytes(signature_payload))
|
||||
return {
|
||||
"algorithm": "ed25519",
|
||||
"key_id": key_id,
|
||||
"value": base64.b64encode(signature).decode("ascii"),
|
||||
"value": base64.b64encode(private_key.sign(_canonical_bytes(signature_payload))).decode("ascii"),
|
||||
}
|
||||
|
||||
|
||||
@@ -413,7 +317,12 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
|
||||
{
|
||||
"key_id": key_id,
|
||||
"status": "active",
|
||||
"public_key": _public_key_base64(private_key),
|
||||
"public_key": base64.b64encode(
|
||||
private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
).decode("ascii"),
|
||||
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
for key_id, private_key in signing_keys
|
||||
@@ -421,18 +330,14 @@ def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at:
|
||||
}
|
||||
|
||||
|
||||
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
return base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _canonical_bytes(payload: object) -> bytes:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def _canonical_sha256(payload: object) -> str:
|
||||
return hashlib.sha256(_canonical_bytes(payload)).hexdigest()
|
||||
|
||||
|
||||
def _json_datetime(value: datetime) -> str:
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
@@ -21,6 +21,10 @@ GIT_REQUIREMENT = re.compile(
|
||||
r"(?P<version>[A-Za-z0-9._+!-]+))$"
|
||||
)
|
||||
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
|
||||
EXACT_PACKAGE = re.compile(
|
||||
r"^(?P<package>govoplan(?:-[a-z0-9-]+)?)(?:\[(?P<extras>[^]]+)\])?=="
|
||||
r"(?P<version>[A-Za-z0-9._+!-]+)$"
|
||||
)
|
||||
|
||||
|
||||
class PackageSetError(ValueError):
|
||||
@@ -38,6 +42,17 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
type=Path,
|
||||
default=META_ROOT / "requirements-release.txt",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
choices=("base", "full"),
|
||||
default="base",
|
||||
help="Base runtime roots or every package selected by govoplan[full].",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--meta-package",
|
||||
type=Path,
|
||||
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
|
||||
)
|
||||
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
@@ -79,11 +94,46 @@ def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[s
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def parse_meta_package(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
|
||||
project = tomllib.loads(path.read_text(encoding="utf-8")).get("project")
|
||||
if not isinstance(project, dict):
|
||||
raise PackageSetError("developer meta-package has no [project] table")
|
||||
if str(project.get("name") or "") != "govoplan":
|
||||
raise PackageSetError("developer meta-package has an unexpected project name")
|
||||
if str(project.get("version") or "").removeprefix("v") != core_version.removeprefix("v"):
|
||||
raise PackageSetError("developer meta-package version does not match Core")
|
||||
optional = project.get("optional-dependencies")
|
||||
full = optional.get("full") if isinstance(optional, dict) else None
|
||||
dependencies = project.get("dependencies")
|
||||
if not isinstance(dependencies, list) or not isinstance(full, list):
|
||||
raise PackageSetError("developer meta-package must declare dependencies and the full extra")
|
||||
values: list[dict[str, object]] = []
|
||||
for raw in (*dependencies, *full):
|
||||
if not isinstance(raw, str) or (match := EXACT_PACKAGE.fullmatch(raw.strip())) is None:
|
||||
raise PackageSetError(f"developer meta-package requirement is not exact: {raw!r}")
|
||||
package = match.group("package")
|
||||
repository = "govoplan-core" if package == "govoplan-core" else package
|
||||
values.append(
|
||||
{
|
||||
"name": package,
|
||||
"version": match.group("version"),
|
||||
"repository": repository,
|
||||
"extras": _extras(match.group("extras")),
|
||||
}
|
||||
)
|
||||
names = [str(item["name"]) for item in values]
|
||||
if names.count("govoplan-core") != 1 or len(names) != len(set(names)):
|
||||
raise PackageSetError("developer meta-package must contain one Core and unique packages")
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def generate_package_set(
|
||||
*,
|
||||
core_version: str,
|
||||
requirements: Path,
|
||||
workspace: Path,
|
||||
profile: str = "base",
|
||||
meta_package: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
core_version = core_version.removeprefix("v")
|
||||
if VERSION.fullmatch(core_version) is None:
|
||||
@@ -91,7 +141,17 @@ def generate_package_set(
|
||||
python_packages: list[dict[str, object]] = []
|
||||
webui_packages: list[dict[str, object]] = []
|
||||
seen_webui: set[str] = set()
|
||||
for requirement in parse_release_requirements(requirements, core_version=core_version):
|
||||
if profile not in {"base", "full"}:
|
||||
raise PackageSetError(f"unsupported release profile: {profile}")
|
||||
selected = (
|
||||
parse_meta_package(
|
||||
meta_package or META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
|
||||
core_version=core_version,
|
||||
)
|
||||
if profile == "full"
|
||||
else parse_release_requirements(requirements, core_version=core_version)
|
||||
)
|
||||
for requirement in selected:
|
||||
repository = workspace / str(requirement["repository"])
|
||||
tag = f"v{requirement['version']}"
|
||||
if not (repository / ".git").is_dir():
|
||||
@@ -134,6 +194,7 @@ def generate_package_set(
|
||||
payload: dict[str, object] = {
|
||||
"schema_version": "1",
|
||||
"release_version": core_version,
|
||||
"profile": profile,
|
||||
"registries": {
|
||||
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
|
||||
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
|
||||
@@ -181,6 +242,8 @@ def main() -> int:
|
||||
core_version=version,
|
||||
requirements=args.requirements.expanduser().resolve(),
|
||||
workspace=workspace,
|
||||
profile=args.profile,
|
||||
meta_package=args.meta_package.expanduser().resolve(),
|
||||
)
|
||||
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
|
||||
print(f"error: {exc}")
|
||||
|
||||
@@ -10,6 +10,8 @@ import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import tomllib
|
||||
from types import ModuleType
|
||||
from typing import Iterator
|
||||
@@ -30,13 +32,14 @@ def synthesize_repository_catalog_entries(
|
||||
version: str,
|
||||
workspace: Path,
|
||||
repository_base: str,
|
||||
source_ref: str | None = None,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
"""Build install entries from tagged, preflighted local source metadata.
|
||||
|
||||
The caller owns source-tag and worktree provenance checks. This function
|
||||
accepts no hand-maintained module catalog registry: distribution metadata
|
||||
identifies the runtime entry point and the runtime ``ModuleManifest`` is
|
||||
the authoritative dependency/interface/frontend description.
|
||||
Distribution metadata identifies the runtime entry point and the runtime
|
||||
``ModuleManifest`` is the authoritative dependency/interface/frontend
|
||||
description. When ``source_ref`` is supplied, metadata is read from that
|
||||
immutable Git tree rather than from the current checkout.
|
||||
"""
|
||||
|
||||
if os.getenv(_INSPECTION_CHILD) == "1":
|
||||
@@ -45,6 +48,7 @@ def synthesize_repository_catalog_entries(
|
||||
version=version,
|
||||
workspace=workspace,
|
||||
repository_base=repository_base,
|
||||
source_ref=source_ref,
|
||||
)
|
||||
command = (
|
||||
sys.executable,
|
||||
@@ -59,6 +63,8 @@ def synthesize_repository_catalog_entries(
|
||||
"--repository-base",
|
||||
repository_base,
|
||||
)
|
||||
if source_ref:
|
||||
command = (*command, "--source-ref", source_ref)
|
||||
environment = os.environ.copy()
|
||||
environment[_INSPECTION_CHILD] = "1"
|
||||
release_root = str(Path(__file__).resolve().parents[1])
|
||||
@@ -98,44 +104,46 @@ def synthesize_repository_catalog_entries_in_process(
|
||||
version: str,
|
||||
workspace: Path,
|
||||
repository_base: str,
|
||||
source_ref: str | None = None,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
specs = {item.name: item for item in load_repository_specs(include_website=False)}
|
||||
spec = specs.get(repo)
|
||||
if spec is None:
|
||||
raise ValueError(f"Cannot synthesize {repo}: repository is not registered.")
|
||||
root = resolve_repo_path(spec, workspace)
|
||||
project = read_project_metadata(root / "pyproject.toml")
|
||||
package = required_text(project, "name", source=f"{repo}/pyproject.toml")
|
||||
project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v")
|
||||
expected_version = version.removeprefix("v")
|
||||
if project_version != expected_version:
|
||||
raise ValueError(
|
||||
f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
|
||||
)
|
||||
description = optional_text(project.get("description"))
|
||||
entry_points = module_entry_points(project, repo=repo)
|
||||
entries: list[dict[str, object]] = []
|
||||
for declared_module_id, target in sorted(entry_points.items()):
|
||||
manifest = load_manifest(root=root, target=target, repo=repo)
|
||||
if manifest.id != declared_module_id:
|
||||
checkout_root = resolve_repo_path(spec, workspace)
|
||||
with materialized_source_tree(checkout_root, source_ref=source_ref) as root:
|
||||
project = read_project_metadata(root / "pyproject.toml")
|
||||
package = required_text(project, "name", source=f"{repo}/pyproject.toml")
|
||||
project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v")
|
||||
expected_version = version.removeprefix("v")
|
||||
if project_version != expected_version:
|
||||
raise ValueError(
|
||||
f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}."
|
||||
f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}."
|
||||
)
|
||||
if manifest.version.removeprefix("v") != expected_version:
|
||||
raise ValueError(
|
||||
f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}."
|
||||
description = optional_text(project.get("description"))
|
||||
entry_points = module_entry_points(project, repo=repo)
|
||||
entries: list[dict[str, object]] = []
|
||||
for declared_module_id, target in sorted(entry_points.items()):
|
||||
manifest = load_manifest(root=root, target=target, repo=repo)
|
||||
if manifest.id != declared_module_id:
|
||||
raise ValueError(
|
||||
f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}."
|
||||
)
|
||||
if manifest.version.removeprefix("v") != expected_version:
|
||||
raise ValueError(
|
||||
f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}."
|
||||
)
|
||||
entry = manifest_catalog_entry(
|
||||
manifest=manifest,
|
||||
repo=repo,
|
||||
package=package,
|
||||
version=expected_version,
|
||||
description=description,
|
||||
root=root,
|
||||
repository_base=repository_base.rstrip("/"),
|
||||
)
|
||||
entry = manifest_catalog_entry(
|
||||
manifest=manifest,
|
||||
repo=repo,
|
||||
package=package,
|
||||
version=expected_version,
|
||||
description=description,
|
||||
root=root,
|
||||
repository_base=repository_base.rstrip("/"),
|
||||
)
|
||||
entries.append(entry)
|
||||
return tuple(entries)
|
||||
entries.append(entry)
|
||||
return tuple(entries)
|
||||
|
||||
|
||||
def manifest_catalog_entry(
|
||||
@@ -156,8 +164,7 @@ def manifest_catalog_entry(
|
||||
"action": "install",
|
||||
"python_package": package,
|
||||
"python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}",
|
||||
"license_features": [f"module.{manifest.id}"],
|
||||
"tags": ["official"],
|
||||
"tags": ["official", "open-source"],
|
||||
}
|
||||
if description:
|
||||
entry["description"] = description
|
||||
@@ -218,6 +225,51 @@ def manifest_catalog_entry(
|
||||
return entry
|
||||
|
||||
|
||||
@contextmanager
|
||||
def materialized_source_tree(root: Path, *, source_ref: str | None) -> Iterator[Path]:
|
||||
if not source_ref:
|
||||
yield root
|
||||
return
|
||||
if not (root / ".git").exists():
|
||||
raise ValueError(f"Cannot inspect {source_ref!r}: {root} is not a Git checkout.")
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-catalog-source-") as value:
|
||||
temporary = Path(value)
|
||||
archive_path = temporary / "source.tar"
|
||||
source_root = temporary / "source"
|
||||
source_root.mkdir()
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(root),
|
||||
"archive",
|
||||
"--format=tar",
|
||||
f"--output={archive_path}",
|
||||
source_ref,
|
||||
],
|
||||
check=False,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "Git archive failed"
|
||||
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: {detail}")
|
||||
if archive_path.stat().st_size > 256 * 1024 * 1024:
|
||||
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive exceeds 256 MiB.")
|
||||
with tarfile.open(archive_path, mode="r:") as archive:
|
||||
members = archive.getmembers()
|
||||
if len(members) > 50_000:
|
||||
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive has too many entries.")
|
||||
for member in members:
|
||||
path = Path(member.name)
|
||||
if path.is_absolute() or ".." in path.parts or member.issym() or member.islnk() or member.isdev():
|
||||
raise ValueError(f"Cannot inspect {source_ref!r} in {root.name}: source archive contains an unsafe entry.")
|
||||
archive.extractall(source_root, members=members, filter="data")
|
||||
yield source_root
|
||||
|
||||
|
||||
def validate_initial_entry_closure(
|
||||
*,
|
||||
catalog_modules: list[object],
|
||||
@@ -367,6 +419,7 @@ def main() -> int:
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--repository-base", required=True)
|
||||
parser.add_argument("--source-ref")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
entries = synthesize_repository_catalog_entries(
|
||||
@@ -374,6 +427,7 @@ def main() -> int:
|
||||
version=args.version,
|
||||
workspace=args.workspace.resolve(),
|
||||
repository_base=args.repository_base,
|
||||
source_ref=args.source_ref,
|
||||
)
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
|
||||
@@ -158,6 +158,7 @@ def build_selective_catalog_candidate(
|
||||
repository_base=repository_base.rstrip("/"),
|
||||
workspace=workspace,
|
||||
)
|
||||
changes.extend(remove_official_license_requirements(candidate))
|
||||
changes.extend(
|
||||
apply_python_artifact_identities(
|
||||
candidate,
|
||||
@@ -728,6 +729,32 @@ def module_entry_repo(entry: dict[str, Any]) -> str | None:
|
||||
return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None
|
||||
|
||||
|
||||
def remove_official_license_requirements(payload: dict[str, Any]) -> list[CatalogEntryChange]:
|
||||
"""Official open-source modules never require commercial entitlements."""
|
||||
|
||||
modules = payload.get("modules")
|
||||
if not isinstance(modules, list):
|
||||
return []
|
||||
changes: list[CatalogEntryChange] = []
|
||||
for entry in modules:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
tags = entry.get("tags")
|
||||
if not isinstance(tags, list) or "official" not in tags or "license_features" not in entry:
|
||||
continue
|
||||
before = entry.pop("license_features")
|
||||
changes.append(
|
||||
CatalogEntryChange(
|
||||
repo=module_entry_repo(entry) or "unknown",
|
||||
module_id=str(entry.get("module_id") or "") or None,
|
||||
field="license_features",
|
||||
before=json.dumps(before, sort_keys=True),
|
||||
after=None,
|
||||
)
|
||||
)
|
||||
return changes
|
||||
|
||||
|
||||
def apply_python_artifact_identities(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
|
||||
@@ -68,6 +68,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
@@ -91,11 +92,18 @@ for item in lock["webui"]:
|
||||
if item["name"] in rows:
|
||||
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
|
||||
rows[item["name"]] = item
|
||||
install_all = os.environ.get("GOVOPLAN_WEBUI_INSTALL_ALL_PACKAGES", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
names = (
|
||||
sorted(name for name in rows if name != "@govoplan/core-webui")
|
||||
if install_all
|
||||
else [
|
||||
line.split("\t", 1)[0]
|
||||
for line in output.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
)
|
||||
requested = []
|
||||
for line in output.read_text(encoding="utf-8").splitlines():
|
||||
if not line:
|
||||
continue
|
||||
name, _source_ref = line.split("\t", 1)
|
||||
for name in names:
|
||||
row = rows.get(name)
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(f"WebUI package lock has no artifact for {name}")
|
||||
|
||||
@@ -15,6 +15,7 @@ Options:
|
||||
--channel <name> Catalog channel. Defaults to stable.
|
||||
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
||||
--expires-days <days> Catalog expiry window. Defaults to 90.
|
||||
--profile <base|full> Package composition to publish. Defaults to full.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Ed25519 private key. May be repeated for rotation.
|
||||
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
|
||||
@@ -46,6 +47,7 @@ VERSION=""
|
||||
CHANNEL="stable"
|
||||
SEQUENCE=""
|
||||
EXPIRES_DAYS="90"
|
||||
PROFILE="full"
|
||||
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
@@ -95,6 +97,11 @@ while [[ $# -gt 0 ]]; do
|
||||
EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--profile)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
PROFILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SIGNING_KEYS+=("$2")
|
||||
@@ -166,14 +173,12 @@ done
|
||||
|
||||
[[ -n "$VERSION" ]] || fail "--version is required"
|
||||
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
||||
[[ "$PROFILE" == "base" || "$PROFILE" == "full" ]] || fail "profile must be base or full: $PROFILE"
|
||||
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
||||
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "not an addideas-govoplan-website git repo: $WEB_ROOT"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||
fi
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||
|
||||
if [[ -z "$BRANCH" ]]; then
|
||||
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
||||
@@ -183,6 +188,10 @@ fi
|
||||
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
||||
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
||||
TAG_NAME="catalog-v$VERSION"
|
||||
TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-catalog.XXXXXXXX")"
|
||||
PACKAGE_SET="$TEMP_ROOT/release-packages.json"
|
||||
PACKAGE_LOCK="$TEMP_ROOT/package-artifacts.lock.json"
|
||||
trap 'rm -rf "$TEMP_ROOT"' EXIT
|
||||
|
||||
run() {
|
||||
printf '+'
|
||||
@@ -197,6 +206,9 @@ GEN_ARGS=(
|
||||
env "GOVOPLAN_CORE_ROOT=$CORE_ROOT"
|
||||
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
|
||||
--version "$VERSION"
|
||||
--package-set "$PACKAGE_SET"
|
||||
--package-lock "$PACKAGE_LOCK"
|
||||
--workspace "$PARENT"
|
||||
--channel "$CHANNEL"
|
||||
--expires-days "$EXPIRES_DAYS"
|
||||
--catalog-output "$CATALOG_PATH"
|
||||
@@ -210,6 +222,18 @@ for signing_key in "${SIGNING_KEYS[@]}"; do
|
||||
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
|
||||
run "$PYTHON" "$META_ROOT/tools/release/generate-release-package-set.py" \
|
||||
--version "$VERSION" \
|
||||
--profile "$PROFILE" \
|
||||
--workspace "$PARENT" \
|
||||
--output "$PACKAGE_SET"
|
||||
run "$PYTHON" "$META_ROOT/tools/release/resolve-package-artifacts.py" \
|
||||
--package-set "$PACKAGE_SET" \
|
||||
--wheelhouse "$TEMP_ROOT/wheelhouse" \
|
||||
--webui-packages "$TEMP_ROOT/webui-packages" \
|
||||
--lock-output "$PACKAGE_LOCK" \
|
||||
--python "$PYTHON" \
|
||||
--npm "$NPM_BIN"
|
||||
run "${GEN_ARGS[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
|
||||
@@ -64,17 +64,26 @@ def resolve(args: argparse.Namespace) -> dict[str, object]:
|
||||
python=args.python,
|
||||
index_url=str(package_set["registries"]["python"]),
|
||||
)
|
||||
_download_webui(
|
||||
webui_registry_metadata = _download_webui(
|
||||
packages=tuple(package_set["webui"]),
|
||||
destination=webui,
|
||||
npm=args.npm,
|
||||
registry=str(package_set["registries"]["npm"]),
|
||||
)
|
||||
python_rows = _verify_wheels(tuple(package_set["python"]), wheels)
|
||||
webui_rows = _verify_webui(tuple(package_set["webui"]), webui)
|
||||
python_rows = _verify_wheels(
|
||||
tuple(package_set["python"]),
|
||||
wheels,
|
||||
registry=str(package_set["registries"]["python"]),
|
||||
)
|
||||
webui_rows = _verify_webui(
|
||||
tuple(package_set["webui"]),
|
||||
webui,
|
||||
registry_metadata=webui_registry_metadata,
|
||||
)
|
||||
lock: dict[str, object] = {
|
||||
"schema_version": "1",
|
||||
"release_version": package_set["release_version"],
|
||||
"profile": package_set.get("profile", "base"),
|
||||
"package_set_sha256": package_set["package_set_sha256"],
|
||||
"registries": package_set["registries"],
|
||||
"python": python_rows,
|
||||
@@ -169,7 +178,7 @@ def _download_wheels(
|
||||
|
||||
def _download_webui(
|
||||
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
|
||||
) -> None:
|
||||
) -> dict[str, dict[str, str]]:
|
||||
environment = dict(os.environ)
|
||||
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
|
||||
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
|
||||
@@ -182,7 +191,31 @@ def _download_webui(
|
||||
os.chmod(npmrc.name, 0o600)
|
||||
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
|
||||
try:
|
||||
metadata: dict[str, dict[str, str]] = {}
|
||||
for item in packages:
|
||||
view = subprocess.run(
|
||||
[npm, "view", f"{item['name']}@{item['version']}", "dist", "--json", "--registry", registry],
|
||||
check=True,
|
||||
env=environment,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
dist = json.loads(view.stdout)
|
||||
if not isinstance(dist, dict):
|
||||
raise PackageArtifactError(f"npm registry returned no distribution metadata for {item['name']}")
|
||||
tarball = dist.get("tarball")
|
||||
integrity = dist.get("integrity")
|
||||
parsed = urlsplit(str(tarball or ""))
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or not parsed.netloc
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or not isinstance(integrity, str)
|
||||
or not integrity.startswith("sha512-")
|
||||
):
|
||||
raise PackageArtifactError(f"npm registry returned unsafe distribution metadata for {item['name']}")
|
||||
metadata[str(item["name"])] = {"url": str(tarball), "integrity": integrity}
|
||||
subprocess.run(
|
||||
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
|
||||
check=True,
|
||||
@@ -191,9 +224,15 @@ def _download_webui(
|
||||
finally:
|
||||
if npmrc is not None:
|
||||
Path(npmrc.name).unlink(missing_ok=True)
|
||||
return metadata
|
||||
|
||||
|
||||
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||
def _verify_wheels(
|
||||
packages: tuple[dict[str, object], ...],
|
||||
root: Path,
|
||||
*,
|
||||
registry: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
expected = {_normalize(str(item["name"])): item for item in packages}
|
||||
rows: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
@@ -204,13 +243,21 @@ def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[
|
||||
if package is None or identity["version"] != package["version"] or name in seen:
|
||||
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
|
||||
seen.add(name)
|
||||
rows.append(_artifact_row(path, package))
|
||||
row = _artifact_row(path, package)
|
||||
if registry:
|
||||
row["url"] = _python_artifact_url(registry, package=package, filename=path.name)
|
||||
rows.append(row)
|
||||
if seen != set(expected):
|
||||
raise PackageArtifactError("registry did not return every selected Python wheel")
|
||||
return sorted(rows, key=lambda item: str(item["name"]))
|
||||
|
||||
|
||||
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||
def _verify_webui(
|
||||
packages: tuple[dict[str, object], ...],
|
||||
root: Path,
|
||||
*,
|
||||
registry_metadata: dict[str, dict[str, str]] | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
expected = {str(item["name"]): item for item in packages}
|
||||
rows: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
@@ -222,7 +269,13 @@ def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[d
|
||||
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
|
||||
seen.add(name)
|
||||
row = _artifact_row(path, package)
|
||||
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
|
||||
integrity = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
|
||||
row["integrity"] = integrity
|
||||
metadata = (registry_metadata or {}).get(name)
|
||||
if metadata:
|
||||
if metadata.get("integrity") != integrity:
|
||||
raise PackageArtifactError(f"npm registry integrity does not match downloaded package: {name}")
|
||||
row["url"] = metadata["url"]
|
||||
rows.append(row)
|
||||
if seen != set(expected):
|
||||
raise PackageArtifactError("registry did not return every selected WebUI package")
|
||||
@@ -283,6 +336,25 @@ def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
|
||||
return row
|
||||
|
||||
|
||||
def _python_artifact_url(
|
||||
registry: str,
|
||||
*,
|
||||
package: dict[str, object],
|
||||
filename: str,
|
||||
) -> str:
|
||||
parsed = urlsplit(registry.rstrip("/"))
|
||||
path = parsed.path.rstrip("/")
|
||||
if not path.endswith("/simple"):
|
||||
raise PackageArtifactError("Python registry URL must end in /simple to derive immutable artifacts")
|
||||
artifact_path = (
|
||||
f"{path.removesuffix('/simple')}/files/"
|
||||
f"{quote(str(package['name']), safe='')}/"
|
||||
f"{quote(str(package['version']), safe='')}/"
|
||||
f"{quote(filename, safe='')}"
|
||||
)
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, artifact_path, "", ""))
|
||||
|
||||
|
||||
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
|
||||
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
|
||||
for row in rows:
|
||||
|
||||
Reference in New Issue
Block a user