add-ideas/govoplan
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

intermediate commit
Some checks failed
Dependency Audit / dependency-audit (push) Failing after 16s
Details
Security Audit / security-audit (push) Failing after 11s
Details

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-14 13:32:09 +02:00
parent dd796b4d3c
commit 05ae81d641
58 changed files with 7526 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

407
tools/release/govoplan_release/selective_catalog.py Normal file
View File

@@ -0,0 +1,407 @@
"""Selective release catalog candidate generation."""
from __future__ import annotations
import base64
from datetime import UTC, datetime, timedelta
import json
from pathlib import Path
import re
from typing import Any
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from govoplan_core.core.module_package_catalog import validate_module_package_catalog
from .catalog import DEFAULT_PUBLIC_BASE_URL, canonical_hash, fetch_json
from .contracts import collect_contracts
from .git_state import collect_repository_snapshot
from .model import CatalogEntryChange, SelectiveCatalogCandidate
from .module_directory import write_module_directory
from .workspace import load_repository_specs, resolve_workspace_root, website_root
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
def build_selective_catalog_candidate(
*,
repo_versions: dict[str, str],
channel: str = "stable",
workspace_root: Path | str | None = None,
output_dir: Path | str | None = None,
base_catalog: Path | str | None = None,
signing_keys: tuple[str, ...] = (),
public_base_url: str = DEFAULT_PUBLIC_BASE_URL,
repository_base: str = GITEA_BASE,
expires_days: int = 90,
sequence: int | None = None,
check_public: bool = True,
write_summary: bool = True,
) -> SelectiveCatalogCandidate:
if not repo_versions:
raise ValueError("At least one repo version must be provided.")
parsed_keys = tuple(parse_signing_key(value) for value in signing_keys)
if not parsed_keys:
raise ValueError("At least one signing key is required.")
workspace = resolve_workspace_root(workspace_root)
web_root = website_root(workspace)
source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel)
payload = read_catalog(source_catalog)
if not isinstance(payload, dict):
raise ValueError("Selective catalog updates require object-style catalogs.")
generated_at = datetime.now(tz=UTC)
resolved_sequence = sequence or next_sequence(payload, generated_at=generated_at)
candidate = json.loads(json.dumps(payload))
candidate["channel"] = channel
candidate["sequence"] = resolved_sequence
candidate["generated_at"] = json_datetime(generated_at)
candidate["expires_at"] = json_datetime(generated_at + timedelta(days=expires_days))
release = candidate.get("release")
if not isinstance(release, dict):
release = {}
release["catalog_url"] = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json"
release["keyring_url"] = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json"
release["selected_units"] = [{"repo": repo, "version": version, "tag": f"v{version.removeprefix('v')}"} for repo, version in sorted(repo_versions.items())]
candidate["release"] = release
repo_contracts = contracts_by_repo(workspace)
changes = apply_repo_updates(
candidate,
repo_versions=repo_versions,
repo_contracts=repo_contracts,
repository_base=repository_base.rstrip("/"),
)
candidate.pop("signature", None)
candidate.pop("signatures", None)
candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys]
output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at)
catalog_path = output_root / "channels" / f"{channel}.json"
keyring_path = output_root / "keyring.json"
summary_path = output_root / "summary.json" if write_summary else None
catalog_path.parent.mkdir(parents=True, exist_ok=True)
catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8")
keyring_payload = keyring_payload_for_candidate(
existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json",
signing_keys=parsed_keys,
generated_at=generated_at,
)
keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
module_directory_files = write_module_directory(
catalog_payload=candidate,
keyring_payload=keyring_payload,
output_root=output_root,
channel=channel,
public_base_url=public_base_url,
)
trusted_keys = trusted_keys_from_keyring(keyring_payload)
validation = validate_module_package_catalog(
catalog_path,
require_trusted=True,
approved_channels=(channel,),
trusted_keys=trusted_keys,
)
validation_warnings = tuple(str(item) for item in validation.get("warnings") or ())
validation_error = validation.get("error")
candidate_catalog_hash = canonical_hash(candidate)
candidate_keyring_hash = canonical_hash(keyring_payload)
public_catalog_url = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json"
public_keyring_url = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json"
published_catalog_hash: str | None = None
published_keyring_hash: str | None = None
if check_public:
published_catalog = fetch_json(public_catalog_url)
if published_catalog["ok"]:
published_catalog_hash = canonical_hash(published_catalog["payload"])
published_keyring = fetch_json(public_keyring_url)
if published_keyring["ok"]:
published_keyring_hash = canonical_hash(published_keyring["payload"])
result = SelectiveCatalogCandidate(
generated_at=json_datetime(generated_at),
channel=channel,
status="ready" if validation.get("valid") else "blocked",
candidate_dir=str(output_root),
catalog_path=str(catalog_path),
keyring_path=str(keyring_path),
summary_path=str(summary_path) if summary_path is not None else None,
source_catalog=str(source_catalog),
public_catalog_url=public_catalog_url,
public_keyring_url=public_keyring_url,
sequence=resolved_sequence,
signature_count=len(candidate["signatures"]) if isinstance(candidate.get("signatures"), list) else 0,
key_count=len(keyring_payload.get("keys", ())) if isinstance(keyring_payload.get("keys"), list) else 0,
candidate_catalog_hash=candidate_catalog_hash,
candidate_keyring_hash=candidate_keyring_hash,
published_catalog_hash=published_catalog_hash,
published_keyring_hash=published_keyring_hash,
candidate_matches_published_catalog=compare_hash(candidate_catalog_hash, published_catalog_hash),
candidate_matches_published_keyring=compare_hash(candidate_keyring_hash, published_keyring_hash),
validation_valid=validation.get("valid") is True,
validation_error=str(validation_error) if validation_error else None,
validation_warnings=validation_warnings,
changes=tuple(changes),
notes=(
"Candidate catalog preserves unchanged entries from the source catalog.",
f"Generated {len(module_directory_files)} module-directory file(s).",
"Publishing is intentionally separate from candidate generation.",
),
)
if summary_path is not None:
summary_path.write_text(json.dumps(dataclass_payload(result), indent=2, sort_keys=True) + "\n", encoding="utf-8")
return result
def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str:
if base_catalog is not None:
value = str(base_catalog)
return value if value.startswith(("http://", "https://")) else Path(value).expanduser()
local = web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json"
if local.exists():
return local
return f"{DEFAULT_PUBLIC_BASE_URL}/catalogs/v1/channels/{channel}.json"
def read_catalog(source: Path | str) -> object:
if isinstance(source, str) and source.startswith(("http://", "https://")):
payload = fetch_json(source)
if not payload["ok"]:
raise ValueError(f"Could not fetch catalog {source}: {payload['error']}")
return payload["payload"]
return json.loads(Path(source).read_text(encoding="utf-8"))
def next_sequence(payload: dict[str, Any], *, generated_at: datetime) -> int:
timestamp_sequence = int(generated_at.strftime("%Y%m%d%H%M"))
try:
current = int(payload.get("sequence") or 0)
except (TypeError, ValueError):
current = 0
return max(current + 1, timestamp_sequence)
def contracts_by_repo(workspace: Path) -> dict[str, Any]:
specs = load_repository_specs(include_website=False)
snapshots = tuple(
collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False)
for spec in specs
)
return {contract.repo: contract for contract in collect_contracts(snapshots)}
def apply_repo_updates(
payload: dict[str, Any],
*,
repo_versions: dict[str, str],
repo_contracts: dict[str, Any],
repository_base: str,
) -> list[CatalogEntryChange]:
changes: list[CatalogEntryChange] = []
modules = payload.get("modules")
if not isinstance(modules, list):
raise ValueError("Catalog payload has no modules list.")
handled: set[str] = set()
if "govoplan-core" in repo_versions:
version = repo_versions["govoplan-core"].removeprefix("v")
tag = f"v{version}"
core_release = payload.get("core_release")
if not isinstance(core_release, dict):
raise ValueError("Catalog payload has no core_release object.")
changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="version", value=version))
changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="python_ref", value=f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}"))
changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="webui_ref", value=f"{repository_base}/govoplan-core.git#{tag}"))
release = payload.get("release")
if isinstance(release, dict) and isinstance(release.get("version"), str):
changes.extend(update_field(release, repo="govoplan-core", module_id=None, field="version", value=version))
changes.extend(update_field(release, repo="govoplan-core", module_id=None, field="tag", value=tag))
handled.add("govoplan-core")
for entry in modules:
if not isinstance(entry, dict):
continue
repo = module_entry_repo(entry)
if repo not in repo_versions:
continue
version = repo_versions[repo].removeprefix("v")
tag = f"v{version}"
module_id = str(entry.get("module_id") or "")
package = str(entry.get("python_package") or repo)
changes.extend(update_field(entry, repo=repo, module_id=module_id, field="version", value=version))
changes.extend(update_field(entry, repo=repo, module_id=module_id, field="python_ref", value=f"{package} @ {repository_base}/{repo}.git@{tag}"))
if entry.get("webui_package"):
changes.extend(update_field(entry, repo=repo, module_id=module_id, field="webui_ref", value=f"{repository_base}/{repo}.git#{tag}"))
contract = repo_contracts.get(repo)
if contract is not None:
provider_payload = [{"name": item.name, "version": item.version} for item in contract.provides_interfaces]
requirement_payload = [
{
"name": item.name,
**({"version_min": item.version_min} if item.version_min else {}),
**({"version_max_exclusive": item.version_max_exclusive} if item.version_max_exclusive else {}),
"optional": item.optional,
}
for item in contract.requires_interfaces
]
if provider_payload:
changes.extend(update_json_field(entry, repo=repo, module_id=module_id, field="provides_interfaces", value=provider_payload))
if requirement_payload:
changes.extend(update_json_field(entry, repo=repo, module_id=module_id, field="requires_interfaces", value=requirement_payload))
handled.add(repo)
missing = sorted(set(repo_versions) - handled)
if missing:
raise ValueError(f"Selected repositories are not represented in the catalog: {', '.join(missing)}")
return changes
def module_entry_repo(entry: dict[str, Any]) -> str | None:
for field in ("python_ref", "webui_ref"):
value = entry.get(field)
if isinstance(value, str):
match = re.search(r"/([^/@#]+)[.]git(?:[@#]|$)", value)
if match:
return match.group(1)
package = entry.get("python_package")
return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None
def update_field(target: dict[str, Any], *, repo: str, module_id: str | None, field: str, value: str) -> list[CatalogEntryChange]:
before = target.get(field)
before_text = before if isinstance(before, str) else None
if before_text == value:
return []
target[field] = value
return [CatalogEntryChange(repo=repo, module_id=module_id, field=field, before=before_text, after=value)]
def update_json_field(target: dict[str, Any], *, repo: str, module_id: str | None, field: str, value: list[dict[str, object]]) -> list[CatalogEntryChange]:
before = target.get(field)
if before == value:
return []
target[field] = value
return [
CatalogEntryChange(
repo=repo,
module_id=module_id,
field=field,
before=json.dumps(before, sort_keys=True) if before is not None else None,
after=json.dumps(value, sort_keys=True),
)
]
def parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
key_id, separator, path_text = value.partition("=")
if not separator or not key_id.strip() or not path_text.strip():
raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
path = Path(path_text).expanduser()
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
if not isinstance(private_key, Ed25519PrivateKey):
raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}")
return key_id.strip(), private_key
def signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
signature_payload = dict(payload)
signature_payload.pop("signature", None)
signature_payload.pop("signatures", None)
return {
"algorithm": "ed25519",
"key_id": key_id,
"value": base64.b64encode(private_key.sign(canonical_bytes(signature_payload))).decode("ascii"),
}
def keyring_payload_for_candidate(
*,
existing_keyring: Path,
signing_keys: tuple[tuple[str, Ed25519PrivateKey], ...],
generated_at: datetime,
) -> dict[str, Any]:
expected = {key_id: public_key_base64(private_key) for key_id, private_key in signing_keys}
if existing_keyring.exists():
payload = json.loads(existing_keyring.read_text(encoding="utf-8"))
keys = payload.get("keys") if isinstance(payload, dict) else None
if isinstance(keys, list):
existing = {
str(item.get("key_id")): str(item.get("public_key") or item.get("public_key_base64"))
for item in keys
if isinstance(item, dict) and item.get("key_id")
}
if all(existing.get(key_id) == public_key for key_id, public_key in expected.items()):
return payload
return {
"keyring_version": "1",
"purpose": "govoplan module package catalog signatures",
"generated_at": json_datetime(generated_at),
"keys": [
{
"key_id": key_id,
"status": "active",
"public_key": public_key_base64(private_key),
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
}
for key_id, private_key in signing_keys
],
}
def trusted_keys_from_keyring(payload: dict[str, Any]) -> dict[str, str]:
keys = payload.get("keys")
if not isinstance(keys, list):
return {}
result: dict[str, str] = {}
for item in keys:
if not isinstance(item, dict):
continue
status = str(item.get("status") or "active").lower()
if status in {"revoked", "disabled", "retired"}:
continue
key_id = str(item.get("key_id") or "")
public_key = str(item.get("public_key") or item.get("public_key_base64") or "")
if key_id and public_key:
result[key_id] = public_key
return result
def public_key_base64(private_key: Ed25519PrivateKey) -> str:
public_bytes = private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
return base64.b64encode(public_bytes).decode("ascii")
def canonical_bytes(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def compare_hash(candidate_hash: str | None, published_hash: str | None) -> bool | None:
if candidate_hash is None or published_hash is None:
return None
return candidate_hash == published_hash
def resolve_output_dir(*, output_dir: Path | str | None, channel: str, generated_at: datetime) -> Path:
if output_dir is not None:
return Path(output_dir).expanduser()
stamp = generated_at.strftime("%Y%m%d-%H%M%S")
return Path.cwd() / "runtime" / "release-candidates" / f"{channel}-{stamp}"
def json_datetime(value: datetime) -> str:
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
def dataclass_payload(value: object) -> dict[str, object]:
from dataclasses import asdict
return asdict(value)