diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index a39abfd..8daa2cd 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -234,9 +234,16 @@ Selective catalog candidates now include this directory tree. Publishing a candidate copies the channel catalog, keyring, and `modules/` tree into the website repository together. -Current limitation: selective catalog generation updates existing catalog -entries. Initial catalog entry synthesis for brand-new modules is still a -separate backend slice. +Selective catalog generation can add a module that is not yet represented in +the source channel. Initial entries are synthesized from the selected +repository's `[project]` metadata, `govoplan.modules` entry point, and runtime +`ModuleManifest`; there is no separate hand-maintained initial-entry list. The +release gate requires the entry point to resolve inside the selected checkout, +exact package/manifest/frontend version agreement, verified source-tag +provenance, a non-conflicting module id, and complete required dependency and +interface closure in the resulting candidate. Classification beyond the +manifest-derived `official` tag remains an explicit later catalog curation +step rather than an inferred business/service label. ## Direction diff --git a/tests/test_release_catalog_entry_synthesis.py b/tests/test_release_catalog_entry_synthesis.py new file mode 100644 index 0000000..54f441a --- /dev/null +++ b/tests/test_release_catalog_entry_synthesis.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +META_ROOT = Path(__file__).resolve().parents[1] +RELEASE_ROOT = META_ROOT / "tools" / "release" +if str(RELEASE_ROOT) not in sys.path: + sys.path.insert(0, str(RELEASE_ROOT)) + +from govoplan_release.catalog_entry_synthesis import validate_initial_entry_closure # noqa: E402 +from govoplan_release.selective_catalog import apply_repo_updates # noqa: E402 + + +class ReleaseCatalogEntrySynthesisTests(unittest.TestCase): + def test_selective_update_synthesizes_initial_entries_from_package_manifests(self) -> None: + payload: dict[str, object] = { + "core_release": {}, + "release": {}, + "modules": [ + { + "module_id": "access", + "version": "0.1.11", + "python_package": "govoplan-access", + "python_ref": "govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.11", + } + ], + } + + changes = apply_repo_updates( + payload, + repo_versions={ + "govoplan-addresses": "0.1.9", + "govoplan-poll": "0.1.11", + "govoplan-scheduling": "0.1.11", + }, + repo_contracts={}, + repository_base="git+ssh://git@git.add-ideas.de/add-ideas", + workspace=META_ROOT.parent, + ) + + modules = { + item["module_id"]: item + for item in payload["modules"] # type: ignore[index] + } + self.assertEqual({"access", "addresses", "poll", "scheduling"}, set(modules)) + self.assertEqual("@govoplan/addresses-webui", modules["addresses"]["webui_package"]) + self.assertIn( + {"name": "addresses.people_search", "version": "0.1.0"}, + modules["addresses"]["provides_interfaces"], + ) + self.assertEqual("govoplan-poll", modules["poll"]["python_package"]) + self.assertEqual(["poll"], modules["scheduling"]["dependencies"]) + self.assertEqual("@govoplan/scheduling-webui", modules["scheduling"]["webui_package"]) + self.assertEqual("requires_review", modules["scheduling"]["migration_safety"]) + self.assertTrue( + any( + item["name"] == "poll.governed_participation" and item["optional"] is False + for item in modules["scheduling"]["requires_interfaces"] + ) + ) + self.assertEqual( + {"govoplan-addresses", "govoplan-poll", "govoplan-scheduling"}, + {change.repo for change in changes}, + ) + self.assertEqual({"catalog_entry"}, {change.field for change in changes}) + + def test_initial_required_dependency_must_be_represented(self) -> None: + with self.assertRaisesRegex(ValueError, "requires catalog module 'poll'"): + validate_initial_entry_closure( + catalog_modules=[ + { + "module_id": "scheduling", + "dependencies": ["poll"], + } + ], + initial_module_ids={"scheduling"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/govoplan_release/catalog_entry_synthesis.py b/tools/release/govoplan_release/catalog_entry_synthesis.py new file mode 100644 index 0000000..5f2ae9d --- /dev/null +++ b/tools/release/govoplan_release/catalog_entry_synthesis.py @@ -0,0 +1,378 @@ +"""Synthesize initial catalog entries from a repository's package and manifest.""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import importlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tomllib +from types import ModuleType +from typing import Iterator + +from govoplan_core.core.modules import ModuleManifest +from govoplan_core.core.versioning import version_satisfies_range + +from .workspace import load_repository_specs, resolve_repo_path + + +_INSPECTION_CHILD = "GOVOPLAN_CATALOG_ENTRY_INSPECTION_CHILD" +_INSPECTION_MARKER = "GOVOPLAN_CATALOG_ENTRY_JSON=" + + +def synthesize_repository_catalog_entries( + *, + repo: str, + version: str, + workspace: Path, + repository_base: str, +) -> tuple[dict[str, object], ...]: + """Build install entries from tagged, preflighted local source metadata. + + The caller owns source-tag and worktree provenance checks. This function + accepts no hand-maintained module catalog registry: distribution metadata + identifies the runtime entry point and the runtime ``ModuleManifest`` is + the authoritative dependency/interface/frontend description. + """ + + if os.getenv(_INSPECTION_CHILD) == "1": + return synthesize_repository_catalog_entries_in_process( + repo=repo, + version=version, + workspace=workspace, + repository_base=repository_base, + ) + command = ( + sys.executable, + "-m", + "govoplan_release.catalog_entry_synthesis", + "--inspect-repo", + repo, + "--version", + version, + "--workspace", + str(workspace), + "--repository-base", + repository_base, + ) + environment = os.environ.copy() + environment[_INSPECTION_CHILD] = "1" + release_root = str(Path(__file__).resolve().parents[1]) + environment["PYTHONPATH"] = os.pathsep.join( + item for item in (release_root, environment.get("PYTHONPATH", "")) if item + ) + try: + result = subprocess.run( + command, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + env=environment, + ) + except subprocess.TimeoutExpired as exc: + raise ValueError(f"Cannot synthesize {repo}: isolated manifest inspection timed out.") from exc + if result.returncode != 0: + detail = result.stderr.strip() or result.stdout.strip() or "unknown inspection error" + raise ValueError(f"Cannot synthesize {repo}: isolated manifest inspection failed: {detail}") + payload_line = next( + (line.removeprefix(_INSPECTION_MARKER) for line in reversed(result.stdout.splitlines()) if line.startswith(_INSPECTION_MARKER)), + None, + ) + if payload_line is None: + raise ValueError(f"Cannot synthesize {repo}: isolated manifest inspection returned no metadata.") + payload = json.loads(payload_line) + if not isinstance(payload, list) or not all(isinstance(item, dict) for item in payload): + raise ValueError(f"Cannot synthesize {repo}: isolated manifest inspection returned invalid metadata.") + return tuple(payload) + + +def synthesize_repository_catalog_entries_in_process( + *, + repo: str, + version: str, + workspace: Path, + repository_base: str, +) -> tuple[dict[str, object], ...]: + specs = {item.name: item for item in load_repository_specs(include_website=False)} + spec = specs.get(repo) + if spec is None: + raise ValueError(f"Cannot synthesize {repo}: repository is not registered.") + root = resolve_repo_path(spec, workspace) + project = read_project_metadata(root / "pyproject.toml") + package = required_text(project, "name", source=f"{repo}/pyproject.toml") + project_version = required_text(project, "version", source=f"{repo}/pyproject.toml").removeprefix("v") + expected_version = version.removeprefix("v") + if project_version != expected_version: + raise ValueError( + f"Cannot synthesize {repo}: project version {project_version!r} does not match selected version {expected_version!r}." + ) + description = optional_text(project.get("description")) + entry_points = module_entry_points(project, repo=repo) + entries: list[dict[str, object]] = [] + for declared_module_id, target in sorted(entry_points.items()): + manifest = load_manifest(root=root, target=target, repo=repo) + if manifest.id != declared_module_id: + raise ValueError( + f"Cannot synthesize {repo}: entry point {declared_module_id!r} returns manifest {manifest.id!r}." + ) + if manifest.version.removeprefix("v") != expected_version: + raise ValueError( + f"Cannot synthesize {repo}/{manifest.id}: manifest version {manifest.version!r} does not match {expected_version!r}." + ) + entry = manifest_catalog_entry( + manifest=manifest, + repo=repo, + package=package, + version=expected_version, + description=description, + root=root, + repository_base=repository_base.rstrip("/"), + ) + entries.append(entry) + return tuple(entries) + + +def manifest_catalog_entry( + *, + manifest: ModuleManifest, + repo: str, + package: str, + version: str, + description: str | None, + root: Path, + repository_base: str, +) -> dict[str, object]: + tag = f"v{version}" + entry: dict[str, object] = { + "module_id": manifest.id, + "name": manifest.name, + "version": version, + "action": "install", + "python_package": package, + "python_ref": f"{package} @ {repository_base}/{repo}.git@{tag}", + "license_features": [f"module.{manifest.id}"], + "tags": ["official"], + } + if description: + entry["description"] = description + if manifest.dependencies: + entry["dependencies"] = list(manifest.dependencies) + if manifest.optional_dependencies: + entry["optional_dependencies"] = list(manifest.optional_dependencies) + if manifest.provides_interfaces: + entry["provides_interfaces"] = [ + {"name": item.name, "version": item.version} + for item in manifest.provides_interfaces + ] + if manifest.requires_interfaces: + entry["requires_interfaces"] = [ + { + "name": item.name, + **({"version_min": item.version_min} if item.version_min else {}), + **({"version_max_exclusive": item.version_max_exclusive} if item.version_max_exclusive else {}), + "optional": item.optional, + } + for item in manifest.requires_interfaces + ] + if manifest.frontend is not None and manifest.frontend.package_name: + validate_webui_package(root=root, package=manifest.frontend.package_name, version=version, repo=repo) + entry["webui_package"] = manifest.frontend.package_name + entry["webui_ref"] = f"{repository_base}/{repo}.git#{tag}" + if manifest.migration_spec is not None: + migration = manifest.migration_spec + entry["migration_safety"] = "requires_review" + entry["migration_notes"] = ( + "Module owns database migrations; review release notes and migration output before activation." + ) + if migration.migration_after: + entry["migration_after"] = list(migration.migration_after) + if migration.migration_before: + entry["migration_before"] = list(migration.migration_before) + if migration.migration_tasks: + entry["migration_tasks"] = [ + { + "task_id": task.task_id, + "phase": task.phase, + "summary": task.summary, + "task_version": task.task_version, + "safety": task.safety, + "idempotent": task.idempotent, + **({"timeout_seconds": task.timeout_seconds} if task.timeout_seconds is not None else {}), + } + for task in migration.migration_tasks + ] + return entry + + +def validate_initial_entry_closure( + *, + catalog_modules: list[object], + initial_module_ids: set[str], +) -> None: + entries = [item for item in catalog_modules if isinstance(item, dict)] + by_id = {str(item.get("module_id") or ""): item for item in entries} + providers: dict[str, list[str]] = {} + for entry in entries: + for raw in entry.get("provides_interfaces") or (): + if isinstance(raw, dict) and isinstance(raw.get("name"), str) and isinstance(raw.get("version"), str): + providers.setdefault(raw["name"], []).append(raw["version"]) + + issues: list[str] = [] + for module_id in sorted(initial_module_ids): + entry = by_id[module_id] + for dependency in entry.get("dependencies") or (): + if isinstance(dependency, str) and dependency not in by_id: + issues.append(f"{module_id} requires catalog module {dependency!r}") + for raw in entry.get("requires_interfaces") or (): + if not isinstance(raw, dict) or raw.get("optional") is True: + continue + name = raw.get("name") + if not isinstance(name, str): + continue + available = providers.get(name, []) + if not any( + version_satisfies_range( + provider_version, + version_min=raw.get("version_min") if isinstance(raw.get("version_min"), str) else None, + version_max_exclusive=raw.get("version_max_exclusive") + if isinstance(raw.get("version_max_exclusive"), str) + else None, + ) + for provider_version in available + ): + issues.append(f"{module_id} requires unavailable compatible interface {name!r}") + if issues: + raise ValueError("Initial catalog entry dependency closure failed: " + "; ".join(issues)) + + +def read_project_metadata(path: Path) -> dict[str, object]: + if not path.exists(): + raise ValueError(f"Cannot synthesize catalog entry: missing {path}.") + with path.open("rb") as handle: + payload = tomllib.load(handle) + project = payload.get("project") + if not isinstance(project, dict): + raise ValueError(f"Cannot synthesize catalog entry: {path} has no [project] table.") + return project + + +def module_entry_points(project: dict[str, object], *, repo: str) -> dict[str, str]: + groups = project.get("entry-points") + raw = groups.get("govoplan.modules") if isinstance(groups, dict) else None + if not isinstance(raw, dict) or not raw: + raise ValueError(f"Cannot synthesize {repo}: pyproject has no govoplan.modules entry point.") + result = { + str(module_id).strip(): str(target).strip() + for module_id, target in raw.items() + if str(module_id).strip() and str(target).strip() + } + if len(result) != len(raw): + raise ValueError(f"Cannot synthesize {repo}: govoplan.modules contains an empty id or target.") + return result + + +def load_manifest(*, root: Path, target: str, repo: str) -> ModuleManifest: + module_name, separator, attribute = target.partition(":") + if not separator or not module_name or not attribute or "[" in attribute: + raise ValueError(f"Cannot synthesize {repo}: unsupported module entry point {target!r}.") + src = (root / "src").resolve() + expected_module = (src / Path(*module_name.split("."))).with_suffix(".py") + expected_package = src / Path(*module_name.split(".")) / "__init__.py" + expected = expected_module if expected_module.exists() else expected_package + if not expected.exists(): + raise ValueError(f"Cannot synthesize {repo}: entry point source {expected_module} is missing.") + + top_level = module_name.split(".", 1)[0] + with isolated_source_import(src=src, top_level=top_level): + try: + module = importlib.import_module(module_name) + except Exception as exc: # noqa: BLE001 - exact load failure is release evidence. + raise ValueError(f"Cannot synthesize {repo}: loading {target!r} failed: {exc}") from exc + origin = Path(str(getattr(module, "__file__", ""))).resolve() + if origin != expected.resolve(): + raise ValueError(f"Cannot synthesize {repo}: entry point resolved outside selected source ({origin}).") + factory = getattr(module, attribute, None) + if not callable(factory): + raise ValueError(f"Cannot synthesize {repo}: entry point target {target!r} is not callable.") + manifest = factory() + if not isinstance(manifest, ModuleManifest): + raise ValueError(f"Cannot synthesize {repo}: entry point {target!r} did not return ModuleManifest.") + return manifest + + +@contextmanager +def isolated_source_import(*, src: Path, top_level: str) -> Iterator[None]: + saved: dict[str, ModuleType] = { + name: module + for name, module in tuple(sys.modules.items()) + if name == top_level or name.startswith(f"{top_level}.") + } + for name in saved: + sys.modules.pop(name, None) + sys.path.insert(0, str(src)) + importlib.invalidate_caches() + try: + yield + finally: + sys.path.remove(str(src)) + for name in tuple(sys.modules): + if name == top_level or name.startswith(f"{top_level}."): + sys.modules.pop(name, None) + sys.modules.update(saved) + importlib.invalidate_caches() + + +def validate_webui_package(*, root: Path, package: str, version: str, repo: str) -> None: + declarations: list[tuple[str, str]] = [] + for path in (root / "package.json", root / "webui" / "package.json"): + if not path.exists(): + continue + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, dict) and isinstance(payload.get("name"), str) and isinstance(payload.get("version"), str): + declarations.append((payload["name"], payload["version"])) + if (package, version) not in declarations: + raise ValueError( + f"Cannot synthesize {repo}: frontend manifest declares {package}@{version}, but package metadata does not." + ) + + +def required_text(payload: dict[str, object], key: str, *, source: str) -> str: + value = optional_text(payload.get(key)) + if value is None: + raise ValueError(f"Cannot synthesize catalog entry: {source} has no {key!r}.") + return value + + +def optional_text(value: object) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def main() -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--inspect-repo", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--repository-base", required=True) + args = parser.parse_args() + try: + entries = synthesize_repository_catalog_entries( + repo=args.inspect_repo, + version=args.version, + workspace=args.workspace.resolve(), + repository_base=args.repository_base, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(_INSPECTION_MARKER + json.dumps(entries, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/govoplan_release/contracts.py b/tools/release/govoplan_release/contracts.py index 7ab1621..35ec568 100644 --- a/tools/release/govoplan_release/contracts.py +++ b/tools/release/govoplan_release/contracts.py @@ -331,6 +331,11 @@ def import_source_path(path: Path, *, module: str | None, level: int) -> Path | if src_root is None or module is None: return None candidate = src_root.joinpath(*module.split(".")).with_suffix(".py") + if not candidate.exists(): + top_level = module.split(".", 1)[0] + workspace = src_root.parent.parent + sibling_src = workspace / top_level.replace("_", "-") / "src" + candidate = sibling_src.joinpath(*module.split(".")).with_suffix(".py") if candidate.exists(): return candidate init = candidate.with_suffix("") / "__init__.py" diff --git a/tools/release/govoplan_release/selective_catalog.py b/tools/release/govoplan_release/selective_catalog.py index c65325d..5031435 100644 --- a/tools/release/govoplan_release/selective_catalog.py +++ b/tools/release/govoplan_release/selective_catalog.py @@ -15,6 +15,10 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from govoplan_core.core.module_package_catalog import validate_module_package_catalog from .catalog import DEFAULT_PUBLIC_BASE_URL, canonical_hash, fetch_json +from .catalog_entry_synthesis import ( + synthesize_repository_catalog_entries, + validate_initial_entry_closure, +) from .contracts import collect_contracts from .git_state import collect_repository_snapshot from .model import CatalogEntryChange, SelectiveCatalogCandidate @@ -98,6 +102,7 @@ def build_selective_catalog_candidate( repo_versions=repo_versions, repo_contracts=repo_contracts, repository_base=repository_base.rstrip("/"), + workspace=workspace, ) enforce_complete_catalog_source_provenance( payload=candidate, @@ -290,6 +295,7 @@ def apply_repo_updates( repo_versions: dict[str, str], repo_contracts: dict[str, Any], repository_base: str, + workspace: Path, ) -> list[CatalogEntryChange]: changes: list[CatalogEntryChange] = [] modules = payload.get("modules") @@ -345,8 +351,43 @@ def apply_repo_updates( handled.add(repo) missing = sorted(set(repo_versions) - handled) - if missing: - raise ValueError(f"Selected repositories are not represented in the catalog: {', '.join(missing)}") + initial_module_ids: set[str] = set() + existing_module_ids = { + str(entry.get("module_id") or "") + for entry in modules + if isinstance(entry, dict) + } + for repo in missing: + entries = synthesize_repository_catalog_entries( + repo=repo, + version=repo_versions[repo], + workspace=workspace, + repository_base=repository_base, + ) + for entry in entries: + module_id = str(entry["module_id"]) + if module_id in existing_module_ids: + raise ValueError( + f"Cannot synthesize {repo}: module id {module_id!r} already belongs to another catalog entry." + ) + modules.append(entry) + existing_module_ids.add(module_id) + initial_module_ids.add(module_id) + changes.append( + CatalogEntryChange( + repo=repo, + module_id=module_id, + field="catalog_entry", + before=None, + after=json.dumps(entry, sort_keys=True), + ) + ) + handled.add(repo) + if initial_module_ids: + validate_initial_entry_closure( + catalog_modules=modules, + initial_module_ids=initial_module_ids, + ) return changes diff --git a/tools/release/govoplan_release/selective_planner.py b/tools/release/govoplan_release/selective_planner.py index 3f34d59..c80206d 100644 --- a/tools/release/govoplan_release/selective_planner.py +++ b/tools/release/govoplan_release/selective_planner.py @@ -196,16 +196,16 @@ def dry_run_steps(*, units: tuple[ReleasePlanUnit, ...], dashboard: ReleaseDashb if initial_units: steps.append( ReleasePlanStep( - id="catalog:initial-entry-synthesis", - title=f"Prepare initial {channel} catalog entries", + id="catalog:initial-version-metadata", + title=f"Prepare version metadata before adding to {channel}", detail=( - "Selected initial-release repositories are not guaranteed to exist in the channel catalog yet: " + "Selected repositories have no source version metadata: " + ", ".join(unit.repo for unit in initial_units) - + ". The current selective catalog writer updates existing entries only." + + ". Initial catalog synthesis requires aligned pyproject, module manifest, and optional frontend versions." ), cwd=dashboard.meta_root, mutating=True, - status="needs-catalog-entry-synthesis", + status="needs-version-metadata", ) ) else: