From 78811f7f6e569a82bdf186f5242400c6926c7001 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 6 Aug 2026 22:42:25 +0200 Subject: [PATCH] Synchronize public module directory publication --- docs/PACKAGE_REGISTRY_RELEASES.md | 16 ++++ ...STEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md | 6 +- tests/test_release_entrypoint_gates.py | 7 ++ tests/test_release_module_directory.py | 75 +++++++++++++++++++ tools/release/generate-release-catalog.py | 31 +++++++- .../govoplan_release/module_directory.py | 54 ++++++++++++- .../govoplan_release/release_intelligence.py | 13 +++- tools/release/publish-release-catalog.sh | 5 +- 8 files changed, 197 insertions(+), 10 deletions(-) diff --git a/docs/PACKAGE_REGISTRY_RELEASES.md b/docs/PACKAGE_REGISTRY_RELEASES.md index 8192a17..e058376 100644 --- a/docs/PACKAGE_REGISTRY_RELEASES.md +++ b/docs/PACKAGE_REGISTRY_RELEASES.md @@ -186,6 +186,15 @@ the current workspace. Each entry binds its Python wheel and optional WebUI tarball to the registry URL, filename, size, SHA-256, package identity, source tag, and source commit before the complete catalog is signed. +The same publication transaction regenerates and prunes the browsable static +directory under `public/catalogs/v1/modules/`. It writes a global +`modules/index.json`, one `/index.json`, and one +`//manifest.json` for every entry in the signed channel. +These files are derived from that exact signed payload and keyring; stale JSON +from an older partial catalog is removed while unrelated static assets are left +untouched. The signed channel remains the trust anchor, while the module +directory provides stable discovery URLs for browsers and external tooling. + Official GovOPlaN modules are open-source directory entries and do not require license entitlements. The generic `license_features` contract remains available for third-party package directories, support/configuration packages, or future @@ -201,6 +210,13 @@ cache. A saved plan is rejected if any package ref, artifact identity, catalog channel, sequence, or signing-key identity differs from the currently validated catalog. +The Admin directory can be searched by module, package, repository, or tag and +filtered by available, installed, update, and blocked/withdrawn states. It +shows the source revision, artifact digest, release notes, and configuration +requirements. Missing dependency/interface providers and unsupported update +windows are surfaced before an operator adds the entry to a plan; installer +preflight remains authoritative. + Package lifecycle and availability are intentionally separate: - install, update, and uninstall change the instance-wide package composition; diff --git a/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md b/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md index aad5b17..4a0ee9e 100644 --- a/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md +++ b/docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md @@ -172,8 +172,10 @@ Implementation status as of the current source tree: after durable administrator access is established. 3. **Read-only online module directory (implemented foundation).** Admin falls back to the signed public stable directory, presents installed/update state, - compatibility and provenance, and retains operator-configured catalogs as an - explicit override. + searchable availability/blocker filters, immutable source/artifact + provenance, configuration requirements, release notes, and technical + compatibility. Withdrawn releases remain visible but cannot be planned. + Operator-configured catalogs remain an explicit override. 4. **Durable module plan and install (implemented local boundary).** Catalog selection creates a reviewed plan; the installer queue, lock, preflight, maintenance gate, digest-verified artifact cache, rollback drill, and run diff --git a/tests/test_release_entrypoint_gates.py b/tests/test_release_entrypoint_gates.py index 7a94126..b7ac51c 100644 --- a/tests/test_release_entrypoint_gates.py +++ b/tests/test_release_entrypoint_gates.py @@ -103,6 +103,13 @@ class ReleaseEntrypointGateTests(unittest.TestCase): self.assertLess(gate, write) + def test_full_catalog_publication_synchronizes_browsable_module_directory(self) -> None: + publisher = (META_ROOT / "tools" / "release" / "publish-release-catalog.sh").read_text() + + self.assertIn('--module-directory-output "$WEB_ROOT/public/catalogs/v1"', publisher) + self.assertIn('git -C "$WEB_ROOT" add -A', publisher) + self.assertIn('"$MODULE_DIRECTORY_PATH"', publisher) + def test_candidate_publication_uses_existing_keyring_as_trust_anchor(self) -> None: publisher = (META_ROOT / "tools" / "release" / "govoplan_release" / "publisher.py").read_text() diff --git a/tests/test_release_module_directory.py b/tests/test_release_module_directory.py index 8775e34..2ebf929 100644 --- a/tests/test_release_module_directory.py +++ b/tests/test_release_module_directory.py @@ -1,7 +1,9 @@ from __future__ import annotations +import json from pathlib import Path import sys +import tempfile import unittest from unittest import mock @@ -14,6 +16,7 @@ if str(RELEASE_TOOLS_ROOT) not in sys.path: from govoplan_release.module_directory import ( # noqa: E402 module_directory_payloads, safe_path_part, + write_module_directory, ) @@ -77,6 +80,78 @@ class ReleaseModuleDirectoryTests(unittest.TestCase): catalog_payload={}, keyring_payload={}, channel="../stable" ) + def test_prune_removes_stale_json_but_keeps_unrelated_assets(self) -> None: + catalog = { + "generated_at": "2026-08-06T12:00:00Z", + "sequence": 8, + "modules": [ + { + "module_id": "files", + "name": "Files", + "version": "1.2.3", + "python_package": "govoplan-files", + "source": { + "repository": "govoplan-files", + "tag": "v1.2.3", + "commit": "a" * 40, + }, + "artifact_integrity": { + "python": {"sha256": "b" * 64}, + }, + } + ], + } + with tempfile.TemporaryDirectory() as value: + output_root = Path(value) + stale = output_root / "modules" / "legacy" / "0.1.0" / "manifest.json" + stale.parent.mkdir(parents=True) + stale.write_text("{}\n", encoding="utf-8") + unrelated = output_root / "modules" / "README.txt" + unrelated.write_text("keep\n", encoding="utf-8") + + written = write_module_directory( + catalog_payload=catalog, + keyring_payload={}, + output_root=output_root, + channel="stable", + prune=True, + ) + + self.assertFalse(stale.exists()) + self.assertTrue(unrelated.exists()) + self.assertEqual(3, len(written)) + manifest = json.loads( + (output_root / "modules" / "files" / "1.2.3" / "manifest.json").read_text() + ) + self.assertEqual("govoplan-files", manifest["module"]["repo"]) + self.assertEqual("v1.2.3", manifest["module"]["python_tag"]) + self.assertEqual("b" * 64, manifest["module"]["artifact_integrity"]["python"]["sha256"]) + + def test_writer_refuses_nested_symlink_targets(self) -> None: + catalog = { + "modules": [{"module_id": "files", "version": "1.2.3"}], + } + with tempfile.TemporaryDirectory() as value: + root = Path(value) + output_root = root / "public" + external = root / "external" + (output_root / "modules").mkdir(parents=True) + external.mkdir() + (output_root / "modules" / "files").symlink_to( + external, + target_is_directory=True, + ) + + with self.assertRaisesRegex(ValueError, "symlinks"): + write_module_directory( + catalog_payload=catalog, + keyring_payload={}, + output_root=output_root, + channel="stable", + ) + + self.assertEqual([], list(external.iterdir())) + if __name__ == "__main__": unittest.main() diff --git a/tools/release/generate-release-catalog.py b/tools/release/generate-release-catalog.py index 2f646a6..6e2819f 100644 --- a/tools/release/generate-release-catalog.py +++ b/tools/release/generate-release-catalog.py @@ -28,6 +28,7 @@ from govoplan_release.catalog_entry_synthesis import ( # noqa: E402 synthesize_repository_catalog_entries, validate_initial_entry_closure, ) +from govoplan_release.module_directory import write_module_directory # noqa: E402 SHA256 = re.compile(r"^[0-9a-f]{64}$") @@ -44,6 +45,11 @@ def main() -> int: parser.add_argument("--expires-days", type=int, default=90) parser.add_argument("--catalog-output", type=Path, required=True) parser.add_argument("--keyring-output", type=Path) + parser.add_argument( + "--module-directory-output", + type=Path, + help="Catalog v1 root under which the browsable modules directory is synchronized.", + ) parser.add_argument( "--catalog-signing-key", action="append", @@ -84,17 +90,32 @@ def main() -> int: output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8") + keyring = _keyring(signing_keys=signing_keys, generated_at=generated_at) if args.keyring_output is not None: keyring_output = args.keyring_output.expanduser() keyring_output.parent.mkdir(parents=True, exist_ok=True) keyring_output.write_text( - json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n", + json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) + module_directory_files: tuple[Path, ...] = () + if args.module_directory_output is not None: + module_directory_files = write_module_directory( + catalog_payload=catalog, + keyring_payload=keyring, + output_root=args.module_directory_output.expanduser(), + channel=args.channel, + public_base_url=args.public_base_url, + prune=True, + ) + print(f"catalog={output}") if args.keyring_output is not None: print(f"keyring={args.keyring_output.expanduser()}") + if args.module_directory_output is not None: + print(f"module_directory={args.module_directory_output.expanduser()}") + print(f"module_directory_files={len(module_directory_files)}") print(f"channel={args.channel}") print(f"sequence={sequence}") print(f"version={version}") @@ -164,11 +185,17 @@ def _catalog_payload( for entry in entries: python_ref = _python_ref(name, python_artifact) entry["python_ref"] = python_ref + repository_url = f"https://git.add-ideas.de/GovOPlaN/{repository}" + source_commit = str(package["commit"]) entry["source"] = { "repository": repository, "tag": package["tag"], - "commit": package["commit"], + "commit": source_commit, + "repository_url": repository_url, + "revision_url": f"{repository_url}/commit/{source_commit}", } + entry["availability"] = "available" + entry["release_notes_url"] = f"{repository_url}/releases/tag/{package['tag']}" integrity: dict[str, object] = { "python": _artifact_integrity(python_artifact, ref=python_ref), } diff --git a/tools/release/govoplan_release/module_directory.py b/tools/release/govoplan_release/module_directory.py index f18745f..50deaa0 100644 --- a/tools/release/govoplan_release/module_directory.py +++ b/tools/release/govoplan_release/module_directory.py @@ -20,6 +20,7 @@ def write_module_directory( output_root: Path, channel: str, public_base_url: str = DEFAULT_PUBLIC_BASE_URL, + prune: bool = False, ) -> tuple[Path, ...]: payloads = module_directory_payloads( catalog_payload=catalog_payload, @@ -29,13 +30,62 @@ def write_module_directory( ) written: list[Path] = [] for relative_path, payload in payloads: - path = output_root / relative_path - path.parent.mkdir(parents=True, exist_ok=True) + path = _prepare_output_path(output_root=output_root, relative_path=relative_path) path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") written.append(path) + if prune: + _prune_stale_module_directory_files( + output_root=output_root, + expected={relative_path for relative_path, _payload in payloads}, + ) return tuple(written) +def _prepare_output_path(*, output_root: Path, relative_path: Path) -> Path: + if output_root.is_symlink(): + raise ValueError("module-directory output root must not be a symlink") + output_root.mkdir(parents=True, exist_ok=True) + current = output_root + for part in relative_path.parent.parts: + current = current / part + if current.is_symlink(): + raise ValueError("module-directory path must not contain symlinks") + if current.exists(): + if not current.is_dir(): + raise ValueError("module-directory parent path must be a directory") + continue + current.mkdir() + path = output_root / relative_path + if path.is_symlink() or (path.exists() and not path.is_file()): + raise ValueError("module-directory output path must be a regular file") + return path + + +def _prune_stale_module_directory_files( + *, + output_root: Path, + expected: set[Path], +) -> None: + module_root = output_root / "modules" + if module_root.is_symlink(): + raise ValueError("module-directory root must not be a symlink") + if not module_root.exists(): + return + for path in module_root.rglob("*.json"): + relative_path = path.relative_to(output_root) + if relative_path not in expected: + path.unlink() + for path in sorted( + (item for item in module_root.rglob("*") if item.is_dir()), + key=lambda item: len(item.parts), + reverse=True, + ): + try: + path.rmdir() + except OSError: + pass + + def module_directory_payloads( *, catalog_payload: dict[str, Any], diff --git a/tools/release/govoplan_release/release_intelligence.py b/tools/release/govoplan_release/release_intelligence.py index 473c4a1..066f9aa 100644 --- a/tools/release/govoplan_release/release_intelligence.py +++ b/tools/release/govoplan_release/release_intelligence.py @@ -185,17 +185,24 @@ def module_rows(payload: object) -> list[dict[str, object]]: for item in raw_modules: if not isinstance(item, dict): continue - repo = module_repo(item) + source = item.get("source") if isinstance(item.get("source"), dict) else {} + repo = string(source.get("repository")) or module_repo(item) + source_tag = string(source.get("tag")) modules.append( { "module_id": string(item.get("module_id")), "name": string(item.get("name")), + "description": string(item.get("description")), "version": string(item.get("version")), "repo": repo, + "source": dict(source), "python_ref": string(item.get("python_ref")), - "python_tag": ref_tag(string(item.get("python_ref"))), + "python_tag": source_tag or ref_tag(string(item.get("python_ref"))), "webui_ref": string(item.get("webui_ref")), - "webui_tag": ref_tag(string(item.get("webui_ref"))), + "webui_tag": source_tag or ref_tag(string(item.get("webui_ref"))), + "artifact_integrity": dict(item.get("artifact_integrity")) if isinstance(item.get("artifact_integrity"), dict) else {}, + "dependencies": tuple(str(value) for value in item.get("dependencies", ()) if isinstance(value, str)) if isinstance(item.get("dependencies"), list) else (), + "optional_dependencies": tuple(str(value) for value in item.get("optional_dependencies", ()) if isinstance(value, str)) if isinstance(item.get("optional_dependencies"), list) else (), "provides_interfaces": interface_list(item.get("provides_interfaces")), "requires_interfaces": requirement_list(item.get("requires_interfaces")), "migration_safety": string(item.get("migration_safety")), diff --git a/tools/release/publish-release-catalog.sh b/tools/release/publish-release-catalog.sh index 8c90ec6..bc39740 100644 --- a/tools/release/publish-release-catalog.sh +++ b/tools/release/publish-release-catalog.sh @@ -187,6 +187,7 @@ fi CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json" KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json" +MODULE_DIRECTORY_PATH="$WEB_ROOT/public/catalogs/v1/modules" TAG_NAME="catalog-v$VERSION" TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-catalog.XXXXXXXX")" PACKAGE_SET="$TEMP_ROOT/release-packages.json" @@ -213,6 +214,7 @@ GEN_ARGS=( --expires-days "$EXPIRES_DAYS" --catalog-output "$CATALOG_PATH" --keyring-output "$KEYRING_PATH" + --module-directory-output "$WEB_ROOT/public/catalogs/v1" --public-base-url "$PUBLIC_BASE_URL" ) if [[ -n "$SEQUENCE" ]]; then @@ -252,7 +254,7 @@ if [[ "$BUILD_WEB" -eq 1 ]]; then fi if [[ "$COMMIT" -eq 1 ]]; then - run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH" + run git -C "$WEB_ROOT" add -A "$CATALOG_PATH" "$KEYRING_PATH" "$MODULE_DIRECTORY_PATH" if [[ "$DRY_RUN" -eq 0 ]]; then if git -C "$WEB_ROOT" diff --cached --quiet; then echo "No addideas-govoplan-website catalog changes to commit." @@ -282,4 +284,5 @@ fi echo "Catalog ready:" echo " $CATALOG_PATH" echo " $KEYRING_PATH" +echo " $MODULE_DIRECTORY_PATH" echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"