Synchronize public module directory publication

tools/release/generate-release-catalog.py

@@ -28,6 +28,7 @@ from govoplan_release.catalog_entry_synthesis import (  # noqa: E402
     synthesize_repository_catalog_entries,
     validate_initial_entry_closure,
 )
+from govoplan_release.module_directory import write_module_directory  # noqa: E402
 
 
 SHA256 = re.compile(r"^[0-9a-f]{64}$")
@@ -44,6 +45,11 @@ def main() -> int:
     parser.add_argument("--expires-days", type=int, default=90)
     parser.add_argument("--catalog-output", type=Path, required=True)
     parser.add_argument("--keyring-output", type=Path)
+    parser.add_argument(
+        "--module-directory-output",
+        type=Path,
+        help="Catalog v1 root under which the browsable modules directory is synchronized.",
+    )
     parser.add_argument(
         "--catalog-signing-key",
         action="append",
@@ -84,17 +90,32 @@ def main() -> int:
     output.parent.mkdir(parents=True, exist_ok=True)
     output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
 
+    keyring = _keyring(signing_keys=signing_keys, generated_at=generated_at)
     if args.keyring_output is not None:
         keyring_output = args.keyring_output.expanduser()
         keyring_output.parent.mkdir(parents=True, exist_ok=True)
         keyring_output.write_text(
-            json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
+            json.dumps(keyring, indent=2, sort_keys=True) + "\n",
             encoding="utf-8",
         )
 
+    module_directory_files: tuple[Path, ...] = ()
+    if args.module_directory_output is not None:
+        module_directory_files = write_module_directory(
+            catalog_payload=catalog,
+            keyring_payload=keyring,
+            output_root=args.module_directory_output.expanduser(),
+            channel=args.channel,
+            public_base_url=args.public_base_url,
+            prune=True,
+        )
+
     print(f"catalog={output}")
     if args.keyring_output is not None:
         print(f"keyring={args.keyring_output.expanduser()}")
+    if args.module_directory_output is not None:
+        print(f"module_directory={args.module_directory_output.expanduser()}")
+        print(f"module_directory_files={len(module_directory_files)}")
     print(f"channel={args.channel}")
     print(f"sequence={sequence}")
     print(f"version={version}")
@@ -164,11 +185,17 @@ def _catalog_payload(
         for entry in entries:
             python_ref = _python_ref(name, python_artifact)
             entry["python_ref"] = python_ref
+            repository_url = f"https://git.add-ideas.de/GovOPlaN/{repository}"
+            source_commit = str(package["commit"])
             entry["source"] = {
                 "repository": repository,
                 "tag": package["tag"],
-                "commit": package["commit"],
+                "commit": source_commit,
+                "repository_url": repository_url,
+                "revision_url": f"{repository_url}/commit/{source_commit}",
             }
+            entry["availability"] = "available"
+            entry["release_notes_url"] = f"{repository_url}/releases/tag/{package['tag']}"
             integrity: dict[str, object] = {
                 "python": _artifact_integrity(python_artifact, ref=python_ref),
             }

tools/release/govoplan_release/module_directory.py

@@ -20,6 +20,7 @@ def write_module_directory(
     output_root: Path,
     channel: str,
     public_base_url: str = DEFAULT_PUBLIC_BASE_URL,
+    prune: bool = False,
 ) -> tuple[Path, ...]:
     payloads = module_directory_payloads(
         catalog_payload=catalog_payload,
@@ -29,13 +30,62 @@ def write_module_directory(
     )
     written: list[Path] = []
     for relative_path, payload in payloads:
-        path = output_root / relative_path
-        path.parent.mkdir(parents=True, exist_ok=True)
+        path = _prepare_output_path(output_root=output_root, relative_path=relative_path)
         path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
         written.append(path)
+    if prune:
+        _prune_stale_module_directory_files(
+            output_root=output_root,
+            expected={relative_path for relative_path, _payload in payloads},
+        )
     return tuple(written)
 
 
+def _prepare_output_path(*, output_root: Path, relative_path: Path) -> Path:
+    if output_root.is_symlink():
+        raise ValueError("module-directory output root must not be a symlink")
+    output_root.mkdir(parents=True, exist_ok=True)
+    current = output_root
+    for part in relative_path.parent.parts:
+        current = current / part
+        if current.is_symlink():
+            raise ValueError("module-directory path must not contain symlinks")
+        if current.exists():
+            if not current.is_dir():
+                raise ValueError("module-directory parent path must be a directory")
+            continue
+        current.mkdir()
+    path = output_root / relative_path
+    if path.is_symlink() or (path.exists() and not path.is_file()):
+        raise ValueError("module-directory output path must be a regular file")
+    return path
+
+
+def _prune_stale_module_directory_files(
+    *,
+    output_root: Path,
+    expected: set[Path],
+) -> None:
+    module_root = output_root / "modules"
+    if module_root.is_symlink():
+        raise ValueError("module-directory root must not be a symlink")
+    if not module_root.exists():
+        return
+    for path in module_root.rglob("*.json"):
+        relative_path = path.relative_to(output_root)
+        if relative_path not in expected:
+            path.unlink()
+    for path in sorted(
+        (item for item in module_root.rglob("*") if item.is_dir()),
+        key=lambda item: len(item.parts),
+        reverse=True,
+    ):
+        try:
+            path.rmdir()
+        except OSError:
+            pass
+
+
 def module_directory_payloads(
     *,
     catalog_payload: dict[str, Any],

tools/release/govoplan_release/release_intelligence.py

@@ -185,17 +185,24 @@ def module_rows(payload: object) -> list[dict[str, object]]:
     for item in raw_modules:
         if not isinstance(item, dict):
             continue
-        repo = module_repo(item)
+        source = item.get("source") if isinstance(item.get("source"), dict) else {}
+        repo = string(source.get("repository")) or module_repo(item)
+        source_tag = string(source.get("tag"))
         modules.append(
             {
                 "module_id": string(item.get("module_id")),
                 "name": string(item.get("name")),
+                "description": string(item.get("description")),
                 "version": string(item.get("version")),
                 "repo": repo,
+                "source": dict(source),
                 "python_ref": string(item.get("python_ref")),
-                "python_tag": ref_tag(string(item.get("python_ref"))),
+                "python_tag": source_tag or ref_tag(string(item.get("python_ref"))),
                 "webui_ref": string(item.get("webui_ref")),
-                "webui_tag": ref_tag(string(item.get("webui_ref"))),
+                "webui_tag": source_tag or ref_tag(string(item.get("webui_ref"))),
+                "artifact_integrity": dict(item.get("artifact_integrity")) if isinstance(item.get("artifact_integrity"), dict) else {},
+                "dependencies": tuple(str(value) for value in item.get("dependencies", ()) if isinstance(value, str)) if isinstance(item.get("dependencies"), list) else (),
+                "optional_dependencies": tuple(str(value) for value in item.get("optional_dependencies", ()) if isinstance(value, str)) if isinstance(item.get("optional_dependencies"), list) else (),
                 "provides_interfaces": interface_list(item.get("provides_interfaces")),
                 "requires_interfaces": requirement_list(item.get("requires_interfaces")),
                 "migration_safety": string(item.get("migration_safety")),

tools/release/publish-release-catalog.sh

@@ -187,6 +187,7 @@ fi
 
 CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
 KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
+MODULE_DIRECTORY_PATH="$WEB_ROOT/public/catalogs/v1/modules"
 TAG_NAME="catalog-v$VERSION"
 TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-catalog.XXXXXXXX")"
 PACKAGE_SET="$TEMP_ROOT/release-packages.json"
@@ -213,6 +214,7 @@ GEN_ARGS=(
   --expires-days "$EXPIRES_DAYS"
   --catalog-output "$CATALOG_PATH"
   --keyring-output "$KEYRING_PATH"
+  --module-directory-output "$WEB_ROOT/public/catalogs/v1"
   --public-base-url "$PUBLIC_BASE_URL"
 )
 if [[ -n "$SEQUENCE" ]]; then
@@ -252,7 +254,7 @@ if [[ "$BUILD_WEB" -eq 1 ]]; then
 fi
 
 if [[ "$COMMIT" -eq 1 ]]; then
-  run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
+  run git -C "$WEB_ROOT" add -A "$CATALOG_PATH" "$KEYRING_PATH" "$MODULE_DIRECTORY_PATH"
   if [[ "$DRY_RUN" -eq 0 ]]; then
     if git -C "$WEB_ROOT" diff --cached --quiet; then
       echo "No addideas-govoplan-website catalog changes to commit."
@@ -282,4 +284,5 @@ fi
 echo "Catalog ready:"
 echo "  $CATALOG_PATH"
 echo "  $KEYRING_PATH"
+echo "  $MODULE_DIRECTORY_PATH"
 echo "  URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"