#!/usr/bin/env python3 """Extract a source-derived GovOPlaN UI, translation, module, and API inventory.""" from __future__ import annotations import argparse import ast from collections import Counter from dataclasses import asdict, is_dataclass import importlib import json from pathlib import Path import re import subprocess import sys from typing import Any META_ROOT = Path(__file__).resolve().parents[2] HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"} PATH_PARAMETER = re.compile(r"\$\{[^{}]*\}|\{[^{}]+\}") def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--output-dir", type=Path, default=META_ROOT / "audit-reports" / "platform-inventory", ) parser.add_argument( "--strict", action="store_true", help="Fail when a used translation key is absent from a generated locale catalog.", ) args = parser.parse_args() catalog = json.loads( (META_ROOT / "repositories.json").read_text(encoding="utf-8") ) workspace_root = Path(catalog["default_parent"]).resolve() webui = _extract_webui() backend_endpoints = _extract_backend_endpoints(catalog, workspace_root) manifests = _extract_manifests(catalog, workspace_root) inventory = _assemble_inventory( webui=webui, backend_endpoints=backend_endpoints, manifests=manifests, ) output_dir = args.output_dir.resolve() output_dir.mkdir(parents=True, exist_ok=True) json_path = output_dir / "platform-interface-inventory.json" markdown_path = output_dir / "platform-interface-inventory.md" json_path.write_text( json.dumps(inventory, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) markdown_path.write_text(_render_markdown(inventory), encoding="utf-8") print(f"Platform inventory JSON: {json_path}") print(f"Platform inventory summary: {markdown_path}") if args.strict and inventory["translation_health"]["missing_catalog_entries"]: print( "Used translation keys are missing from generated catalogs.", file=sys.stderr, ) return 1 return 0 def _extract_webui() -> dict[str, Any]: helper = META_ROOT / "tools" / "inventory" / "extract-webui-structure.mjs" completed = subprocess.run( ["node", str(helper), str(META_ROOT)], check=True, capture_output=True, text=True, ) return json.loads(completed.stdout) def _extract_backend_endpoints( catalog: dict[str, Any], workspace_root: Path, ) -> list[dict[str, Any]]: endpoints: list[dict[str, Any]] = [] for repository in catalog["repositories"]: repository_root = workspace_root / repository["path"] source_root = repository_root / "src" if not source_root.is_dir(): continue for source_path in sorted(source_root.rglob("*.py")): try: tree = ast.parse( source_path.read_text(encoding="utf-8"), filename=str(source_path), ) except (OSError, SyntaxError, UnicodeDecodeError): continue prefixes = _router_prefixes(tree) for node in ast.walk(tree): if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue for decorator in node.decorator_list: endpoint = _endpoint_from_decorator( decorator, prefixes=prefixes, ) if endpoint is None: continue endpoints.append( { "repository": repository["name"], "file": str(source_path.relative_to(repository_root)), "line": node.lineno, "handler": node.name, **endpoint, } ) return sorted( endpoints, key=lambda item: ( item["repository"], item["path"], item["method"], item["file"], item["line"], ), ) def _router_prefixes(tree: ast.AST) -> dict[str, str]: prefixes: dict[str, str] = {} for node in ast.walk(tree): if not isinstance(node, (ast.Assign, ast.AnnAssign)): continue value = node.value if not isinstance(value, ast.Call): continue function_name = _call_name(value.func) if function_name not in {"APIRouter", "fastapi.APIRouter"}: continue prefix = "" for keyword in value.keywords: if keyword.arg == "prefix": prefix = _static_string(keyword.value) or "" targets = node.targets if isinstance(node, ast.Assign) else [node.target] for target in targets: if isinstance(target, ast.Name): prefixes[target.id] = prefix return prefixes def _endpoint_from_decorator( decorator: ast.expr, *, prefixes: dict[str, str], ) -> dict[str, str] | None: if not isinstance(decorator, ast.Call) or not isinstance( decorator.func, ast.Attribute ): return None method = decorator.func.attr.lower() if method not in HTTP_METHODS or not decorator.args: return None route = _static_string(decorator.args[0]) if route is None: return None owner = decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else "" prefix = prefixes.get(owner, "") return { "method": method.upper(), "path": _join_route(prefix, route), "router": owner, } def _extract_manifests( catalog: dict[str, Any], workspace_root: Path, ) -> list[dict[str, Any]]: source_roots = [ workspace_root / repository["path"] / "src" for repository in catalog["repositories"] if (workspace_root / repository["path"] / "src").is_dir() ] sys.path[:0] = [str(path) for path in source_roots] manifests: list[dict[str, Any]] = [] for repository in catalog["repositories"]: source_root = workspace_root / repository["path"] / "src" if not source_root.is_dir(): continue for manifest_path in sorted(source_root.glob("*/backend/manifest.py")): module_name = ".".join( manifest_path.relative_to(source_root).with_suffix("").parts ) loaded = importlib.import_module(module_name) manifest = loaded.get_manifest() frontend = manifest.frontend manifests.append( { "repository": repository["name"], "id": manifest.id, "name": manifest.name, "version": manifest.version, "dependencies": list(manifest.dependencies), "optional_dependencies": list(manifest.optional_dependencies), "required_capabilities": list(manifest.required_capabilities), "provided_interfaces": [ {"name": item.name, "version": item.version} for item in manifest.provides_interfaces ], "runtime_capabilities": sorted(manifest.capability_factories), "permissions": [ { "scope": permission.scope, "label": permission.label, "level": permission.level, } for permission in manifest.permissions ], "frontend": ( { "package": frontend.package_name, "routes": [ _plain_value(route) for route in frontend.routes ], "nav_items": [ _plain_value(item) for item in frontend.nav_items ], "view_surfaces": [ _plain_value(surface) for surface in frontend.view_surfaces ], } if frontend is not None else None ), } ) return sorted(manifests, key=lambda item: item["id"]) def _assemble_inventory( *, webui: dict[str, Any], backend_endpoints: list[dict[str, Any]], manifests: list[dict[str, Any]], ) -> dict[str, Any]: frontend_refs = webui["frontendApiReferences"] frontend_paths = { canonical_api_path(reference["path"]) for reference in frontend_refs if canonical_api_path(reference["path"]) } unreferenced = [ endpoint for endpoint in backend_endpoints if canonical_api_path(endpoint["path"]) not in frontend_paths ] usages = {item["key"] for item in webui["translationUsages"]} catalogs = webui["translationCatalog"] catalog_keys = { locale: set(entries) for locale, entries in catalogs.items() } expected_locales = sorted(catalog_keys) missing_catalog_entries = [ { "key": key, "missing_locales": [ locale for locale in expected_locales if key not in catalog_keys[locale] ], } for key in sorted(usages) if any(key not in catalog_keys[locale] for locale in expected_locales) ] fields = webui["fields"] help_candidates = [field for field in fields if field["helpCandidate"]] return { "schema_version": 1, "scope": { "source": "local GovOPlaN repository catalog", "limitations": [ "Static extraction cannot resolve runtime-computed labels, routes, or API paths.", "A backend endpoint without a static WebUI reference may intentionally serve public clients, workers, connectors, or external integrations.", "A field marked as a help candidate may receive contextual help from a surrounding dynamic component.", ], }, "modules": manifests, "ui": { "fields": fields, "labels": webui["labels"], "visible_text": webui["visibleText"], "routes": webui["routes"], "navigation": webui["navigation"], "ui_capabilities": webui["uiCapabilities"], "help_candidates": help_candidates, }, "translations": { "catalog": catalogs, "usages": webui["translationUsages"], "dynamic_usages": webui["dynamicTranslationUsages"], }, "translation_health": { "locales": expected_locales, "used_keys": len(usages), "missing_catalog_entries": missing_catalog_entries, }, "api": { "backend_endpoints": backend_endpoints, "frontend_references": frontend_refs, "unreferenced_by_static_webui_scan": unreferenced, }, "summary": { "modules": len(manifests), "ui_fields": len(fields), "ui_fields_with_static_help": len(fields) - len(help_candidates), "help_review_candidates": len(help_candidates), "label_attributes": len(webui["labels"]), "visible_text_nodes": len(webui["visibleText"]), "frontend_routes": len(webui["routes"]), "backend_endpoints": len(backend_endpoints), "frontend_api_references": len(frontend_refs), "backend_endpoints_without_static_webui_reference": len(unreferenced), }, } def _render_markdown(inventory: dict[str, Any]) -> str: summary = inventory["summary"] missing = inventory["translation_health"]["missing_catalog_entries"] help_by_repository = Counter( item["repository"] for item in inventory["ui"]["help_candidates"] ) endpoint_by_repository = Counter( item["repository"] for item in inventory["api"]["unreferenced_by_static_webui_scan"] ) lines = [ "# GovOPlaN Platform Interface Inventory", "", "Generated from runtime module manifests plus TypeScript and Python ASTs.", "Counts are source evidence, not a claim that every surface is enabled or reachable.", "", "## Summary", "", f"- Modules: {summary['modules']}", f"- UI fields: {summary['ui_fields']}", f"- Fields with statically associated help: {summary['ui_fields_with_static_help']}", f"- Help review candidates: {summary['help_review_candidates']}", f"- Label attributes: {summary['label_attributes']}", f"- Frontend routes: {summary['frontend_routes']}", f"- Backend endpoints: {summary['backend_endpoints']}", f"- Frontend API references: {summary['frontend_api_references']}", ( "- Backend endpoints without a static WebUI reference: " f"{summary['backend_endpoints_without_static_webui_reference']}" ), f"- Used translation keys missing from a locale catalog: {len(missing)}", "", "## Help Review Candidates", "", "| Repository | Fields |", "| --- | ---: |", ] lines.extend( f"| `{repository}` | {count} |" for repository, count in sorted(help_by_repository.items()) ) lines.extend( [ "", "## Endpoints Without Static WebUI References", "", "These are review candidates. Public APIs, worker callbacks, connector", "endpoints, health checks, and dynamically assembled paths are legitimate", "reasons for appearing here.", "", "| Repository | Endpoints |", "| --- | ---: |", ] ) lines.extend( f"| `{repository}` | {count} |" for repository, count in sorted(endpoint_by_repository.items()) ) lines.extend( [ "", "## Interpretation", "", "Use the JSON artifact for exact file and line evidence. Missing help is", "a triage list, not an automatic defect. Endpoint coverage requires an", "owner classification before enforcement. Runtime-computed structures", "need explicit manifest metadata to become canonically visible.", "", ] ) return "\n".join(lines) def canonical_api_path(value: str) -> str: path = value.split("?", 1)[0].strip() if "/api/" in path: path = path[path.index("/api/") :] path = re.sub(r"^/api/v\d+", "", path) path = PATH_PARAMETER.sub("{}", path) path = re.sub(r"/+", "/", path) return path.rstrip("/") or "/" def _join_route(prefix: str, route: str) -> str: return f"/{prefix.strip('/')}/{route.strip('/')}".replace("//", "/") def _static_string(node: ast.AST) -> str | None: if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value if isinstance(node, ast.JoinedStr): pieces: list[str] = [] for value in node.values: if isinstance(value, ast.Constant) and isinstance(value.value, str): pieces.append(value.value) elif isinstance(value, ast.FormattedValue): pieces.append("${}") else: return None return "".join(pieces) return None def _call_name(node: ast.AST) -> str: if isinstance(node, ast.Name): return node.id if isinstance(node, ast.Attribute): parent = _call_name(node.value) return f"{parent}.{node.attr}" if parent else node.attr return "" def _plain_value(value: Any) -> Any: if is_dataclass(value): return { key: _plain_value(item) for key, item in asdict(value).items() } if isinstance(value, tuple): return [_plain_value(item) for item in value] if isinstance(value, dict): return { str(key): _plain_value(item) for key, item in value.items() } return value if __name__ == "__main__": raise SystemExit(main())