463 lines
16 KiB
Python
463 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract a source-derived GovOPlaN UI, translation, module, and API inventory."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ast
|
|
from collections import Counter
|
|
from dataclasses import asdict, is_dataclass
|
|
import importlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
META_ROOT = Path(__file__).resolve().parents[2]
|
|
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
|
|
PATH_PARAMETER = re.compile(r"\$\{[^{}]*\}|\{[^{}]+\}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=Path,
|
|
default=META_ROOT / "audit-reports" / "platform-inventory",
|
|
)
|
|
parser.add_argument(
|
|
"--strict",
|
|
action="store_true",
|
|
help="Fail when a used translation key is absent from a generated locale catalog.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
catalog = json.loads(
|
|
(META_ROOT / "repositories.json").read_text(encoding="utf-8")
|
|
)
|
|
workspace_root = Path(catalog["default_parent"]).resolve()
|
|
webui = _extract_webui()
|
|
backend_endpoints = _extract_backend_endpoints(catalog, workspace_root)
|
|
manifests = _extract_manifests(catalog, workspace_root)
|
|
inventory = _assemble_inventory(
|
|
webui=webui,
|
|
backend_endpoints=backend_endpoints,
|
|
manifests=manifests,
|
|
)
|
|
|
|
output_dir = args.output_dir.resolve()
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
json_path = output_dir / "platform-interface-inventory.json"
|
|
markdown_path = output_dir / "platform-interface-inventory.md"
|
|
json_path.write_text(
|
|
json.dumps(inventory, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
markdown_path.write_text(_render_markdown(inventory), encoding="utf-8")
|
|
print(f"Platform inventory JSON: {json_path}")
|
|
print(f"Platform inventory summary: {markdown_path}")
|
|
|
|
if args.strict and inventory["translation_health"]["missing_catalog_entries"]:
|
|
print(
|
|
"Used translation keys are missing from generated catalogs.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def _extract_webui() -> dict[str, Any]:
|
|
helper = META_ROOT / "tools" / "inventory" / "extract-webui-structure.mjs"
|
|
completed = subprocess.run(
|
|
["node", str(helper), str(META_ROOT)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return json.loads(completed.stdout)
|
|
|
|
|
|
def _extract_backend_endpoints(
|
|
catalog: dict[str, Any],
|
|
workspace_root: Path,
|
|
) -> list[dict[str, Any]]:
|
|
endpoints: list[dict[str, Any]] = []
|
|
for repository in catalog["repositories"]:
|
|
repository_root = workspace_root / repository["path"]
|
|
source_root = repository_root / "src"
|
|
if not source_root.is_dir():
|
|
continue
|
|
for source_path in sorted(source_root.rglob("*.py")):
|
|
try:
|
|
tree = ast.parse(
|
|
source_path.read_text(encoding="utf-8"),
|
|
filename=str(source_path),
|
|
)
|
|
except (OSError, SyntaxError, UnicodeDecodeError):
|
|
continue
|
|
prefixes = _router_prefixes(tree)
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
continue
|
|
for decorator in node.decorator_list:
|
|
endpoint = _endpoint_from_decorator(
|
|
decorator,
|
|
prefixes=prefixes,
|
|
)
|
|
if endpoint is None:
|
|
continue
|
|
endpoints.append(
|
|
{
|
|
"repository": repository["name"],
|
|
"file": str(source_path.relative_to(repository_root)),
|
|
"line": node.lineno,
|
|
"handler": node.name,
|
|
**endpoint,
|
|
}
|
|
)
|
|
return sorted(
|
|
endpoints,
|
|
key=lambda item: (
|
|
item["repository"],
|
|
item["path"],
|
|
item["method"],
|
|
item["file"],
|
|
item["line"],
|
|
),
|
|
)
|
|
|
|
|
|
def _router_prefixes(tree: ast.AST) -> dict[str, str]:
|
|
prefixes: dict[str, str] = {}
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
|
|
continue
|
|
value = node.value
|
|
if not isinstance(value, ast.Call):
|
|
continue
|
|
function_name = _call_name(value.func)
|
|
if function_name not in {"APIRouter", "fastapi.APIRouter"}:
|
|
continue
|
|
prefix = ""
|
|
for keyword in value.keywords:
|
|
if keyword.arg == "prefix":
|
|
prefix = _static_string(keyword.value) or ""
|
|
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
for target in targets:
|
|
if isinstance(target, ast.Name):
|
|
prefixes[target.id] = prefix
|
|
return prefixes
|
|
|
|
|
|
def _endpoint_from_decorator(
|
|
decorator: ast.expr,
|
|
*,
|
|
prefixes: dict[str, str],
|
|
) -> dict[str, str] | None:
|
|
if not isinstance(decorator, ast.Call) or not isinstance(
|
|
decorator.func, ast.Attribute
|
|
):
|
|
return None
|
|
method = decorator.func.attr.lower()
|
|
if method not in HTTP_METHODS or not decorator.args:
|
|
return None
|
|
route = _static_string(decorator.args[0])
|
|
if route is None:
|
|
return None
|
|
owner = decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else ""
|
|
prefix = prefixes.get(owner, "")
|
|
return {
|
|
"method": method.upper(),
|
|
"path": _join_route(prefix, route),
|
|
"router": owner,
|
|
}
|
|
|
|
|
|
def _extract_manifests(
|
|
catalog: dict[str, Any],
|
|
workspace_root: Path,
|
|
) -> list[dict[str, Any]]:
|
|
source_roots = [
|
|
workspace_root / repository["path"] / "src"
|
|
for repository in catalog["repositories"]
|
|
if (workspace_root / repository["path"] / "src").is_dir()
|
|
]
|
|
sys.path[:0] = [str(path) for path in source_roots]
|
|
manifests: list[dict[str, Any]] = []
|
|
for repository in catalog["repositories"]:
|
|
source_root = workspace_root / repository["path"] / "src"
|
|
if not source_root.is_dir():
|
|
continue
|
|
for manifest_path in sorted(source_root.glob("*/backend/manifest.py")):
|
|
module_name = ".".join(
|
|
manifest_path.relative_to(source_root).with_suffix("").parts
|
|
)
|
|
loaded = importlib.import_module(module_name)
|
|
manifest = loaded.get_manifest()
|
|
frontend = manifest.frontend
|
|
manifests.append(
|
|
{
|
|
"repository": repository["name"],
|
|
"id": manifest.id,
|
|
"name": manifest.name,
|
|
"version": manifest.version,
|
|
"dependencies": list(manifest.dependencies),
|
|
"optional_dependencies": list(manifest.optional_dependencies),
|
|
"required_capabilities": list(manifest.required_capabilities),
|
|
"provided_interfaces": [
|
|
{"name": item.name, "version": item.version}
|
|
for item in manifest.provides_interfaces
|
|
],
|
|
"runtime_capabilities": sorted(manifest.capability_factories),
|
|
"permissions": [
|
|
{
|
|
"scope": permission.scope,
|
|
"label": permission.label,
|
|
"level": permission.level,
|
|
}
|
|
for permission in manifest.permissions
|
|
],
|
|
"frontend": (
|
|
{
|
|
"package": frontend.package_name,
|
|
"routes": [
|
|
_plain_value(route) for route in frontend.routes
|
|
],
|
|
"nav_items": [
|
|
_plain_value(item) for item in frontend.nav_items
|
|
],
|
|
"view_surfaces": [
|
|
_plain_value(surface)
|
|
for surface in frontend.view_surfaces
|
|
],
|
|
}
|
|
if frontend is not None
|
|
else None
|
|
),
|
|
}
|
|
)
|
|
return sorted(manifests, key=lambda item: item["id"])
|
|
|
|
|
|
def _assemble_inventory(
|
|
*,
|
|
webui: dict[str, Any],
|
|
backend_endpoints: list[dict[str, Any]],
|
|
manifests: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
frontend_refs = webui["frontendApiReferences"]
|
|
frontend_paths = {
|
|
canonical_api_path(reference["path"])
|
|
for reference in frontend_refs
|
|
if canonical_api_path(reference["path"])
|
|
}
|
|
unreferenced = [
|
|
endpoint
|
|
for endpoint in backend_endpoints
|
|
if canonical_api_path(endpoint["path"]) not in frontend_paths
|
|
]
|
|
usages = {item["key"] for item in webui["translationUsages"]}
|
|
catalogs = webui["translationCatalog"]
|
|
catalog_keys = {
|
|
locale: set(entries)
|
|
for locale, entries in catalogs.items()
|
|
}
|
|
expected_locales = sorted(catalog_keys)
|
|
missing_catalog_entries = [
|
|
{
|
|
"key": key,
|
|
"missing_locales": [
|
|
locale
|
|
for locale in expected_locales
|
|
if key not in catalog_keys[locale]
|
|
],
|
|
}
|
|
for key in sorted(usages)
|
|
if any(key not in catalog_keys[locale] for locale in expected_locales)
|
|
]
|
|
fields = webui["fields"]
|
|
help_candidates = [field for field in fields if field["helpCandidate"]]
|
|
return {
|
|
"schema_version": 1,
|
|
"scope": {
|
|
"source": "local GovOPlaN repository catalog",
|
|
"limitations": [
|
|
"Static extraction cannot resolve runtime-computed labels, routes, or API paths.",
|
|
"A backend endpoint without a static WebUI reference may intentionally serve public clients, workers, connectors, or external integrations.",
|
|
"A field marked as a help candidate may receive contextual help from a surrounding dynamic component.",
|
|
],
|
|
},
|
|
"modules": manifests,
|
|
"ui": {
|
|
"fields": fields,
|
|
"labels": webui["labels"],
|
|
"visible_text": webui["visibleText"],
|
|
"routes": webui["routes"],
|
|
"navigation": webui["navigation"],
|
|
"ui_capabilities": webui["uiCapabilities"],
|
|
"help_candidates": help_candidates,
|
|
},
|
|
"translations": {
|
|
"catalog": catalogs,
|
|
"usages": webui["translationUsages"],
|
|
"dynamic_usages": webui["dynamicTranslationUsages"],
|
|
},
|
|
"translation_health": {
|
|
"locales": expected_locales,
|
|
"used_keys": len(usages),
|
|
"missing_catalog_entries": missing_catalog_entries,
|
|
},
|
|
"api": {
|
|
"backend_endpoints": backend_endpoints,
|
|
"frontend_references": frontend_refs,
|
|
"unreferenced_by_static_webui_scan": unreferenced,
|
|
},
|
|
"summary": {
|
|
"modules": len(manifests),
|
|
"ui_fields": len(fields),
|
|
"ui_fields_with_static_help": len(fields) - len(help_candidates),
|
|
"help_review_candidates": len(help_candidates),
|
|
"label_attributes": len(webui["labels"]),
|
|
"visible_text_nodes": len(webui["visibleText"]),
|
|
"frontend_routes": len(webui["routes"]),
|
|
"backend_endpoints": len(backend_endpoints),
|
|
"frontend_api_references": len(frontend_refs),
|
|
"backend_endpoints_without_static_webui_reference": len(unreferenced),
|
|
},
|
|
}
|
|
|
|
|
|
def _render_markdown(inventory: dict[str, Any]) -> str:
|
|
summary = inventory["summary"]
|
|
missing = inventory["translation_health"]["missing_catalog_entries"]
|
|
help_by_repository = Counter(
|
|
item["repository"] for item in inventory["ui"]["help_candidates"]
|
|
)
|
|
endpoint_by_repository = Counter(
|
|
item["repository"]
|
|
for item in inventory["api"]["unreferenced_by_static_webui_scan"]
|
|
)
|
|
lines = [
|
|
"# GovOPlaN Platform Interface Inventory",
|
|
"",
|
|
"Generated from runtime module manifests plus TypeScript and Python ASTs.",
|
|
"Counts are source evidence, not a claim that every surface is enabled or reachable.",
|
|
"",
|
|
"## Summary",
|
|
"",
|
|
f"- Modules: {summary['modules']}",
|
|
f"- UI fields: {summary['ui_fields']}",
|
|
f"- Fields with statically associated help: {summary['ui_fields_with_static_help']}",
|
|
f"- Help review candidates: {summary['help_review_candidates']}",
|
|
f"- Label attributes: {summary['label_attributes']}",
|
|
f"- Frontend routes: {summary['frontend_routes']}",
|
|
f"- Backend endpoints: {summary['backend_endpoints']}",
|
|
f"- Frontend API references: {summary['frontend_api_references']}",
|
|
(
|
|
"- Backend endpoints without a static WebUI reference: "
|
|
f"{summary['backend_endpoints_without_static_webui_reference']}"
|
|
),
|
|
f"- Used translation keys missing from a locale catalog: {len(missing)}",
|
|
"",
|
|
"## Help Review Candidates",
|
|
"",
|
|
"| Repository | Fields |",
|
|
"| --- | ---: |",
|
|
]
|
|
lines.extend(
|
|
f"| `{repository}` | {count} |"
|
|
for repository, count in sorted(help_by_repository.items())
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Endpoints Without Static WebUI References",
|
|
"",
|
|
"These are review candidates. Public APIs, worker callbacks, connector",
|
|
"endpoints, health checks, and dynamically assembled paths are legitimate",
|
|
"reasons for appearing here.",
|
|
"",
|
|
"| Repository | Endpoints |",
|
|
"| --- | ---: |",
|
|
]
|
|
)
|
|
lines.extend(
|
|
f"| `{repository}` | {count} |"
|
|
for repository, count in sorted(endpoint_by_repository.items())
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Interpretation",
|
|
"",
|
|
"Use the JSON artifact for exact file and line evidence. Missing help is",
|
|
"a triage list, not an automatic defect. Endpoint coverage requires an",
|
|
"owner classification before enforcement. Runtime-computed structures",
|
|
"need explicit manifest metadata to become canonically visible.",
|
|
"",
|
|
]
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def canonical_api_path(value: str) -> str:
|
|
path = value.split("?", 1)[0].strip()
|
|
if "/api/" in path:
|
|
path = path[path.index("/api/") :]
|
|
path = re.sub(r"^/api/v\d+", "", path)
|
|
path = PATH_PARAMETER.sub("{}", path)
|
|
path = re.sub(r"/+", "/", path)
|
|
return path.rstrip("/") or "/"
|
|
|
|
|
|
def _join_route(prefix: str, route: str) -> str:
|
|
return f"/{prefix.strip('/')}/{route.strip('/')}".replace("//", "/")
|
|
|
|
|
|
def _static_string(node: ast.AST) -> str | None:
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
return node.value
|
|
if isinstance(node, ast.JoinedStr):
|
|
pieces: list[str] = []
|
|
for value in node.values:
|
|
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
|
pieces.append(value.value)
|
|
elif isinstance(value, ast.FormattedValue):
|
|
pieces.append("${}")
|
|
else:
|
|
return None
|
|
return "".join(pieces)
|
|
return None
|
|
|
|
|
|
def _call_name(node: ast.AST) -> str:
|
|
if isinstance(node, ast.Name):
|
|
return node.id
|
|
if isinstance(node, ast.Attribute):
|
|
parent = _call_name(node.value)
|
|
return f"{parent}.{node.attr}" if parent else node.attr
|
|
return ""
|
|
|
|
|
|
def _plain_value(value: Any) -> Any:
|
|
if is_dataclass(value):
|
|
return {
|
|
key: _plain_value(item)
|
|
for key, item in asdict(value).items()
|
|
}
|
|
if isinstance(value, tuple):
|
|
return [_plain_value(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return {
|
|
str(key): _plain_value(item)
|
|
for key, item in value.items()
|
|
}
|
|
return value
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|