GovOPlaN/govoplan
1
0
Fork 0
Code Issues 19 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enforce backend endpoint classification

Browse Source
This commit is contained in:
zemion
2026-08-02 05:30:20 +02:00
parent f7a30682b3
commit 50e9607e72
5 changed files with 1899 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/inventory/endpoint-surface-declarations.json
+1591
View File
File diff suppressed because it is too large Load Diff
tools/inventory/platform-interface-inventory.py
+195 -29
View File
@@ -19,6 +19,18 @@ from typing import Any
META_ROOT = Path(__file__).resolve().parents[2]
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
PATH_PARAMETER = re.compile(r"\$\{[^{}]*\}|\{[^{}]+\}")
ENDPOINT_SURFACE_CATEGORIES = {
"ui_reachable",
"intentionally_headless",
"public_integration",
"worker_internal",
"compatibility",
"missing_ui",
"removable",
}
DEFAULT_ENDPOINT_DECLARATIONS = (
META_ROOT / "tools" / "inventory" / "endpoint-surface-declarations.json"
)
def main() -> int:
@@ -31,21 +43,29 @@ def main() -> int:
parser.add_argument(
"--strict",
action="store_true",
help="Fail when a used translation key is absent from a generated locale catalog.",
help="Fail on missing translations or incomplete endpoint-surface declarations.",
)
parser.add_argument(
"--endpoint-declarations",
type=Path,
default=DEFAULT_ENDPOINT_DECLARATIONS,
help="Versioned endpoint-surface declaration registry.",
)
args = parser.parse_args()
catalog = json.loads(
(META_ROOT / "repositories.json").read_text(encoding="utf-8")
)
catalog = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
workspace_root = Path(catalog["default_parent"]).resolve()
webui = _extract_webui()
backend_endpoints = _extract_backend_endpoints(catalog, workspace_root)
manifests = _extract_manifests(catalog, workspace_root)
endpoint_declarations = _load_endpoint_declarations(
args.endpoint_declarations.resolve()
)
inventory = _assemble_inventory(
webui=webui,
backend_endpoints=backend_endpoints,
manifests=manifests,
endpoint_declarations=endpoint_declarations,
)
output_dir = args.output_dir.resolve()
@@ -60,12 +80,26 @@ def main() -> int:
print(f"Platform inventory JSON: {json_path}")
print(f"Platform inventory summary: {markdown_path}")
if args.strict and inventory["translation_health"]["missing_catalog_entries"]:
print(
"Used translation keys are missing from generated catalogs.",
file=sys.stderr,
)
return 1
if args.strict:
failures: list[str] = []
if inventory["translation_health"]["missing_catalog_entries"]:
failures.append("used translation keys are missing from generated catalogs")
if inventory["api"]["unclassified_endpoints"]:
failures.append(
f"{len(inventory['api']['unclassified_endpoints'])} backend "
"endpoints have no WebUI evidence or surface declaration"
)
if inventory["api"]["stale_endpoint_declarations"]:
failures.append(
f"{len(inventory['api']['stale_endpoint_declarations'])} "
"endpoint declarations do not match a backend endpoint"
)
if failures:
print(
"Strict platform inventory failed: " + "; ".join(failures) + ".",
file=sys.stderr,
)
return 1
return 0
@@ -167,7 +201,9 @@ def _endpoint_from_decorator(
route = _static_string(decorator.args[0])
if route is None:
return None
owner = decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else ""
owner = (
decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else ""
)
prefix = prefixes.get(owner, "")
return {
"method": method.upper(),
@@ -247,6 +283,7 @@ def _assemble_inventory(
webui: dict[str, Any],
backend_endpoints: list[dict[str, Any]],
manifests: list[dict[str, Any]],
endpoint_declarations: dict[tuple[str, str, str], dict[str, Any]],
) -> dict[str, Any]:
frontend_refs = webui["frontendApiReferences"]
frontend_paths = {
@@ -254,25 +291,61 @@ def _assemble_inventory(
for reference in frontend_refs
if canonical_api_path(reference["path"])
}
endpoint_keys = {endpoint_key(endpoint) for endpoint in backend_endpoints}
classified_endpoints: list[dict[str, Any]] = []
for endpoint in backend_endpoints:
key = endpoint_key(endpoint)
canonical_path = key[2]
static_webui_reference = canonical_path in frontend_paths
declaration = endpoint_declarations.get(key)
surface = (
{
"category": "ui_reachable",
"rationale": "A canonical API path reference exists in WebUI source.",
"source": "static_webui_scan",
}
if static_webui_reference
else (
{**declaration, "source": "declaration"}
if declaration is not None
else None
)
)
classified_endpoints.append(
{
**endpoint,
"canonical_path": canonical_path,
"static_webui_reference": static_webui_reference,
"surface": surface,
}
)
unreferenced = [
endpoint
for endpoint in backend_endpoints
if canonical_api_path(endpoint["path"]) not in frontend_paths
for endpoint in classified_endpoints
if not endpoint["static_webui_reference"]
]
unclassified = [
endpoint for endpoint in unreferenced if endpoint["surface"] is None
]
stale_declarations = [
declaration
for key, declaration in endpoint_declarations.items()
if key not in endpoint_keys
]
classification_counts = Counter(
endpoint["surface"]["category"]
for endpoint in classified_endpoints
if endpoint["surface"] is not None
)
usages = {item["key"] for item in webui["translationUsages"]}
catalogs = webui["translationCatalog"]
catalog_keys = {
locale: set(entries)
for locale, entries in catalogs.items()
}
catalog_keys = {locale: set(entries) for locale, entries in catalogs.items()}
expected_locales = sorted(catalog_keys)
missing_catalog_entries = [
{
"key": key,
"missing_locales": [
locale
for locale in expected_locales
if key not in catalog_keys[locale]
locale for locale in expected_locales if key not in catalog_keys[locale]
],
}
for key in sorted(usages)
@@ -311,9 +384,12 @@ def _assemble_inventory(
"missing_catalog_entries": missing_catalog_entries,
},
"api": {
"backend_endpoints": backend_endpoints,
"backend_endpoints": classified_endpoints,
"frontend_references": frontend_refs,
"unreferenced_by_static_webui_scan": unreferenced,
"unclassified_endpoints": unclassified,
"stale_endpoint_declarations": stale_declarations,
"classification_counts": dict(sorted(classification_counts.items())),
},
"summary": {
"modules": len(manifests),
@@ -326,6 +402,8 @@ def _assemble_inventory(
"backend_endpoints": len(backend_endpoints),
"frontend_api_references": len(frontend_refs),
"backend_endpoints_without_static_webui_reference": len(unreferenced),
"unclassified_backend_endpoints": len(unclassified),
"stale_endpoint_declarations": len(stale_declarations),
},
}
@@ -340,6 +418,7 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
item["repository"]
for item in inventory["api"]["unreferenced_by_static_webui_scan"]
)
classification_counts = inventory["api"]["classification_counts"]
lines = [
"# GovOPlaN Platform Interface Inventory",
"",
@@ -360,6 +439,8 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
"- Backend endpoints without a static WebUI reference: "
f"{summary['backend_endpoints_without_static_webui_reference']}"
),
f"- Unclassified backend endpoints: {summary['unclassified_backend_endpoints']}",
f"- Stale endpoint declarations: {summary['stale_endpoint_declarations']}",
f"- Used translation keys missing from a locale catalog: {len(missing)}",
"",
"## Help Review Candidates",
@@ -388,6 +469,19 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
f"| `{repository}` | {count} |"
for repository, count in sorted(endpoint_by_repository.items())
)
lines.extend(
[
"",
"## Endpoint Surface Classifications",
"",
"| Classification | Endpoints |",
"| --- | ---: |",
]
)
lines.extend(
f"| `{category}` | {count} |"
for category, count in sorted(classification_counts.items())
)
lines.extend(
[
"",
@@ -408,11 +502,89 @@ def canonical_api_path(value: str) -> str:
if "/api/" in path:
path = path[path.index("/api/") :]
path = re.sub(r"^/api/v\d+", "", path)
path = re.sub(r"(?<=[^/])\$\{[^{}]*\}$", "", path)
path = PATH_PARAMETER.sub("{}", path)
path = re.sub(r"/+", "/", path)
return path.rstrip("/") or "/"
def endpoint_key(endpoint: dict[str, Any]) -> tuple[str, str, str]:
return (
str(endpoint["repository"]),
str(endpoint["method"]).upper(),
canonical_api_path(str(endpoint["path"])),
)
def _load_endpoint_declarations(
path: Path,
) -> dict[tuple[str, str, str], dict[str, Any]]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(
f"Endpoint declaration registry does not exist: {path}"
) from exc
except json.JSONDecodeError as exc:
raise ValueError(
f"Endpoint declaration registry is invalid JSON: {exc}"
) from exc
if not isinstance(payload, dict) or payload.get("schema_version") != 1:
raise ValueError("Endpoint declaration registry must use schema_version 1.")
entries = payload.get("endpoints")
if not isinstance(entries, list):
raise ValueError(
"Endpoint declaration registry must contain an endpoints list."
)
declarations: dict[tuple[str, str, str], dict[str, Any]] = {}
for index, entry in enumerate(entries):
if not isinstance(entry, dict):
raise ValueError(f"Endpoint declaration {index} must be an object.")
repository = entry.get("repository")
method = entry.get("method")
raw_path = entry.get("path")
category = entry.get("category")
rationale = entry.get("rationale")
if not isinstance(repository, str) or not repository.strip():
raise ValueError(f"Endpoint declaration {index} has no repository.")
if not isinstance(method, str) or method.lower() not in HTTP_METHODS:
raise ValueError(f"Endpoint declaration {index} has an invalid method.")
if not isinstance(raw_path, str) or not raw_path.startswith("/"):
raise ValueError(f"Endpoint declaration {index} has an invalid path.")
canonical_path = canonical_api_path(raw_path)
if raw_path != canonical_path:
raise ValueError(
f"Endpoint declaration {index} path must be canonical: {canonical_path}"
)
if category not in ENDPOINT_SURFACE_CATEGORIES:
raise ValueError(f"Endpoint declaration {index} has an invalid category.")
if not isinstance(rationale, str) or not rationale.strip():
raise ValueError(f"Endpoint declaration {index} has no rationale.")
tracking_issue = entry.get("tracking_issue")
if category == "missing_ui" and (
not isinstance(tracking_issue, str) or not tracking_issue.strip()
):
raise ValueError(
f"Endpoint declaration {index} requires a tracking_issue for missing_ui."
)
key = (repository, method.upper(), canonical_path)
if key in declarations:
raise ValueError(f"Duplicate endpoint declaration: {key!r}.")
declarations[key] = {
"repository": repository,
"method": method.upper(),
"path": canonical_path,
"category": category,
"rationale": rationale.strip(),
**(
{"tracking_issue": tracking_issue.strip()}
if isinstance(tracking_issue, str) and tracking_issue.strip()
else {}
),
}
return declarations
def _join_route(prefix: str, route: str) -> str:
return f"/{prefix.strip('/')}/{route.strip('/')}".replace("//", "/")
@@ -444,17 +616,11 @@ def _call_name(node: ast.AST) -> str:
def _plain_value(value: Any) -> Any:
if is_dataclass(value):
return {
key: _plain_value(item)
for key, item in asdict(value).items()
}
return {key: _plain_value(item) for key, item in asdict(value).items()}
if isinstance(value, tuple):
return [_plain_value(item) for item in value]
if isinstance(value, dict):
return {
str(key): _plain_value(item)
for key, item in value.items()
}
return {str(key): _plain_value(item) for key, item in value.items()}
return value