GovOPlaN/govoplan
1
0
Fork 0
Code Issues 19 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enforce backend endpoint classification

Browse Source
This commit is contained in:
zemion
2026-08-02 05:30:20 +02:00
parent f7a30682b3
commit 50e9607e72
5 changed files with 1899 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/module-matrix.yml
+3
View File
@@ -46,6 +46,9 @@ jobs:
- name: Install WebUI release dependencies with test scripts
working-directory: govoplan
run: bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
- name: Validate platform endpoint surface declarations
working-directory: govoplan
run: .venv/bin/python tools/inventory/platform-interface-inventory.py --strict
- name: Validate Search against PostgreSQL
working-directory: govoplan
env:
docs/PLATFORM_CONTROL_PLANE.md
+22
View File
@@ -45,6 +45,28 @@ The command writes:
- `audit-reports/platform-inventory/platform-interface-inventory.json`
- `audit-reports/platform-inventory/platform-interface-inventory.md`
Use `--strict` in CI. In addition to translation coverage, strict mode requires
every backend endpoint without a statically visible WebUI path to have an exact
entry in
`tools/inventory/endpoint-surface-declarations.json`. The registry is keyed by
repository, HTTP method, and canonical version-independent path. It accepts:
- `ui_reachable`: a mounted router, generic action, or provider path hides the
reference from static extraction;
- `intentionally_headless`: a capability/API is deliberately consumed without
its own UI;
- `public_integration`: a documented public or interoperability endpoint;
- `worker_internal`: a worker, scheduler, reconciliation, or monitoring path;
- `compatibility`: a retained transition endpoint with a current replacement;
- `missing_ui`: a real UI gap, which must include a Gitea tracking issue;
- `removable`: a reviewed dead endpoint pending removal.
Strict mode also rejects declarations that no longer match source. When an
endpoint is added, changed, or removed, update its declaration in the same
change. Do not classify an endpoint from a string mismatch alone: first check
mounted prefixes, dynamic action paths, public clients, worker use, and
capability consumers.
It combines:
1. loaded module manifests
tests/test_platform_interface_inventory.py
+88
View File
@@ -2,7 +2,9 @@ from __future__ import annotations
import ast
import importlib.util
import json
from pathlib import Path
import tempfile
import unittest
@@ -30,6 +32,92 @@ class PlatformInterfaceInventoryTests(unittest.TestCase):
inventory.canonical_api_path("/api/v2/campaigns/{campaign_id}"),
"/campaigns/{}",
)
self.assertEqual(
inventory.canonical_api_path("/api/v1/calendar/events/delta${querySuffix}"),
"/calendar/events/delta",
)
self.assertEqual(
inventory.canonical_api_path("/api/v1/calendar/events/${eventId}"),
"/calendar/events/{}",
)
def test_endpoint_declarations_are_exact_and_require_missing_ui_issue(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "endpoints.json"
path.write_text(
json.dumps(
{
"schema_version": 1,
"endpoints": [
{
"repository": "govoplan-example",
"method": "GET",
"path": "/example/items/{}",
"category": "public_integration",
"rationale": "Published integration API.",
}
],
}
),
encoding="utf-8",
)
declarations = inventory._load_endpoint_declarations(path)
self.assertIn(
("govoplan-example", "GET", "/example/items/{}"),
declarations,
)
payload = json.loads(path.read_text(encoding="utf-8"))
payload["endpoints"][0]["category"] = "missing_ui"
path.write_text(json.dumps(payload), encoding="utf-8")
with self.assertRaisesRegex(ValueError, "tracking_issue"):
inventory._load_endpoint_declarations(path)
def test_inventory_reports_unclassified_and_stale_endpoint_declarations(
self,
) -> None:
webui = {
"frontendApiReferences": [],
"translationUsages": [],
"translationCatalog": {"en": {}, "de": {}},
"fields": [],
"labels": [],
"visibleText": [],
"routes": [],
"navigation": [],
"uiCapabilities": [],
"dynamicTranslationUsages": [],
}
endpoint = {
"repository": "govoplan-example",
"method": "GET",
"path": "/api/v1/example/items",
"file": "src/example.py",
"line": 1,
"handler": "items",
"router": "router",
}
stale = {
"repository": "govoplan-example",
"method": "GET",
"path": "/example/removed",
"category": "removable",
"rationale": "Removal is pending.",
}
result = inventory._assemble_inventory(
webui=webui,
backend_endpoints=[endpoint],
manifests=[],
endpoint_declarations={
("govoplan-example", "GET", "/example/removed"): stale,
},
)
self.assertEqual(1, result["summary"]["unclassified_backend_endpoints"])
self.assertEqual(1, result["summary"]["stale_endpoint_declarations"])
self.assertIsNone(result["api"]["backend_endpoints"][0]["surface"])
def test_fastapi_route_scanner_includes_router_prefix(self) -> None:
tree = ast.parse(
tools/inventory/endpoint-surface-declarations.json
+1591
View File
File diff suppressed because it is too large Load Diff
tools/inventory/platform-interface-inventory.py
+191 -25
View File
@@ -19,6 +19,18 @@ from typing import Any
META_ROOT = Path(__file__).resolve().parents[2]
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
PATH_PARAMETER = re.compile(r"\$\{[^{}]*\}|\{[^{}]+\}")
ENDPOINT_SURFACE_CATEGORIES = {
"ui_reachable",
"intentionally_headless",
"public_integration",
"worker_internal",
"compatibility",
"missing_ui",
"removable",
}
DEFAULT_ENDPOINT_DECLARATIONS = (
META_ROOT / "tools" / "inventory" / "endpoint-surface-declarations.json"
)
def main() -> int:
@@ -31,21 +43,29 @@ def main() -> int:
parser.add_argument(
"--strict",
action="store_true",
help="Fail when a used translation key is absent from a generated locale catalog.",
help="Fail on missing translations or incomplete endpoint-surface declarations.",
)
parser.add_argument(
"--endpoint-declarations",
type=Path,
default=DEFAULT_ENDPOINT_DECLARATIONS,
help="Versioned endpoint-surface declaration registry.",
)
args = parser.parse_args()
catalog = json.loads(
(META_ROOT / "repositories.json").read_text(encoding="utf-8")
)
catalog = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
workspace_root = Path(catalog["default_parent"]).resolve()
webui = _extract_webui()
backend_endpoints = _extract_backend_endpoints(catalog, workspace_root)
manifests = _extract_manifests(catalog, workspace_root)
endpoint_declarations = _load_endpoint_declarations(
args.endpoint_declarations.resolve()
)
inventory = _assemble_inventory(
webui=webui,
backend_endpoints=backend_endpoints,
manifests=manifests,
endpoint_declarations=endpoint_declarations,
)
output_dir = args.output_dir.resolve()
@@ -60,9 +80,23 @@ def main() -> int:
print(f"Platform inventory JSON: {json_path}")
print(f"Platform inventory summary: {markdown_path}")
if args.strict and inventory["translation_health"]["missing_catalog_entries"]:
if args.strict:
failures: list[str] = []
if inventory["translation_health"]["missing_catalog_entries"]:
failures.append("used translation keys are missing from generated catalogs")
if inventory["api"]["unclassified_endpoints"]:
failures.append(
f"{len(inventory['api']['unclassified_endpoints'])} backend "
"endpoints have no WebUI evidence or surface declaration"
)
if inventory["api"]["stale_endpoint_declarations"]:
failures.append(
f"{len(inventory['api']['stale_endpoint_declarations'])} "
"endpoint declarations do not match a backend endpoint"
)
if failures:
print(
"Used translation keys are missing from generated catalogs.",
"Strict platform inventory failed: " + "; ".join(failures) + ".",
file=sys.stderr,
)
return 1
@@ -167,7 +201,9 @@ def _endpoint_from_decorator(
route = _static_string(decorator.args[0])
if route is None:
return None
owner = decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else ""
owner = (
decorator.func.value.id if isinstance(decorator.func.value, ast.Name) else ""
)
prefix = prefixes.get(owner, "")
return {
"method": method.upper(),
@@ -247,6 +283,7 @@ def _assemble_inventory(
webui: dict[str, Any],
backend_endpoints: list[dict[str, Any]],
manifests: list[dict[str, Any]],
endpoint_declarations: dict[tuple[str, str, str], dict[str, Any]],
) -> dict[str, Any]:
frontend_refs = webui["frontendApiReferences"]
frontend_paths = {
@@ -254,25 +291,61 @@ def _assemble_inventory(
for reference in frontend_refs
if canonical_api_path(reference["path"])
}
endpoint_keys = {endpoint_key(endpoint) for endpoint in backend_endpoints}
classified_endpoints: list[dict[str, Any]] = []
for endpoint in backend_endpoints:
key = endpoint_key(endpoint)
canonical_path = key[2]
static_webui_reference = canonical_path in frontend_paths
declaration = endpoint_declarations.get(key)
surface = (
{
"category": "ui_reachable",
"rationale": "A canonical API path reference exists in WebUI source.",
"source": "static_webui_scan",
}
if static_webui_reference
else (
{**declaration, "source": "declaration"}
if declaration is not None
else None
)
)
classified_endpoints.append(
{
**endpoint,
"canonical_path": canonical_path,
"static_webui_reference": static_webui_reference,
"surface": surface,
}
)
unreferenced = [
endpoint
for endpoint in backend_endpoints
if canonical_api_path(endpoint["path"]) not in frontend_paths
for endpoint in classified_endpoints
if not endpoint["static_webui_reference"]
]
unclassified = [
endpoint for endpoint in unreferenced if endpoint["surface"] is None
]
stale_declarations = [
declaration
for key, declaration in endpoint_declarations.items()
if key not in endpoint_keys
]
classification_counts = Counter(
endpoint["surface"]["category"]
for endpoint in classified_endpoints
if endpoint["surface"] is not None
)
usages = {item["key"] for item in webui["translationUsages"]}
catalogs = webui["translationCatalog"]
catalog_keys = {
locale: set(entries)
for locale, entries in catalogs.items()
}
catalog_keys = {locale: set(entries) for locale, entries in catalogs.items()}
expected_locales = sorted(catalog_keys)
missing_catalog_entries = [
{
"key": key,
"missing_locales": [
locale
for locale in expected_locales
if key not in catalog_keys[locale]
locale for locale in expected_locales if key not in catalog_keys[locale]
],
}
for key in sorted(usages)
@@ -311,9 +384,12 @@ def _assemble_inventory(
"missing_catalog_entries": missing_catalog_entries,
},
"api": {
"backend_endpoints": backend_endpoints,
"backend_endpoints": classified_endpoints,
"frontend_references": frontend_refs,
"unreferenced_by_static_webui_scan": unreferenced,
"unclassified_endpoints": unclassified,
"stale_endpoint_declarations": stale_declarations,
"classification_counts": dict(sorted(classification_counts.items())),
},
"summary": {
"modules": len(manifests),
@@ -326,6 +402,8 @@ def _assemble_inventory(
"backend_endpoints": len(backend_endpoints),
"frontend_api_references": len(frontend_refs),
"backend_endpoints_without_static_webui_reference": len(unreferenced),
"unclassified_backend_endpoints": len(unclassified),
"stale_endpoint_declarations": len(stale_declarations),
},
}
@@ -340,6 +418,7 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
item["repository"]
for item in inventory["api"]["unreferenced_by_static_webui_scan"]
)
classification_counts = inventory["api"]["classification_counts"]
lines = [
"# GovOPlaN Platform Interface Inventory",
"",
@@ -360,6 +439,8 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
"- Backend endpoints without a static WebUI reference: "
f"{summary['backend_endpoints_without_static_webui_reference']}"
),
f"- Unclassified backend endpoints: {summary['unclassified_backend_endpoints']}",
f"- Stale endpoint declarations: {summary['stale_endpoint_declarations']}",
f"- Used translation keys missing from a locale catalog: {len(missing)}",
"",
"## Help Review Candidates",
@@ -388,6 +469,19 @@ def _render_markdown(inventory: dict[str, Any]) -> str:
f"| `{repository}` | {count} |"
for repository, count in sorted(endpoint_by_repository.items())
)
lines.extend(
[
"",
"## Endpoint Surface Classifications",
"",
"| Classification | Endpoints |",
"| --- | ---: |",
]
)
lines.extend(
f"| `{category}` | {count} |"
for category, count in sorted(classification_counts.items())
)
lines.extend(
[
"",
@@ -408,11 +502,89 @@ def canonical_api_path(value: str) -> str:
if "/api/" in path:
path = path[path.index("/api/") :]
path = re.sub(r"^/api/v\d+", "", path)
path = re.sub(r"(?<=[^/])\$\{[^{}]*\}$", "", path)
path = PATH_PARAMETER.sub("{}", path)
path = re.sub(r"/+", "/", path)
return path.rstrip("/") or "/"
def endpoint_key(endpoint: dict[str, Any]) -> tuple[str, str, str]:
return (
str(endpoint["repository"]),
str(endpoint["method"]).upper(),
canonical_api_path(str(endpoint["path"])),
)
def _load_endpoint_declarations(
path: Path,
) -> dict[tuple[str, str, str], dict[str, Any]]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(
f"Endpoint declaration registry does not exist: {path}"
) from exc
except json.JSONDecodeError as exc:
raise ValueError(
f"Endpoint declaration registry is invalid JSON: {exc}"
) from exc
if not isinstance(payload, dict) or payload.get("schema_version") != 1:
raise ValueError("Endpoint declaration registry must use schema_version 1.")
entries = payload.get("endpoints")
if not isinstance(entries, list):
raise ValueError(
"Endpoint declaration registry must contain an endpoints list."
)
declarations: dict[tuple[str, str, str], dict[str, Any]] = {}
for index, entry in enumerate(entries):
if not isinstance(entry, dict):
raise ValueError(f"Endpoint declaration {index} must be an object.")
repository = entry.get("repository")
method = entry.get("method")
raw_path = entry.get("path")
category = entry.get("category")
rationale = entry.get("rationale")
if not isinstance(repository, str) or not repository.strip():
raise ValueError(f"Endpoint declaration {index} has no repository.")
if not isinstance(method, str) or method.lower() not in HTTP_METHODS:
raise ValueError(f"Endpoint declaration {index} has an invalid method.")
if not isinstance(raw_path, str) or not raw_path.startswith("/"):
raise ValueError(f"Endpoint declaration {index} has an invalid path.")
canonical_path = canonical_api_path(raw_path)
if raw_path != canonical_path:
raise ValueError(
f"Endpoint declaration {index} path must be canonical: {canonical_path}"
)
if category not in ENDPOINT_SURFACE_CATEGORIES:
raise ValueError(f"Endpoint declaration {index} has an invalid category.")
if not isinstance(rationale, str) or not rationale.strip():
raise ValueError(f"Endpoint declaration {index} has no rationale.")
tracking_issue = entry.get("tracking_issue")
if category == "missing_ui" and (
not isinstance(tracking_issue, str) or not tracking_issue.strip()
):
raise ValueError(
f"Endpoint declaration {index} requires a tracking_issue for missing_ui."
)
key = (repository, method.upper(), canonical_path)
if key in declarations:
raise ValueError(f"Duplicate endpoint declaration: {key!r}.")
declarations[key] = {
"repository": repository,
"method": method.upper(),
"path": canonical_path,
"category": category,
"rationale": rationale.strip(),
**(
{"tracking_issue": tracking_issue.strip()}
if isinstance(tracking_issue, str) and tracking_issue.strip()
else {}
),
}
return declarations
def _join_route(prefix: str, route: str) -> str:
return f"/{prefix.strip('/')}/{route.strip('/')}".replace("//", "/")
@@ -444,17 +616,11 @@ def _call_name(node: ast.AST) -> str:
def _plain_value(value: Any) -> Any:
if is_dataclass(value):
return {
key: _plain_value(item)
for key, item in asdict(value).items()
}
return {key: _plain_value(item) for key, item in asdict(value).items()}
if isinstance(value, tuple):
return [_plain_value(item) for item in value]
if isinstance(value, dict):
return {
str(key): _plain_value(item)
for key, item in value.items()
}
return {str(key): _plain_value(item) for key, item in value.items()}
return value