From c241085806cdb1011a2746ef955d2fa0de19161b Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 1 Aug 2026 17:48:37 +0200 Subject: [PATCH] feat: expose institutional architecture and runtime state --- AGENTS.md | 6 + README.md | 18 ++ docs/SCALABILITY_PROFILES.md | 31 ++- pyproject.toml | 2 +- src/govoplan_ops/backend/api/v1/routes.py | 306 +++++++++++++++++++++- src/govoplan_ops/backend/manifest.py | 73 +++++- tests/test_governance_inventory.py | 86 +++++- tests/test_operational_checks.py | 19 ++ webui/src/api/ops.ts | 120 +++++++++ webui/src/features/ops/OpsPage.tsx | 255 +++++++++++++++++- 10 files changed, 906 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 732491a..822c764 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,11 @@ # GovOPlaN Ops Codex Guide +## Documentation Contract + +- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior. +- Keep feature content here; `govoplan-docs` projects it without importing Ops internals. +- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes. + ## Scope This repository owns the GovOPlaN operations module seed: runtime health, diff --git a/README.md b/README.md index 18ee62b..d4b2eec 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,15 @@ This repository owns: - backend module manifest `ops` - operator-facing status APIs +- runtime-node registration, heartbeat, composition, stale-node, and expected + replica projection +- audited API and worker drain/cancel controls +- recovery-operation status and evidence-chain summaries - governance inventory for module-declared permissions, roles, capabilities, policies, documentation, access-control hooks, search providers, and migration ownership +- sanitized runtime health, freshness, conflict, and recovery state for + configured external-provider bindings - deployment profile and sizing assumption summaries - WebUI route contribution `@govoplan/ops-webui` - future operational runbooks that describe the configured platform rather than @@ -28,7 +34,19 @@ Core exposes the registry contract but does not own an operations dashboard. Ops projects the provider-neutral registry metadata and runtime checks into the operator-facing governance surface. +Provider declarations describe supported behavior; module-owned runtime-state +providers describe the currently configured bindings. An optional provider +failure is isolated and reported as an attention state without suppressing the +rest of the governance inventory. Secrets, endpoints, and raw provider errors +are not part of this projection. + ## Runbooks - `docs/SCALABILITY_PROFILES.md` explains how to use the Ops page with the canonical sizing matrix, readiness model, and profile-selection worksheet. + +The Runtime cluster panel is backed by Core's shared PostgreSQL coordination +tables. A drain request is durable and is observed on the node heartbeat: API +readiness closes and workers stop taking new queue work. The Recovery panel +shows operations requiring forward recovery or manual intervention; it does not +claim that a production database backup exists. diff --git a/docs/SCALABILITY_PROFILES.md b/docs/SCALABILITY_PROFILES.md index d37d981..70b2751 100644 --- a/docs/SCALABILITY_PROFILES.md +++ b/docs/SCALABILITY_PROFILES.md @@ -15,6 +15,12 @@ live Ops page. campaigns, imports, exports, or workflow automation. 6. Record the current profile and open measurements before moving to a larger topology. +7. In a replicated profile, compare active non-stale API and worker counts with + configured expectations, and resolve composition skew before rollout. +8. Drain a node before replacement, then verify it is no longer ready/consuming + before terminating it. +9. Inspect recovery-required and outcome-unknown operations; follow the recorded + recovery mode rather than retrying or rolling back blindly. ## Live Profile Signals @@ -30,10 +36,24 @@ The Ops API reports: - HTTP/certificate deployment posture through the `deployment_security` check - readiness blockers - reference deployment profiles and sizing assumptions +- runtime node identity, role, software/module composition, queues, heartbeat, + stale state, and drain state +- configured versus active API and worker replica counts +- recovery operation status, mode, checkpoint count, and last update These values are intentionally diagnostic. They do not replace deployment configuration management, backups, monitoring, or restore drills. +Drain controls are cooperative. API and worker processes observe the request on +their next heartbeat. API readiness then fails; workers cancel their configured +queue consumers but may still be finishing already claimed work. Confirm the +process state and queue evidence before forcefully terminating a node. + +Recovery evidence is similarly conservative. A forward-recovery or +manual-intervention record means an operator must repair the current release or +restore a separately verified coordinated backup. Ops does not convert that +state into a safe rollback. + `deployment_security` is inactive for local/test profiles. In staging or pilot profiles it warns when secure cookies or CORS origins still look local. In production it becomes readiness-critical because TLS certificates, proxy @@ -93,23 +113,26 @@ Stateless and horizontally replicable: - API workers when `MASTER_KEY_B64`, `DATABASE_URL`, storage, queue, and module configuration are shared - Background workers when queues and idempotency keys are used -- Scheduler replicas only when leader election or an external lock exists +- Scheduler processes only through Core's renewable PostgreSQL lease and + fencing-token runner; deploy one desired scheduler replica Stateful or singleton-sensitive: - PostgreSQL -- local file storage when not replaced by object storage +- local file storage when not replaced by object storage (or a one-host shared + volume under the `host-shared` profile) - Redis/queue state - module installer daemon and package mutation operations - migration execution -- scheduler without distributed locking +- migration execution, although competing jobs are serialized by a PostgreSQL + advisory lock - outgoing campaign append/send jobs unless claim tokens are enforced ## Readiness And Degraded Modes | Component | Ready When | Degraded Mode | | --- | --- | --- | -| API | Database reachable, migrations current, enabled module registry builds, maintenance mode understood | Read-only/admin-only where routes allow it; otherwise fail closed. | +| API | Database reachable, migrations current, enabled module registry builds, maintenance mode understood, and the node is not draining | Read-only/admin-only where routes allow it; otherwise fail closed. | | WebUI | Static assets match backend module metadata contract | Show unavailable modules/routes with reason; do not invent routes. | | PostgreSQL | Accepts connections and migration head is current | Block writes and package changes if migration state is unknown. | | Storage | Configured backend is reachable and writable for write flows | Read-only file views may continue if storage is read-only but reachable. | diff --git a/pyproject.toml b/pyproject.toml index 79fa70f..fb3bcb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.12" authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.8", + "govoplan-core>=0.1.14", "govoplan-access>=0.1.8", ] diff --git a/src/govoplan_ops/backend/api/v1/routes.py b/src/govoplan_ops/backend/api/v1/routes.py index a050b19..11f49aa 100644 --- a/src/govoplan_ops/backend/api/v1/routes.py +++ b/src/govoplan_ops/backend/api/v1/routes.py @@ -9,9 +9,11 @@ from typing import Any from urllib.parse import urlsplit from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError +from govoplan_core.audit.logging import audit_event from govoplan_core.auth import ApiPrincipal, require_any_scope from govoplan_core.core.maintenance import saved_maintenance_mode from govoplan_core.core.module_installer import ( @@ -20,7 +22,20 @@ from govoplan_core.core.module_installer import ( read_module_installer_run, ) from govoplan_core.core.operations import OperationalCheckProviderRegistration +from govoplan_core.core.recovery import ( + RecoveryOperation, +) from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.core.provider_governance import ( + ExternalProviderStateContext, + collect_external_provider_states, +) +from govoplan_core.core.runtime_coordination import ( + RuntimeCoordinationError, + cancel_runtime_node_drain, + list_runtime_nodes, + request_runtime_node_drain, +) from govoplan_core.db.session import get_database from govoplan_core.settings import settings as core_settings @@ -33,6 +48,10 @@ _worker_check_cache: tuple[float, dict[str, Any]] | None = None _worker_check_cache_lock = threading.Lock() +class RuntimeDrainRequest(BaseModel): + reason: str = Field(default="operator request", min_length=1, max_length=500) + + @router.get("/status") def ops_status( request: Request, @@ -64,6 +83,91 @@ def run_ops_checks( return _ops_status_payload(request, force_module_checks=True) +@router.get("/runtime/nodes") +def runtime_nodes( + principal: ApiPrincipal = Depends(require_any_scope(*OPS_READ_SCOPES)), +) -> dict[str, Any]: + del principal + return _runtime_cluster_status() + + +@router.post("/runtime/nodes/{node_id}/drain") +def drain_runtime_node( + node_id: str, + body: RuntimeDrainRequest, + principal: ApiPrincipal = Depends(require_any_scope(*OPS_RUN_SCOPES)), +) -> dict[str, Any]: + try: + with get_database().SessionLocal() as session: + node = request_runtime_node_drain( + session, + installation_id=core_settings.installation_id, + node_id=node_id, + reason=body.reason, + ) + audit_event( + session, + tenant_id=None, + scope="system", + user_id=getattr(principal.user, "id", None), + api_key_id=principal.api_key_id, + action="ops.runtime_node.drain_requested", + object_type="runtime_node", + object_id=node.node_id, + details={ + "installation_id": core_settings.installation_id, + "reason": node.drain_reason, + "incarnation": node.incarnation, + }, + ) + session.commit() + return { + "node_id": node.node_id, + "state": node.state, + "drain_reason": node.drain_reason, + } + except RuntimeCoordinationError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + + +@router.delete("/runtime/nodes/{node_id}/drain") +def cancel_runtime_drain( + node_id: str, + principal: ApiPrincipal = Depends(require_any_scope(*OPS_RUN_SCOPES)), +) -> dict[str, Any]: + try: + with get_database().SessionLocal() as session: + node = cancel_runtime_node_drain( + session, + installation_id=core_settings.installation_id, + node_id=node_id, + ) + audit_event( + session, + tenant_id=None, + scope="system", + user_id=getattr(principal.user, "id", None), + api_key_id=principal.api_key_id, + action="ops.runtime_node.drain_cancelled", + object_type="runtime_node", + object_id=node.node_id, + details={ + "installation_id": core_settings.installation_id, + "incarnation": node.incarnation, + }, + ) + session.commit() + return {"node_id": node.node_id, "state": node.state} + except RuntimeCoordinationError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + + def _ops_status_payload( request: Request, *, @@ -75,6 +179,7 @@ def _ops_status_payload( redis_check = _redis_check() current_profile = _current_profile() worker_check = _worker_check() + runtime_cluster = _runtime_cluster_status() module_checks = _module_operational_checks( registry, force=force_module_checks, @@ -85,13 +190,17 @@ def _ops_status_payload( _maintenance_check(maintenance_mode), redis_check, worker_check, + _runtime_cluster_check(runtime_cluster), _storage_check(), _backup_restore_check(), _deployment_security_check(current_profile), *module_checks, ] readiness = _readiness(checks, maintenance_mode) - governance = _governance_inventory(registry) + governance = _governance_inventory( + registry, + external_provider_states=_external_provider_states(registry), + ) return { "summary": { "app_env": core_settings.app_env, @@ -105,15 +214,137 @@ def _ops_status_payload( "file_storage_backend": core_settings.file_storage_backend, "worker_metrics": worker_check.get("metrics", {}), "operational_probe_count": len(module_checks), + "runtime_node_count": len(runtime_cluster.get("nodes", [])), + "recovery_required_count": runtime_cluster.get("recovery", {}).get("requires_attention", 0), }, "readiness": readiness, "checks": checks, "governance": governance, "deployment_profiles": _deployment_profiles(current_profile), "sizing": _sizing_assumptions(), + "runtime_cluster": runtime_cluster, } +def _runtime_cluster_status() -> dict[str, Any]: + try: + with get_database().SessionLocal() as session: + nodes = list_runtime_nodes( + session, + installation_id=core_settings.installation_id, + stale_after_seconds=core_settings.runtime_stale_after_seconds, + ) + operations = ( + session.query(RecoveryOperation) + .filter( + RecoveryOperation.installation_id + == core_settings.installation_id + ) + .order_by(RecoveryOperation.updated_at.desc()) + .limit(50) + .all() + ) + except SQLAlchemyError as exc: + return { + "available": False, + "detail": f"Runtime coordination query failed: {exc}", + "state_profile": core_settings.state_profile, + "nodes": [], + "recovery": {"operations": [], "requires_attention": 0}, + } + active_nodes = [ + node + for node in nodes + if node["state"] != "stopped" and not node["stale"] + ] + active_api = sum(node["role"] == "api" for node in active_nodes) + active_workers = sum(node["role"] == "worker" for node in active_nodes) + operation_payloads = [ + { + "id": operation.id, + "module_id": operation.module_id, + "operation_type": operation.operation_type, + "resource_type": operation.resource_type, + "resource_id": operation.resource_id, + "mode": operation.mode, + "status": operation.status, + "checkpoint_count": operation.checkpoint_count, + "evidence_head_sha256": operation.evidence_head_sha256, + "failure_summary": operation.failure_summary, + "updated_at": operation.updated_at.isoformat(), + } + for operation in operations + ] + requires_attention = sum( + operation["status"] + in { + "outcome_unknown", + "recovery_required", + "recovering", + "manual_intervention", + } + for operation in operation_payloads + ) + return { + "available": True, + "detail": "Shared runtime directory is available.", + "installation_id": core_settings.installation_id, + "state_profile": core_settings.state_profile, + "expected": { + "api": core_settings.runtime_expected_api_replicas, + "worker": core_settings.runtime_expected_worker_replicas, + }, + "active": {"api": active_api, "worker": active_workers}, + "nodes": nodes, + "recovery": { + "operations": operation_payloads, + "requires_attention": requires_attention, + }, + } + + +def _runtime_cluster_check(cluster: dict[str, Any]) -> dict[str, Any]: + if not cluster.get("available"): + return _check( + "runtime_cluster", + "Runtime cluster", + "error", + str(cluster.get("detail") or "Runtime directory unavailable."), + readiness_critical=True, + ) + nodes = cluster.get("nodes") or [] + stale = [node for node in nodes if node.get("stale")] + expected = cluster.get("expected") or {} + active = cluster.get("active") or {} + missing_api = max(0, int(expected.get("api") or 0) - int(active.get("api") or 0)) + missing_workers = max(0, int(expected.get("worker") or 0) - int(active.get("worker") or 0)) + state = "ok" + detail = f"{len(nodes)} runtime node record(s); {len(stale)} stale." + readiness_critical = False + if missing_api or missing_workers: + state = "error" if cluster.get("state_profile") == "shared" else "warning" + readiness_critical = state == "error" + detail += f" Missing expected replicas: api={missing_api}, worker={missing_workers}." + elif stale: + state = "warning" + return _check( + "runtime_cluster", + "Runtime cluster", + state, + detail, + readiness_critical=readiness_critical, + metrics={ + "nodes": len(nodes), + "stale": len(stale), + "active_api": int(active.get("api") or 0), + "active_workers": int(active.get("worker") or 0), + "recovery_required": int( + cluster.get("recovery", {}).get("requires_attention") or 0 + ), + }, + ) + + def _registry(request: Request) -> PlatformRegistry: registry = getattr(request.app.state, "govoplan_registry", None) if not isinstance(registry, PlatformRegistry): @@ -121,10 +352,27 @@ def _registry(request: Request) -> PlatformRegistry: return registry -def _governance_inventory(registry: PlatformRegistry) -> dict[str, Any]: +def _governance_inventory( + registry: PlatformRegistry, + *, + external_provider_states: Mapping[str, Mapping[str, object]] | None = None, +) -> dict[str, Any]: modules: list[dict[str, Any]] = [] for manifest in registry.manifests(): capability_names = tuple(sorted(manifest.capability_factories)) + architecture = ( + manifest.architecture.to_dict() if manifest.architecture else None + ) + external_providers = tuple( + { + **declaration.to_dict(), + "runtime_state": dict( + (external_provider_states or {}).get(declaration.id, {}) + ) + or None, + } + for declaration in manifest.external_providers + ) access_control_count = ( len(manifest.resource_acl_providers) + len(manifest.ownership_providers) @@ -144,6 +392,9 @@ def _governance_inventory(registry: PlatformRegistry) -> dict[str, Any]: "access_control_count": access_control_count, "search_provider_count": len(manifest.search_providers) + len(manifest.search_sources), "migration_managed": manifest.migration_spec is not None, + "architecture": architecture, + "external_provider_count": len(external_providers), + "external_providers": external_providers, }) return { "summary": { @@ -159,11 +410,62 @@ def _governance_inventory(registry: PlatformRegistry) -> dict[str, Any]: "access_control_count": sum(item["access_control_count"] for item in modules), "search_provider_count": sum(item["search_provider_count"] for item in modules), "migration_module_count": sum(bool(item["migration_managed"]) for item in modules), + "architecture_declared_module_count": sum( + bool(item["architecture"]) for item in modules + ), + "external_provider_count": sum( + item["external_provider_count"] for item in modules + ), + "configured_external_provider_count": sum( + bool( + provider.get("runtime_state", {}).get("configured") + if isinstance(provider.get("runtime_state"), Mapping) + else False + ) + for item in modules + for provider in item["external_providers"] + ), + "provider_attention_count": sum( + ( + str(provider.get("runtime_state", {}).get("health")) + in {"warning", "error", "unknown"} + or str(provider.get("runtime_state", {}).get("recovery")) + == "attention" + ) + if isinstance(provider.get("runtime_state"), Mapping) + else False + for item in modules + for provider in item["external_providers"] + ), + "supported_module_count": sum( + bool(item["architecture"]) + and item["architecture"]["maturity"] in {"supported", "lts"} + for item in modules + ), }, "modules": modules, } +def _external_provider_states( + registry: PlatformRegistry, +) -> dict[str, dict[str, object]]: + registrations = registry.external_provider_state_providers() + if not registrations: + return {} + try: + with get_database().session() as session: + return collect_external_provider_states( + registrations, + ExternalProviderStateContext(session=session), + ) + except Exception: # noqa: BLE001 - preserve Ops inventory during DB outages. + return collect_external_provider_states( + registrations, + ExternalProviderStateContext(session=None), + ) + + def _database_status() -> dict[str, Any]: try: with get_database().session() as session: diff --git a/src/govoplan_ops/backend/manifest.py b/src/govoplan_ops/backend/manifest.py index bd8af6e..d6737dc 100644 --- a/src/govoplan_ops/backend/manifest.py +++ b/src/govoplan_ops/backend/manifest.py @@ -1,7 +1,8 @@ from __future__ import annotations from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER -from govoplan_core.core.modules import FrontendModule, FrontendRoute, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate +from govoplan_core.core.modules import DocumentationCondition, DocumentationTopic, FrontendModule, FrontendRoute, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate +from govoplan_core.core.provider_governance import ModuleArchitectureDeclaration, ModuleArchitectureDocumentation, ModuleMaturityEvidence from govoplan_core.core.views import ViewSurface OPS_READ_SCOPE = "ops:operations:read" @@ -9,6 +10,33 @@ OPS_READ_SCOPES = (OPS_READ_SCOPE, "system:settings:read", "admin:settings:read" OPS_RUN_SCOPE = "ops:operations:run" OPS_RUN_SCOPES = (OPS_RUN_SCOPE, "system:settings:write") +ARCHITECTURE = ModuleArchitectureDeclaration( + layer="runtime_meta", + kind="operations", + maturity="vertical_slice", + evidence=( + ModuleMaturityEvidence( + kind="test", + reference="tests/test_governance_inventory.py", + summary="Tests safe operational projection of module governance declarations.", + ), + ModuleMaturityEvidence( + kind="documentation", + reference="docs/SCALABILITY_PROFILES.md", + summary="Documents operational topology and scaling posture.", + ), + ), + known_limits=( + "Provider health observations depend on module-owned operational probes and may be unavailable until configured.", + ), + owned_concepts=("operations status projection", "bounded operational probes"), + non_owned_concepts=("domain repair", "external provider credentials"), + documentation=ModuleArchitectureDocumentation( + recovery=("docs/SCALABILITY_PROFILES.md",), + operations=("docs/SCALABILITY_PROFILES.md",), + ), +) + def _permission(scope: str, label: str, description: str) -> PermissionDefinition: module_id, resource, action = scope.split(":", 2) @@ -65,6 +93,48 @@ manifest = ModuleManifest( level="system", ), ), + documentation=( + DocumentationTopic( + id="ops.health-governance-and-sizing", + title="Inspect platform health and deployment posture", + summary="Ops combines module-owned health checks with deployment profile, governance inventory, worker assumptions, and sizing guidance.", + body="Read-only status distinguishes configured capabilities from healthy integrations. Authorized operators can run bounded probes; a probe must not perform unbounded business work or silently repair data. Use readiness and worker results when diagnosing a node, and use the deployment profile and sizing assumptions when planning horizontal capacity.", + documentation_types=("admin", "user"), + audience=("operator", "system_admin"), + related_modules=("audit", "docs", "notifications"), + metadata={"kind": "reference"}, + ), + DocumentationTopic( + id="ops.runtime-coordination-and-recovery", + title="Drain runtime nodes and inspect recovery evidence", + summary="Ops projects shared runtime heartbeats, replica gaps, drain controls, and recovery states that require operator attention.", + body="Use the runtime table to identify stale or composition-skewed API and worker replicas. Drain before replacement so API readiness closes and workers stop taking new queue work; cancellation is available while the node is still draining. The recovery table reports durable Core recovery operations, but a recorded forward-recovery or manual-intervention state is not an automatic database restore.", + documentation_types=("admin", "user"), + audience=("operator", "system_admin"), + conditions=( + DocumentationCondition( + required_modules=("ops",), + any_scopes=OPS_READ_SCOPES, + ), + ), + related_modules=("audit", "notifications"), + metadata={ + "kind": "workflow", + "route": "/ops", + "screen": "Runtime cluster and recovery evidence", + "steps": [ + "Compare active non-stale nodes with the configured API and worker replica expectations.", + "Request drain and wait for the node to report draining before replacing it.", + "Inspect every recovery-required, outcome-unknown, or manual-intervention operation and follow its recorded recovery mode.", + "Verify replacement composition, readiness, queue consumers, and recovery evidence before closing the operation.", + ], + "limitations": [ + "Drain is observed on the runtime heartbeat interval and does not forcibly terminate active work.", + "Ops does not create database backups or make an unsafe post-migration rollback reversible.", + ], + }, + ), + ), route_factory=_route_factory, nav_items=(NavItem(path="/ops", label="Ops", icon="activity", required_any=OPS_READ_SCOPES, order=890),), frontend=FrontendModule( @@ -76,6 +146,7 @@ manifest = ModuleManifest( ViewSurface(id="ops.widget.health", module_id="ops", kind="section", label="Operations health widget", order=100), ), ), + architecture=ARCHITECTURE, ) diff --git a/tests/test_governance_inventory.py b/tests/test_governance_inventory.py index 5626659..eee698b 100644 --- a/tests/test_governance_inventory.py +++ b/tests/test_governance_inventory.py @@ -11,6 +11,14 @@ from govoplan_core.core.modules import ( RoleTemplate, ) from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.core.provider_governance import ( + ExternalProviderDeclaration, + ModuleArchitectureDeclaration, + ModuleArchitectureDocumentation, + ModuleMaturityEvidence, + ProviderBehaviorDeclaration, + ProviderObjectDeclaration, +) from govoplan_ops.backend.api.v1.routes import _governance_inventory @@ -60,9 +68,70 @@ class GovernanceInventoryTests(unittest.TestCase): ), ), migration_spec=MigrationSpec(module_id="example"), + architecture=ModuleArchitectureDeclaration( + layer="governance_accountability", + kind="governance", + maturity="vertical_slice", + evidence=( + ModuleMaturityEvidence( + kind="test", + reference="tests/test_governance_inventory.py", + summary="Exercises the governance projection.", + ), + ModuleMaturityEvidence( + kind="documentation", + reference="example.reference", + summary="Documents the example provider.", + ), + ), + known_limits=("The example has no runtime implementation.",), + supported_authority_modes=("external_mirror",), + owned_concepts=("example projections",), + documentation=ModuleArchitectureDocumentation( + operations=("example.reference",), + ), + ), + external_providers=( + ExternalProviderDeclaration( + id="example.read", + module_id="example", + label="Example reader", + maturity="read", + operations=("read", "search"), + objects=( + ProviderObjectDeclaration( + object_type="example.record", + field_groups=("identity", "summary"), + authority_modes=("external_mirror",), + default_authority_mode="external_mirror", + ), + ), + behavior=ProviderBehaviorDeclaration( + freshness="Request-time read with a five-minute cache.", + health="Reported through example.health.", + max_read_items=100, + outage="Reads fail closed and retain the last inspected snapshot.", + classifications=("internal",), + purposes=("operations",), + retention="No provider-owned payload is retained.", + ), + documentation_topic_ids=("example.reference",), + ), + ), )) - payload = _governance_inventory(registry) + payload = _governance_inventory( + registry, + external_provider_states={ + "example.read": { + "configured": True, + "active": True, + "health": "warning", + "freshness": "stale", + "recovery": "attention", + } + }, + ) self.assertEqual(payload["summary"]["module_count"], 1) self.assertEqual(payload["summary"]["permission_count"], 1) @@ -71,6 +140,21 @@ class GovernanceInventoryTests(unittest.TestCase): self.assertEqual(payload["summary"]["policy_count"], 1) self.assertEqual(payload["summary"]["documented_module_count"], 1) self.assertEqual(payload["summary"]["migration_module_count"], 1) + self.assertEqual(payload["summary"]["architecture_declared_module_count"], 1) + self.assertEqual(payload["summary"]["external_provider_count"], 1) + self.assertEqual( + payload["summary"]["configured_external_provider_count"], + 1, + ) + self.assertEqual(payload["summary"]["provider_attention_count"], 1) + self.assertEqual(payload["modules"][0]["architecture"]["maturity"], "vertical_slice") + self.assertEqual(payload["modules"][0]["external_providers"][0]["id"], "example.read") + self.assertEqual( + payload["modules"][0]["external_providers"][0]["runtime_state"][ + "freshness" + ], + "stale", + ) self.assertEqual(payload["modules"][0]["module_id"], "example") self.assertNotIn("capability_factories", payload["modules"][0]) self.assertFalse(any( diff --git a/tests/test_operational_checks.py b/tests/test_operational_checks.py index 4c41b7c..92bee4c 100644 --- a/tests/test_operational_checks.py +++ b/tests/test_operational_checks.py @@ -63,3 +63,22 @@ def test_module_operational_check_failure_is_isolated() -> None: assert result["state"] == "error" assert "secret detail" not in result["detail"] + +def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None: + check = routes._runtime_cluster_check( + { + "available": True, + "state_profile": "shared", + "nodes": [ + {"role": "api", "state": "active", "stale": False}, + {"role": "worker", "state": "active", "stale": True}, + ], + "expected": {"api": 2, "worker": 1}, + "active": {"api": 1, "worker": 0}, + "recovery": {"requires_attention": 1}, + } + ) + + assert check["state"] == "error" + assert check["readiness_critical"] is True + assert check["metrics"]["recovery_required"] == 1 diff --git a/webui/src/api/ops.ts b/webui/src/api/ops.ts index dc8b729..a3b72fe 100644 --- a/webui/src/api/ops.ts +++ b/webui/src/api/ops.ts @@ -37,6 +37,91 @@ export type OpsGovernanceModule = { access_control_count: number; search_provider_count: number; migration_managed: boolean; + architecture?: { + contract_version: string; + layer: string; + kind: string; + maturity: string; + known_limits: string[]; + supported_authority_modes: string[]; + owned_concepts: string[]; + non_owned_concepts: string[]; + evidence: Array<{ kind: string; reference: string; summary: string }>; + } | null; + external_provider_count: number; + external_providers: Array<{ + id: string; + label: string; + maturity: string; + authority_modes: string[]; + operations: string[]; + behavior: { outage?: string | null }; + runtime_state?: { + configured: boolean; + active: boolean; + authority_mode?: string | null; + authority_modes?: string[]; + health: string; + freshness: string; + conflict: string; + recovery: string; + observed_at: string; + last_success_at?: string | null; + bindings?: Array<{ + binding_ref: string; + authority_mode: string; + active: boolean; + health: string; + freshness: string; + conflict: string; + recovery: string; + }>; + } | null; + }>; +}; + +export type OpsRuntimeNode = { + node_id: string; + incarnation: string; + role: string; + software_version: string; + composition_hash: string; + queues: string[]; + state: "active" | "draining" | "stopped" | string; + started_at: string; + last_heartbeat_at: string; + drain_requested_at?: string | null; + drain_reason?: string | null; + stopped_at?: string | null; + stale: boolean; +}; + +export type OpsRecoveryOperation = { + id: string; + module_id: string; + operation_type: string; + resource_type?: string | null; + resource_id?: string | null; + mode: string; + status: string; + checkpoint_count: number; + evidence_head_sha256?: string | null; + failure_summary?: string | null; + updated_at: string; +}; + +export type OpsRuntimeCluster = { + available: boolean; + detail: string; + installation_id?: string; + state_profile: string; + expected?: { api: number; worker: number }; + active?: { api: number; worker: number }; + nodes: OpsRuntimeNode[]; + recovery: { + operations: OpsRecoveryOperation[]; + requires_attention: number; + }; }; export type OpsStatus = { @@ -62,6 +147,8 @@ export type OpsStatus = { queue_depths?: Record; }; operational_probe_count: number; + runtime_node_count: number; + recovery_required_count: number; }; readiness: { ready: boolean; @@ -85,11 +172,17 @@ export type OpsStatus = { access_control_count: number; search_provider_count: number; migration_module_count: number; + architecture_declared_module_count: number; + external_provider_count: number; + configured_external_provider_count: number; + provider_attention_count: number; + supported_module_count: number; }; modules: OpsGovernanceModule[]; }; deployment_profiles: OpsDeploymentProfile[]; sizing: OpsSizingAssumption[]; + runtime_cluster: OpsRuntimeCluster; }; export function fetchOpsStatus(settings: ApiSettings): Promise { @@ -99,3 +192,30 @@ export function fetchOpsStatus(settings: ApiSettings): Promise { export function runOpsChecks(settings: ApiSettings): Promise { return apiFetch(settings, "/api/v1/ops/checks/run", { method: "POST" }); } + +export function drainRuntimeNode( + settings: ApiSettings, + nodeId: string, + reason = "operator request" +): Promise<{ node_id: string; state: string; drain_reason: string }> { + return apiFetch( + settings, + `/api/v1/ops/runtime/nodes/${encodeURIComponent(nodeId)}/drain`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ reason }) + } + ); +} + +export function cancelRuntimeNodeDrain( + settings: ApiSettings, + nodeId: string +): Promise<{ node_id: string; state: string }> { + return apiFetch( + settings, + `/api/v1/ops/runtime/nodes/${encodeURIComponent(nodeId)}/drain`, + { method: "DELETE" } + ); +} diff --git a/webui/src/features/ops/OpsPage.tsx b/webui/src/features/ops/OpsPage.tsx index a238ac7..004719f 100644 --- a/webui/src/features/ops/OpsPage.tsx +++ b/webui/src/features/ops/OpsPage.tsx @@ -1,8 +1,9 @@ import { useEffect, useState } from "react"; -import { RefreshCw } from "lucide-react"; +import { PauseCircle, PlayCircle, RefreshCw } from "lucide-react"; import { Button, Card, + ConfirmDialog, DataGrid, DismissibleAlert, LoadingFrame, @@ -18,10 +19,14 @@ import { "@govoplan/core-webui"; import { fetchOpsStatus, + cancelRuntimeNodeDrain, + drainRuntimeNode, runOpsChecks, type OpsCheck, type OpsDeploymentProfile, type OpsGovernanceModule, + type OpsRecoveryOperation, + type OpsRuntimeNode, type OpsSizingAssumption, type OpsStatus } from "../../api/ops"; @@ -30,6 +35,8 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: const [status, setStatus] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(""); + const [drainTarget, setDrainTarget] = useState(null); + const [nodeActionId, setNodeActionId] = useState(""); async function load() { setLoading(true); @@ -55,6 +62,34 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: } } + async function confirmDrain() { + if (!drainTarget) return; + setNodeActionId(drainTarget.node_id); + setError(""); + try { + await drainRuntimeNode(settings, drainTarget.node_id); + setDrainTarget(null); + setStatus(await fetchOpsStatus(settings)); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setNodeActionId(""); + } + } + + async function cancelDrain(node: OpsRuntimeNode) { + setNodeActionId(node.node_id); + setError(""); + try { + await cancelRuntimeNodeDrain(settings, node.node_id); + setStatus(await fetchOpsStatus(settings)); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setNodeActionId(""); + } + } + useEffect(() => {void load();}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]); const checks = status?.checks ?? []; @@ -89,6 +124,9 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: + + +
@@ -96,6 +134,20 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: + + void cancelDrain(node)} + /> + + + + + + @@ -109,11 +161,159 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
+ setDrainTarget(null)} + onConfirm={() => void confirmDrain()} + /> ); } +function RuntimeNodeTable({ + nodes, + canManage, + busyNodeId, + onDrain, + onCancelDrain +}: { + nodes: OpsRuntimeNode[]; + canManage: boolean; + busyNodeId: string; + onDrain: (node: OpsRuntimeNode) => void; + onCancelDrain: (node: OpsRuntimeNode) => void; +}) { + const columns: DataGridColumn[] = [ + { + id: "node", + header: "Node", + width: "minmax(220px, 1fr)", + minWidth: 200, + resizable: true, + sortable: true, + filterable: true, + value: (node) => `${node.node_id} ${node.role}`, + render: (node) =>
{node.node_id}{node.role} · {node.software_version}
+ }, + { + id: "state", + header: "State", + width: 150, + sortable: true, + filterable: true, + value: (node) => node.stale ? "stale" : node.state, + render: (node) => + }, + { + id: "heartbeat", + header: "Last heartbeat", + width: "minmax(210px, .8fr)", + minWidth: 190, + resizable: true, + sortable: true, + value: (node) => node.last_heartbeat_at, + render: (node) => new Date(node.last_heartbeat_at).toLocaleString() + }, + { + id: "queues", + header: "Queues", + width: "minmax(180px, .7fr)", + minWidth: 160, + resizable: true, + filterable: true, + value: (node) => node.queues.join(" "), + render: (node) => node.queues.join(", ") || "-" + }, + { + id: "actions", + header: "", + width: 56, + value: (node) => node.state, + render: (node) => { + if (!canManage || node.state === "stopped") return null; + const busy = busyNodeId === node.node_id; + return node.state === "draining" ? ( + + ) : ( + + ); + } + } + ]; + return `${node.node_id}:${node.incarnation}`} emptyText="No runtime nodes reported." />; +} + +function RecoveryTable({ operations }: { operations: OpsRecoveryOperation[] }) { + const columns: DataGridColumn[] = [ + { + id: "operation", + header: "Operation", + width: "minmax(240px, 1fr)", + minWidth: 220, + resizable: true, + sortable: true, + filterable: true, + value: (operation) => `${operation.module_id} ${operation.operation_type}`, + render: (operation) =>
{operation.operation_type}{operation.module_id} · {operation.mode}
+ }, + { + id: "resource", + header: "Resource", + width: "minmax(180px, .8fr)", + minWidth: 160, + resizable: true, + filterable: true, + value: (operation) => `${operation.resource_type ?? ""} ${operation.resource_id ?? ""}`, + render: (operation) => operation.resource_type ? `${operation.resource_type}: ${operation.resource_id ?? "-"}` : "-" + }, + { + id: "status", + header: "Status", + width: 170, + sortable: true, + filterable: true, + value: (operation) => operation.status, + render: (operation) => + }, + { + id: "evidence", + header: "Evidence", + width: 150, + sortable: true, + value: (operation) => operation.checkpoint_count, + render: (operation) => `${operation.checkpoint_count} checkpoint(s)` + }, + { + id: "updated", + header: "Updated", + width: "minmax(210px, .8fr)", + minWidth: 190, + resizable: true, + sortable: true, + value: (operation) => operation.updated_at, + render: (operation) => new Date(operation.updated_at).toLocaleString() + } + ]; + return operation.id} emptyText="No recovery operations recorded." />; +} + function GovernanceTable({ modules }: { modules: OpsGovernanceModule[] }) { const columns: DataGridColumn[] = [ { @@ -145,6 +345,44 @@ function GovernanceTable({ modules }: { modules: OpsGovernanceModule[] }) { value: (module) => `${module.capability_count} ${module.policy_count}`, render: (module) => `${module.capability_count} capabilities · ${module.policy_count} policies` }, + { + id: "architecture", + header: "Architecture", + width: "minmax(210px, .9fr)", + minWidth: 190, + resizable: true, + sortable: true, + filterable: true, + value: (module) => module.architecture + ? `${module.architecture.layer} ${module.architecture.kind} ${module.architecture.maturity}` + : "undeclared", + render: (module) => module.architecture ? ( +
+ {module.architecture.maturity} + {module.architecture.layer} · {module.architecture.kind} + {module.architecture.known_limits.length ? {module.architecture.known_limits.length} known limit(s) : null} +
+ ) : staged declaration pending + }, + { + id: "providers", + header: "External providers", + width: "minmax(230px, 1fr)", + minWidth: 210, + resizable: true, + filterable: true, + value: (module) => `${module.external_provider_count} ${module.external_providers.map((provider) => `${provider.id} ${provider.maturity} ${provider.authority_modes.join(" ")} ${provider.runtime_state?.health ?? "unobserved"} ${provider.runtime_state?.freshness ?? ""}`).join(" ")}`, + render: (module) => module.external_provider_count ? ( +
+ {module.external_provider_count} declared + {module.external_providers.map((provider) => ( + + {provider.label}: {provider.maturity} · {provider.runtime_state ? `${provider.runtime_state.health}/${provider.runtime_state.freshness} · ${provider.runtime_state.bindings?.length ?? 0} binding(s)` : "unobserved"} + + ))} +
+ ) : none declared + }, { id: "controls", header: "i18n:govoplan-ops.controls.0cdb80fb", @@ -227,6 +465,21 @@ function stateTone(state: string): string { return "inactive"; } +function recoveryTone(state: string): string { + if (["succeeded", "recovered"].includes(state)) return "success"; + if (["failed", "recovery_required"].includes(state)) return "error"; + if (["running", "recovering", "prepared"].includes(state)) return "warning"; + return "inactive"; +} + +function runtimeNodeMetricDetail(status: OpsStatus | null): string { + const cluster = status?.runtime_cluster; + if (!cluster?.available) return cluster?.detail ?? "Runtime directory unavailable"; + const active = cluster.active ?? { api: 0, worker: 0 }; + const expected = cluster.expected ?? { api: 0, worker: 0 }; + return `${active.api}/${expected.api} API · ${active.worker}/${expected.worker} workers`; +} + function workerMetricDetail(status: OpsStatus | null): string { if (!status?.summary.celery_enabled) return "i18n:govoplan-ops.celery_worker_setting.323d7737"; const metrics = status.summary.worker_metrics;