Complete operational monitoring summary

src/govoplan_ops/backend/api/v1/routes.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import shutil
 import threading
 import time
 from collections.abc import Mapping
@@ -203,6 +204,9 @@ def _ops_status_payload(
         registry,
         force=force_module_checks,
     )
+    storage_check = _storage_check()
+    backup_check = _backup_restore_check()
+    recovery_metrics = runtime_cluster.get("recovery", {}).get("metrics", {})
     checks = [
         _check(
             "module_registry",
@@ -221,8 +225,8 @@ def _ops_status_payload(
         worker_check,
         database_capacity,
         _runtime_cluster_check(runtime_cluster),
-        _storage_check(),
-        _backup_restore_check(),
+        storage_check,
+        backup_check,
         _deployment_security_check(current_profile),
         *module_checks,
     ]
@@ -246,12 +250,19 @@ def _ops_status_payload(
                 core_settings.database_connection_available
             ),
             "file_storage_backend": core_settings.file_storage_backend,
+            "storage_metrics": storage_check.get("metrics", {}),
+            "backup_state": backup_check.get("state", "unknown"),
             "worker_metrics": worker_check.get("metrics", {}),
             "operational_probe_count": len(module_checks),
             "runtime_node_count": len(runtime_cluster.get("nodes", [])),
             "recovery_required_count": runtime_cluster.get("recovery", {}).get(
                 "requires_attention", 0
             ),
+            "failed_operation_count": int(recovery_metrics.get("failed") or 0),
+            "outcome_unknown_count": int(
+                recovery_metrics.get("outcome_unknown") or 0
+            ),
+            "active_operation_count": int(recovery_metrics.get("active") or 0),
         },
         "readiness": readiness,
         "checks": checks,
@@ -288,7 +299,11 @@ def _runtime_cluster_status(
             "detail": f"Runtime coordination query failed: {exc}",
             "state_profile": core_settings.state_profile,
             "nodes": [],
-            "recovery": {"operations": [], "requires_attention": 0},
+            "recovery": {
+                "operations": [],
+                "requires_attention": 0,
+                "metrics": _recovery_metrics([]),
+            },
         }
     active_nodes = [
         node for node in nodes if node["state"] == "active" and not node["stale"]
@@ -349,16 +364,7 @@ def _runtime_cluster_status(
         }
         for operation in operations
     ]
-    requires_attention = sum(
-        operation["status"]
-        in {
-            "outcome_unknown",
-            "recovery_required",
-            "recovering",
-            "manual_intervention",
-        }
-        for operation in operation_payloads
-    )
+    recovery_metrics = _recovery_metrics(operation_payloads)
     return {
         "available": True,
         "detail": "Shared runtime directory is available.",
@@ -388,11 +394,32 @@ def _runtime_cluster_status(
         "nodes": nodes,
         "recovery": {
             "operations": operation_payloads,
-            "requires_attention": requires_attention,
+            "requires_attention": recovery_metrics["requires_attention"],
+            "metrics": recovery_metrics,
         },
     }
 
 
+def _recovery_metrics(operations: list[dict[str, Any]]) -> dict[str, int]:
+    statuses = [str(operation.get("status") or "") for operation in operations]
+    failed = sum(value in {"failed", "manual_intervention"} for value in statuses)
+    outcome_unknown = statuses.count("outcome_unknown")
+    recovery_required = sum(
+        value in {"recovery_required", "recovering"} for value in statuses
+    )
+    active = sum(
+        value in {"planned", "prepared", "running", "recovering"}
+        for value in statuses
+    )
+    return {
+        "failed": failed,
+        "outcome_unknown": outcome_unknown,
+        "recovery_required": recovery_required,
+        "active": active,
+        "requires_attention": failed + outcome_unknown + recovery_required,
+    }
+
+
 def _runtime_cluster_check(cluster: dict[str, Any]) -> dict[str, Any]:
     if not cluster.get("available"):
         return _check(
@@ -841,6 +868,10 @@ def _storage_check() -> dict[str, Any]:
             if configured
             else "S3 file storage needs endpoint and bucket settings.",
             readiness_critical=not configured,
+            metrics={
+                "backend": "s3",
+                "capacity_observable": False,
+            },
         )
     root = Path(str(core_settings.file_storage_local_root or "runtime/files"))
     if root.exists() and root.is_dir():
@@ -851,12 +882,14 @@ def _storage_check() -> dict[str, Any]:
             if writable
             else f"Local file storage root is not writable: {root}"
         )
+        metrics = _local_storage_capacity(root)
         return _check(
             "file_storage",
             "File storage",
             state,
             detail,
             readiness_critical=not writable,
+            metrics=metrics,
         )
     return _check(
         "file_storage",
@@ -864,9 +897,28 @@ def _storage_check() -> dict[str, Any]:
         "warning",
         f"Local file storage root does not exist yet: {root}",
         readiness_critical=False,
+        metrics={"backend": "local", "capacity_observable": False},
     )
 
 
+def _local_storage_capacity(root: Path) -> dict[str, Any]:
+    try:
+        usage = shutil.disk_usage(root)
+    except OSError:
+        return {"backend": "local", "capacity_observable": False}
+    used_percent = (
+        round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
+    )
+    return {
+        "backend": "local",
+        "capacity_observable": True,
+        "capacity_total_bytes": int(usage.total),
+        "capacity_used_bytes": int(usage.used),
+        "capacity_free_bytes": int(usage.free),
+        "capacity_used_percent": used_percent,
+    }
+
+
 def _module_operational_checks(
     registry: PlatformRegistry,
     *,