"""Collect sanitized evidence for the Kubernetes multi-host profile.""" from __future__ import annotations from collections.abc import Callable, Mapping, Sequence from datetime import UTC, datetime import json import shutil import subprocess import time from typing import Any from urllib.request import Request, urlopen JsonObject = dict[str, Any] CommandRunner = Callable[[Sequence[str]], JsonObject] JsonFetcher = Callable[[str, str], JsonObject] def collect_kubernetes_evidence( *, installation_id: str, namespace: str, ops_url: str, api_key: str, exercise_api_pod_loss: bool = False, timeout_seconds: float = 180.0, command_runner: CommandRunner | None = None, json_fetcher: JsonFetcher | None = None, ) -> JsonObject: """Inspect a live cluster and optionally exercise one API pod replacement.""" run_json = command_runner or _kubectl_json fetch_json = json_fetcher or _fetch_json nodes = run_json(("get", "nodes", "-o", "json")) pods = run_json( ( "-n", namespace, "get", "pods", "-l", f"app.kubernetes.io/instance={installation_id}", "-o", "json", ) ) deployments = run_json( ( "-n", namespace, "get", "deployments", "-l", f"app.kubernetes.io/instance={installation_id}", "-o", "json", ) ) ops = fetch_json(ops_url, api_key) snapshot = _evaluate_snapshot( installation_id=installation_id, namespace=namespace, nodes=nodes, pods=pods, deployments=deployments, ops=ops, ) replacement: JsonObject | None = None if exercise_api_pod_loss: replacement = _exercise_api_pod_loss( installation_id=installation_id, namespace=namespace, ops_url=ops_url, api_key=api_key, initial_pods=pods, timeout_seconds=timeout_seconds, run_json=run_json, fetch_json=fetch_json, ) evidence = { "schema_version": 1, "evidence_kind": "govoplan.kubernetes-multi-host", "collected_at": datetime.now(UTC).isoformat(), "installation_id": installation_id, "namespace": namespace, "snapshot": snapshot, "api_pod_loss": replacement, } failures = [ check["id"] for check in snapshot["checks"] if check["state"] != "passed" ] if replacement and replacement["state"] != "passed": failures.append("api_pod_loss") evidence["result"] = { "state": "passed" if not failures else "failed", "failed_checks": failures, } return evidence def _evaluate_snapshot( *, installation_id: str, namespace: str, nodes: Mapping[str, Any], pods: Mapping[str, Any], deployments: Mapping[str, Any], ops: Mapping[str, Any], ) -> JsonObject: ready_nodes = [ item for item in _items(nodes) if not bool(item.get("spec", {}).get("unschedulable")) and _condition(item, "Ready") == "True" ] pod_rows = [_pod_summary(item) for item in _items(pods)] deployment_rows = [_deployment_summary(item) for item in _items(deployments)] ready_api = [ item for item in pod_rows if item["component"] == "api" and item["ready"] ] ready_web = [ item for item in pod_rows if item["component"] == "web" and item["ready"] ] runtime = ops.get("runtime_cluster") runtime = runtime if isinstance(runtime, Mapping) else {} checks = ops.get("checks") check_rows = checks if isinstance(checks, list) else [] check_by_id = { str(item.get("id")): item for item in check_rows if isinstance(item, Mapping) } ops_readiness = ops.get("readiness") ops_readiness = ops_readiness if isinstance(ops_readiness, Mapping) else {} snapshot_checks = [ _check( "ready_nodes", len(ready_nodes) >= 2, f"{len(ready_nodes)} ready schedulable node(s)", ), _check( "api_node_spread", len(ready_api) >= 2 and len({item["node"] for item in ready_api}) >= 2, f"{len(ready_api)} ready API pod(s) on {len({item['node'] for item in ready_api})} node(s)", ), _check( "web_node_spread", len(ready_web) >= 2 and len({item["node"] for item in ready_web}) >= 2, f"{len(ready_web)} ready WebUI pod(s) on {len({item['node'] for item in ready_web})} node(s)", ), _check( "deployment_availability", bool(deployment_rows) and all(item["available"] >= item["desired"] for item in deployment_rows), f"{len(deployment_rows)} deployment(s) inspected", ), _check( "ops_readiness", bool(ops_readiness.get("ready")), str(ops_readiness.get("detail") or "Ops readiness response inspected"), ), _check( "runtime_composition", not bool(runtime.get("composition", {}).get("skewed")), "Active runtime composition matches the loaded module graph", ), _check( "runtime_versions", not bool(runtime.get("software_versions", {}).get("skewed")), "Active runtime software versions are consistent", ), _check( "worker_queue_coverage", not bool(runtime.get("queues", {}).get("missing")), "Every configured queue has an active worker", ), _check( "database_connection_budget", str(check_by_id.get("database_capacity", {}).get("state")) == "ok", str( check_by_id.get("database_capacity", {}).get("detail") or "Database capacity check is unavailable" ), ), ] return { "installation_id": installation_id, "namespace": namespace, "ready_node_names": sorted(_name(item) for item in ready_nodes), "pods": sorted(pod_rows, key=lambda item: item["name"]), "deployments": sorted(deployment_rows, key=lambda item: item["name"]), "runtime": { "expected": dict(runtime.get("expected") or {}), "active": dict(runtime.get("active") or {}), "composition": dict(runtime.get("composition") or {}), "software_versions": dict(runtime.get("software_versions") or {}), "queues": dict(runtime.get("queues") or {}), }, "database_capacity": dict( check_by_id.get("database_capacity", {}).get("metrics") or {} ), "checks": snapshot_checks, } def _exercise_api_pod_loss( *, installation_id: str, namespace: str, ops_url: str, api_key: str, initial_pods: Mapping[str, Any], timeout_seconds: float, run_json: CommandRunner, fetch_json: JsonFetcher, ) -> JsonObject: candidates = [ _pod_summary(item) for item in _items(initial_pods) if item.get("metadata", {}).get("labels", {}).get("app.kubernetes.io/component") == "api" and _condition(item, "Ready") == "True" ] if len(candidates) < 2: return { "state": "failed", "detail": "At least two ready API pods are required for the loss drill.", } victim = sorted(candidates, key=lambda item: item["name"])[0] initial_uids = {item["uid"] for item in candidates} desired_ready = len(candidates) run_json( ( "-n", namespace, "delete", "pod", victim["name"], "--wait=false", "-o", "json", ) ) deadline = time.monotonic() + timeout_seconds health_failures = 0 replacement: JsonObject | None = None while time.monotonic() < deadline: try: current_ops = fetch_json(ops_url, api_key) if not bool(current_ops.get("readiness", {}).get("ready")): health_failures += 1 except (OSError, ValueError): health_failures += 1 current = run_json( ( "-n", namespace, "get", "pods", "-l", f"app.kubernetes.io/instance={installation_id},app.kubernetes.io/component=api", "-o", "json", ) ) ready = [ _pod_summary(item) for item in _items(current) if _condition(item, "Ready") == "True" ] new_pods = [item for item in ready if item["uid"] not in initial_uids] if len(ready) >= desired_ready and new_pods: replacement = sorted(new_pods, key=lambda item: item["name"])[-1] break time.sleep(2.0) passed = replacement is not None and health_failures == 0 return { "state": "passed" if passed else "failed", "victim": victim, "replacement": replacement, "readiness_failures": health_failures, "detail": ( "API pod was replaced without an observed readiness outage." if passed else "API replacement timed out or readiness failed during replacement." ), } def _kubectl_json(arguments: Sequence[str]) -> JsonObject: kubectl = shutil.which("kubectl") if kubectl is None: raise ValueError("kubectl is required for Kubernetes evidence collection") result = subprocess.run( (kubectl, *arguments), check=False, capture_output=True, text=True, timeout=60, ) if result.returncode: detail = result.stderr.strip() or result.stdout.strip() raise ValueError(f"kubectl failed: {detail}") try: payload = json.loads(result.stdout) except json.JSONDecodeError as exc: raise ValueError("kubectl did not return JSON") from exc if not isinstance(payload, dict): raise ValueError("kubectl returned a non-object JSON payload") return payload def _fetch_json(url: str, api_key: str) -> JsonObject: request = Request( url, headers={"Accept": "application/json", "X-API-Key": api_key}, ) with urlopen(request, timeout=15) as response: payload = json.load(response) if not isinstance(payload, dict): raise ValueError("Ops API returned a non-object JSON payload") return payload def _items(payload: Mapping[str, Any]) -> list[Mapping[str, Any]]: items = payload.get("items") if not isinstance(items, list): raise ValueError("Kubernetes list response has no items") return [item for item in items if isinstance(item, Mapping)] def _condition(item: Mapping[str, Any], condition_type: str) -> str | None: conditions = item.get("status", {}).get("conditions", []) for condition in conditions if isinstance(conditions, list) else []: if isinstance(condition, Mapping) and condition.get("type") == condition_type: return str(condition.get("status")) return None def _name(item: Mapping[str, Any]) -> str: return str(item.get("metadata", {}).get("name") or "") def _pod_summary(item: Mapping[str, Any]) -> JsonObject: metadata = item.get("metadata", {}) labels = metadata.get("labels", {}) return { "name": _name(item), "uid": str(metadata.get("uid") or ""), "component": str(labels.get("app.kubernetes.io/component") or ""), "worker_pool": labels.get("govoplan.add-ideas.de/worker-pool"), "node": str(item.get("spec", {}).get("nodeName") or ""), "ready": _condition(item, "Ready") == "True", } def _deployment_summary(item: Mapping[str, Any]) -> JsonObject: status = item.get("status", {}) return { "name": _name(item), "component": str( item.get("metadata", {}) .get("labels", {}) .get("app.kubernetes.io/component") or "" ), "desired": int(item.get("spec", {}).get("replicas") or 0), "available": int(status.get("availableReplicas") or 0), "updated": int(status.get("updatedReplicas") or 0), } def _check(check_id: str, passed: bool, detail: str) -> JsonObject: return { "id": check_id, "state": "passed" if passed else "failed", "detail": detail, } __all__ = ["collect_kubernetes_evidence"]