381 lines
12 KiB
Python
381 lines
12 KiB
Python
"""Collect sanitized evidence for the Kubernetes multi-host profile."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from datetime import UTC, datetime
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from typing import Any
|
|
from urllib.request import Request, urlopen
|
|
|
|
|
|
JsonObject = dict[str, Any]
|
|
CommandRunner = Callable[[Sequence[str]], JsonObject]
|
|
JsonFetcher = Callable[[str, str], JsonObject]
|
|
|
|
|
|
def collect_kubernetes_evidence(
|
|
*,
|
|
installation_id: str,
|
|
namespace: str,
|
|
ops_url: str,
|
|
api_key: str,
|
|
exercise_api_pod_loss: bool = False,
|
|
timeout_seconds: float = 180.0,
|
|
command_runner: CommandRunner | None = None,
|
|
json_fetcher: JsonFetcher | None = None,
|
|
) -> JsonObject:
|
|
"""Inspect a live cluster and optionally exercise one API pod replacement."""
|
|
|
|
run_json = command_runner or _kubectl_json
|
|
fetch_json = json_fetcher or _fetch_json
|
|
nodes = run_json(("get", "nodes", "-o", "json"))
|
|
pods = run_json(
|
|
(
|
|
"-n",
|
|
namespace,
|
|
"get",
|
|
"pods",
|
|
"-l",
|
|
f"app.kubernetes.io/instance={installation_id}",
|
|
"-o",
|
|
"json",
|
|
)
|
|
)
|
|
deployments = run_json(
|
|
(
|
|
"-n",
|
|
namespace,
|
|
"get",
|
|
"deployments",
|
|
"-l",
|
|
f"app.kubernetes.io/instance={installation_id}",
|
|
"-o",
|
|
"json",
|
|
)
|
|
)
|
|
ops = fetch_json(ops_url, api_key)
|
|
snapshot = _evaluate_snapshot(
|
|
installation_id=installation_id,
|
|
namespace=namespace,
|
|
nodes=nodes,
|
|
pods=pods,
|
|
deployments=deployments,
|
|
ops=ops,
|
|
)
|
|
replacement: JsonObject | None = None
|
|
if exercise_api_pod_loss:
|
|
replacement = _exercise_api_pod_loss(
|
|
installation_id=installation_id,
|
|
namespace=namespace,
|
|
ops_url=ops_url,
|
|
api_key=api_key,
|
|
initial_pods=pods,
|
|
timeout_seconds=timeout_seconds,
|
|
run_json=run_json,
|
|
fetch_json=fetch_json,
|
|
)
|
|
evidence = {
|
|
"schema_version": 1,
|
|
"evidence_kind": "govoplan.kubernetes-multi-host",
|
|
"collected_at": datetime.now(UTC).isoformat(),
|
|
"installation_id": installation_id,
|
|
"namespace": namespace,
|
|
"snapshot": snapshot,
|
|
"api_pod_loss": replacement,
|
|
}
|
|
failures = [
|
|
check["id"] for check in snapshot["checks"] if check["state"] != "passed"
|
|
]
|
|
if replacement and replacement["state"] != "passed":
|
|
failures.append("api_pod_loss")
|
|
evidence["result"] = {
|
|
"state": "passed" if not failures else "failed",
|
|
"failed_checks": failures,
|
|
}
|
|
return evidence
|
|
|
|
|
|
def _evaluate_snapshot(
|
|
*,
|
|
installation_id: str,
|
|
namespace: str,
|
|
nodes: Mapping[str, Any],
|
|
pods: Mapping[str, Any],
|
|
deployments: Mapping[str, Any],
|
|
ops: Mapping[str, Any],
|
|
) -> JsonObject:
|
|
ready_nodes = [
|
|
item
|
|
for item in _items(nodes)
|
|
if not bool(item.get("spec", {}).get("unschedulable"))
|
|
and _condition(item, "Ready") == "True"
|
|
]
|
|
pod_rows = [_pod_summary(item) for item in _items(pods)]
|
|
deployment_rows = [_deployment_summary(item) for item in _items(deployments)]
|
|
ready_api = [
|
|
item for item in pod_rows if item["component"] == "api" and item["ready"]
|
|
]
|
|
ready_web = [
|
|
item for item in pod_rows if item["component"] == "web" and item["ready"]
|
|
]
|
|
runtime = ops.get("runtime_cluster")
|
|
runtime = runtime if isinstance(runtime, Mapping) else {}
|
|
checks = ops.get("checks")
|
|
check_rows = checks if isinstance(checks, list) else []
|
|
check_by_id = {
|
|
str(item.get("id")): item for item in check_rows if isinstance(item, Mapping)
|
|
}
|
|
ops_readiness = ops.get("readiness")
|
|
ops_readiness = ops_readiness if isinstance(ops_readiness, Mapping) else {}
|
|
snapshot_checks = [
|
|
_check(
|
|
"ready_nodes",
|
|
len(ready_nodes) >= 2,
|
|
f"{len(ready_nodes)} ready schedulable node(s)",
|
|
),
|
|
_check(
|
|
"api_node_spread",
|
|
len(ready_api) >= 2 and len({item["node"] for item in ready_api}) >= 2,
|
|
f"{len(ready_api)} ready API pod(s) on {len({item['node'] for item in ready_api})} node(s)",
|
|
),
|
|
_check(
|
|
"web_node_spread",
|
|
len(ready_web) >= 2 and len({item["node"] for item in ready_web}) >= 2,
|
|
f"{len(ready_web)} ready WebUI pod(s) on {len({item['node'] for item in ready_web})} node(s)",
|
|
),
|
|
_check(
|
|
"deployment_availability",
|
|
bool(deployment_rows)
|
|
and all(item["available"] >= item["desired"] for item in deployment_rows),
|
|
f"{len(deployment_rows)} deployment(s) inspected",
|
|
),
|
|
_check(
|
|
"ops_readiness",
|
|
bool(ops_readiness.get("ready")),
|
|
str(ops_readiness.get("detail") or "Ops readiness response inspected"),
|
|
),
|
|
_check(
|
|
"runtime_composition",
|
|
not bool(runtime.get("composition", {}).get("skewed")),
|
|
"Active runtime composition matches the loaded module graph",
|
|
),
|
|
_check(
|
|
"runtime_versions",
|
|
not bool(runtime.get("software_versions", {}).get("skewed")),
|
|
"Active runtime software versions are consistent",
|
|
),
|
|
_check(
|
|
"worker_queue_coverage",
|
|
not bool(runtime.get("queues", {}).get("missing")),
|
|
"Every configured queue has an active worker",
|
|
),
|
|
_check(
|
|
"database_connection_budget",
|
|
str(check_by_id.get("database_capacity", {}).get("state")) == "ok",
|
|
str(
|
|
check_by_id.get("database_capacity", {}).get("detail")
|
|
or "Database capacity check is unavailable"
|
|
),
|
|
),
|
|
]
|
|
return {
|
|
"installation_id": installation_id,
|
|
"namespace": namespace,
|
|
"ready_node_names": sorted(_name(item) for item in ready_nodes),
|
|
"pods": sorted(pod_rows, key=lambda item: item["name"]),
|
|
"deployments": sorted(deployment_rows, key=lambda item: item["name"]),
|
|
"runtime": {
|
|
"expected": dict(runtime.get("expected") or {}),
|
|
"active": dict(runtime.get("active") or {}),
|
|
"composition": dict(runtime.get("composition") or {}),
|
|
"software_versions": dict(runtime.get("software_versions") or {}),
|
|
"queues": dict(runtime.get("queues") or {}),
|
|
},
|
|
"database_capacity": dict(
|
|
check_by_id.get("database_capacity", {}).get("metrics") or {}
|
|
),
|
|
"checks": snapshot_checks,
|
|
}
|
|
|
|
|
|
def _exercise_api_pod_loss(
|
|
*,
|
|
installation_id: str,
|
|
namespace: str,
|
|
ops_url: str,
|
|
api_key: str,
|
|
initial_pods: Mapping[str, Any],
|
|
timeout_seconds: float,
|
|
run_json: CommandRunner,
|
|
fetch_json: JsonFetcher,
|
|
) -> JsonObject:
|
|
candidates = [
|
|
_pod_summary(item)
|
|
for item in _items(initial_pods)
|
|
if item.get("metadata", {}).get("labels", {}).get("app.kubernetes.io/component")
|
|
== "api"
|
|
and _condition(item, "Ready") == "True"
|
|
]
|
|
if len(candidates) < 2:
|
|
return {
|
|
"state": "failed",
|
|
"detail": "At least two ready API pods are required for the loss drill.",
|
|
}
|
|
victim = sorted(candidates, key=lambda item: item["name"])[0]
|
|
initial_uids = {item["uid"] for item in candidates}
|
|
desired_ready = len(candidates)
|
|
run_json(
|
|
(
|
|
"-n",
|
|
namespace,
|
|
"delete",
|
|
"pod",
|
|
victim["name"],
|
|
"--wait=false",
|
|
"-o",
|
|
"json",
|
|
)
|
|
)
|
|
deadline = time.monotonic() + timeout_seconds
|
|
health_failures = 0
|
|
replacement: JsonObject | None = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
current_ops = fetch_json(ops_url, api_key)
|
|
if not bool(current_ops.get("readiness", {}).get("ready")):
|
|
health_failures += 1
|
|
except (OSError, ValueError):
|
|
health_failures += 1
|
|
current = run_json(
|
|
(
|
|
"-n",
|
|
namespace,
|
|
"get",
|
|
"pods",
|
|
"-l",
|
|
f"app.kubernetes.io/instance={installation_id},app.kubernetes.io/component=api",
|
|
"-o",
|
|
"json",
|
|
)
|
|
)
|
|
ready = [
|
|
_pod_summary(item)
|
|
for item in _items(current)
|
|
if _condition(item, "Ready") == "True"
|
|
]
|
|
new_pods = [item for item in ready if item["uid"] not in initial_uids]
|
|
if len(ready) >= desired_ready and new_pods:
|
|
replacement = sorted(new_pods, key=lambda item: item["name"])[-1]
|
|
break
|
|
time.sleep(2.0)
|
|
passed = replacement is not None and health_failures == 0
|
|
return {
|
|
"state": "passed" if passed else "failed",
|
|
"victim": victim,
|
|
"replacement": replacement,
|
|
"readiness_failures": health_failures,
|
|
"detail": (
|
|
"API pod was replaced without an observed readiness outage."
|
|
if passed
|
|
else "API replacement timed out or readiness failed during replacement."
|
|
),
|
|
}
|
|
|
|
|
|
def _kubectl_json(arguments: Sequence[str]) -> JsonObject:
|
|
kubectl = shutil.which("kubectl")
|
|
if kubectl is None:
|
|
raise ValueError("kubectl is required for Kubernetes evidence collection")
|
|
result = subprocess.run(
|
|
(kubectl, *arguments),
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
)
|
|
if result.returncode:
|
|
detail = result.stderr.strip() or result.stdout.strip()
|
|
raise ValueError(f"kubectl failed: {detail}")
|
|
try:
|
|
payload = json.loads(result.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError("kubectl did not return JSON") from exc
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("kubectl returned a non-object JSON payload")
|
|
return payload
|
|
|
|
|
|
def _fetch_json(url: str, api_key: str) -> JsonObject:
|
|
request = Request(
|
|
url,
|
|
headers={"Accept": "application/json", "X-API-Key": api_key},
|
|
)
|
|
with urlopen(request, timeout=15) as response:
|
|
payload = json.load(response)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("Ops API returned a non-object JSON payload")
|
|
return payload
|
|
|
|
|
|
def _items(payload: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
|
items = payload.get("items")
|
|
if not isinstance(items, list):
|
|
raise ValueError("Kubernetes list response has no items")
|
|
return [item for item in items if isinstance(item, Mapping)]
|
|
|
|
|
|
def _condition(item: Mapping[str, Any], condition_type: str) -> str | None:
|
|
conditions = item.get("status", {}).get("conditions", [])
|
|
for condition in conditions if isinstance(conditions, list) else []:
|
|
if isinstance(condition, Mapping) and condition.get("type") == condition_type:
|
|
return str(condition.get("status"))
|
|
return None
|
|
|
|
|
|
def _name(item: Mapping[str, Any]) -> str:
|
|
return str(item.get("metadata", {}).get("name") or "")
|
|
|
|
|
|
def _pod_summary(item: Mapping[str, Any]) -> JsonObject:
|
|
metadata = item.get("metadata", {})
|
|
labels = metadata.get("labels", {})
|
|
return {
|
|
"name": _name(item),
|
|
"uid": str(metadata.get("uid") or ""),
|
|
"component": str(labels.get("app.kubernetes.io/component") or ""),
|
|
"worker_pool": labels.get("govoplan.add-ideas.de/worker-pool"),
|
|
"node": str(item.get("spec", {}).get("nodeName") or ""),
|
|
"ready": _condition(item, "Ready") == "True",
|
|
}
|
|
|
|
|
|
def _deployment_summary(item: Mapping[str, Any]) -> JsonObject:
|
|
status = item.get("status", {})
|
|
return {
|
|
"name": _name(item),
|
|
"component": str(
|
|
item.get("metadata", {})
|
|
.get("labels", {})
|
|
.get("app.kubernetes.io/component")
|
|
or ""
|
|
),
|
|
"desired": int(item.get("spec", {}).get("replicas") or 0),
|
|
"available": int(status.get("availableReplicas") or 0),
|
|
"updated": int(status.get("updatedReplicas") or 0),
|
|
}
|
|
|
|
|
|
def _check(check_id: str, passed: bool, detail: str) -> JsonObject:
|
|
return {
|
|
"id": check_id,
|
|
"state": "passed" if passed else "failed",
|
|
"detail": detail,
|
|
}
|
|
|
|
|
|
__all__ = ["collect_kubernetes_evidence"]
|