feat: add bounded Kubernetes runtime verification

tests/test_deployment_installer.py

@@ -31,6 +31,9 @@ from govoplan_deploy.bundle import (  # noqa: E402
 )
 from govoplan_deploy.cli import _receipt_uses_direct_web_port, main  # noqa: E402
 import govoplan_deploy.cli as deployment_cli  # noqa: E402
+from govoplan_deploy.cluster_evidence import (  # noqa: E402
+    collect_kubernetes_evidence,
+)
 from govoplan_deploy.kubernetes import render_kubernetes  # noqa: E402
 from govoplan_deploy.model import (  # noqa: E402
     SpecError,
@@ -55,7 +58,104 @@ def run_cli(arguments: list[str]) -> tuple[int, str, str]:
     return result, stdout.getvalue(), stderr.getvalue()
 
 
+def _kubernetes_test_pod(name: str, component: str, node: str) -> dict:
+    return {
+        "metadata": {
+            "name": name,
+            "uid": f"uid-{name}",
+            "labels": {"app.kubernetes.io/component": component},
+        },
+        "spec": {"nodeName": node},
+        "status": {"conditions": [{"type": "Ready", "status": "True"}]},
+    }
+
+
+def _kubernetes_test_deployment(component: str, replicas: int) -> dict:
+    return {
+        "metadata": {
+            "name": f"govoplan-cluster-{component}",
+            "labels": {"app.kubernetes.io/component": component},
+        },
+        "spec": {"replicas": replicas},
+        "status": {"availableReplicas": replicas, "updatedReplicas": replicas},
+    }
+
+
 class DeploymentInstallerTests(unittest.TestCase):
+    def test_kubernetes_evidence_requires_two_node_spread_and_safe_runtime(
+        self,
+    ) -> None:
+        resources = {
+            "nodes": {
+                "items": [
+                    {
+                        "metadata": {"name": name},
+                        "spec": {},
+                        "status": {"conditions": [{"type": "Ready", "status": "True"}]},
+                    }
+                    for name in ("node-a", "node-b")
+                ]
+            },
+            "pods": {
+                "items": [
+                    _kubernetes_test_pod("api-a", "api", "node-a"),
+                    _kubernetes_test_pod("api-b", "api", "node-b"),
+                    _kubernetes_test_pod("web-a", "web", "node-a"),
+                    _kubernetes_test_pod("web-b", "web", "node-b"),
+                    _kubernetes_test_pod("worker-a", "worker", "node-a"),
+                ]
+            },
+            "deployments": {
+                "items": [
+                    _kubernetes_test_deployment("api", 2),
+                    _kubernetes_test_deployment("web", 2),
+                    _kubernetes_test_deployment("worker", 1),
+                ]
+            },
+        }
+
+        def run(arguments):
+            return resources[
+                "nodes"
+                if "nodes" in arguments
+                else "deployments"
+                if "deployments" in arguments
+                else "pods"
+            ]
+
+        evidence = collect_kubernetes_evidence(
+            installation_id="govoplan-cluster",
+            namespace="govoplan",
+            ops_url="https://govoplan.example.test/api/v1/ops/status",
+            api_key="not-retained",
+            command_runner=run,
+            json_fetcher=lambda _url, _key: {
+                "readiness": {"ready": True},
+                "runtime_cluster": {
+                    "expected": {"api": 2, "worker": 1},
+                    "active": {"api": 2, "worker": 1},
+                    "composition": {"skewed": False},
+                    "software_versions": {"skewed": False},
+                    "queues": {"missing": []},
+                },
+                "checks": [
+                    {
+                        "id": "database_capacity",
+                        "state": "ok",
+                        "detail": "Within budget",
+                        "metrics": {"peak": 30, "available": 90},
+                    }
+                ],
+            },
+        )
+
+        self.assertEqual("passed", evidence["result"]["state"])
+        self.assertNotIn("not-retained", json.dumps(evidence))
+        self.assertEqual(
+            ["node-a", "node-b"],
+            evidence["snapshot"]["ready_node_names"],
+        )
+
     def test_pre_migration_failure_restores_checksum_verified_applied_bundle(
         self,
     ) -> None:
@@ -212,6 +312,7 @@ class DeploymentInstallerTests(unittest.TestCase):
                 "FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
                 "FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
                 "FILE_STORAGE_S3_BUCKET": "govoplan",
+                "GOVOPLAN_DB_CONNECTION_LIMIT": "100",
             },
         )
 
@@ -257,6 +358,103 @@ class DeploymentInstallerTests(unittest.TestCase):
                 "containers"
             ][0]["readinessProbe"]["httpGet"]["httpHeaders"],
         )
+        worker_command = deployments["govoplan-cluster-worker"]["spec"]["template"][
+            "spec"
+        ]["containers"][0]["command"]
+        self.assertIn("--concurrency", worker_command)
+        self.assertEqual(
+            "62",
+            manifest["metadata"]["annotations"][
+                "govoplan.add-ideas.de/database-connection-peak"
+            ],
+        )
+
+    def test_kubernetes_export_splits_worker_queues_and_rejects_capacity_overrun(
+        self,
+    ) -> None:
+        spec = default_spec(
+            installation_id="govoplan-cluster",
+            postgres_mode="external",
+            redis_mode="external",
+            storage_mode="s3",
+            api_replicas=2,
+            web_replicas=2,
+            worker_replicas=3,
+            api_image="registry.example.test/govoplan-api@sha256:" + "a" * 64,
+            web_image="registry.example.test/govoplan-web@sha256:" + "b" * 64,
+        )
+        environment = initial_secrets(
+            spec,
+            supplied={
+                "DATABASE_URL": "postgresql+psycopg://user:db-secret@postgres.example.test/govoplan",
+                "GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:db-secret@postgres.example.test/govoplan",
+                "REDIS_URL": "rediss://:redis-secret@redis.example.test/0",
+                "FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
+                "FILE_STORAGE_S3_REGION": "eu-test-1",
+                "FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
+                "FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
+                "FILE_STORAGE_S3_BUCKET": "govoplan",
+                "GOVOPLAN_DB_CONNECTION_LIMIT": "100",
+            },
+        )
+        queues = environment["CELERY_QUEUES"].split(",")
+        environment["GOVOPLAN_WORKER_POOLS"] = json.dumps(
+            [
+                {
+                    "name": "delivery",
+                    "queues": queues[:3],
+                    "replicas": 2,
+                    "concurrency": 2,
+                },
+                {
+                    "name": "platform",
+                    "queues": queues[3:],
+                    "replicas": 1,
+                    "concurrency": 1,
+                },
+            ]
+        )
+
+        manifest = render_kubernetes(spec, environment)
+        workers = [
+            item
+            for item in manifest["items"]
+            if item["kind"] == "Deployment"
+            and item["metadata"]["labels"].get("app.kubernetes.io/component")
+            == "worker"
+        ]
+
+        self.assertEqual(2, len(workers))
+        self.assertEqual(
+            {"delivery", "platform"},
+            {
+                item["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"]
+                for item in workers
+            },
+        )
+        for deployment in workers:
+            container = deployment["spec"]["template"]["spec"]["containers"][0]
+            explicit_environment = {
+                item["name"]: item.get("value")
+                for item in container["env"]
+                if "value" in item
+            }
+            self.assertEqual(
+                deployment["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"],
+                explicit_environment["GOVOPLAN_WORKER_POOL"],
+            )
+            self.assertEqual(
+                set(
+                    container["command"][
+                        container["command"].index("--queues") + 1
+                    ].split(",")
+                ),
+                set(explicit_environment["CELERY_QUEUES"].split(",")),
+            )
+
+        environment["GOVOPLAN_DB_CONNECTION_LIMIT"] = "40"
+        with self.assertRaisesRegex(ValueError, "connection peak"):
+            render_kubernetes(spec, environment)
 
     def test_kubernetes_export_rejects_mutable_release_images(self) -> None:
         spec = default_spec(