feat: add bounded Kubernetes runtime verification

2026-08-02 05:30:26 +02:00
parent 50e9607e72
commit ed31409034
6 changed files with 1094 additions and 11 deletions
@@ -2,7 +2,9 @@

 from __future__ import annotations

+from dataclasses import dataclass
 from hashlib import sha256
+import json
 from pathlib import Path
 import re
 from typing import Any, Mapping
@@ -26,6 +28,17 @@ _CONFIG_KEYS = (
    "ENABLED_MODULES",
    "CELERY_ENABLED",
    "CELERY_QUEUES",
+    "GOVOPLAN_DB_CONNECTION_LIMIT",
+    "GOVOPLAN_DB_CONNECTION_RESERVE",
+    "GOVOPLAN_API_DB_POOL_SIZE",
+    "GOVOPLAN_API_DB_MAX_OVERFLOW",
+    "GOVOPLAN_WORKER_DB_POOL_SIZE",
+    "GOVOPLAN_WORKER_DB_MAX_OVERFLOW",
+    "GOVOPLAN_SCHEDULER_DB_POOL_SIZE",
+    "GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW",
+    "GOVOPLAN_MIGRATION_DB_POOL_SIZE",
+    "GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW",
+    "GOVOPLAN_WORKER_POOLS",
    "CORS_ORIGINS",
    "GOVOPLAN_TRUSTED_HOSTS",
    "FORWARDED_ALLOW_IPS",
@@ -43,6 +56,15 @@ _CONFIG_KEYS = (
    "FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
    "FILE_STORAGE_S3_ENDPOINT_TRUSTED",
 )
+_QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
+
+
+@dataclass(frozen=True, slots=True)
+class _WorkerPool:
+    name: str
+    queues: tuple[str, ...]
+    replicas: int
+    concurrency: int


 def render_kubernetes(
@@ -57,6 +79,8 @@ def render_kubernetes(
    """Render runtime roles only; shared state services stay externally managed."""

    _validate_cluster_profile(spec, environment, namespace, secret_name)
+    worker_pools = _worker_pools(spec, environment)
+    database_capacity = _database_capacity(spec, environment, worker_pools)
    name = _resource_name(spec.installation_id)
    public_host = urlsplit(spec.public_url).hostname or "localhost"
    labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
@@ -75,6 +99,8 @@ def render_kubernetes(
            "GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
            "GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
            "GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
+            "GOVOPLAN_DB_CONNECTION_PEAK": str(database_capacity["peak"]),
+            "GOVOPLAN_DB_CONNECTION_AVAILABLE": str(database_capacity["available"]),
            "FILE_STORAGE_BACKEND": "s3",
            "FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
            "FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
@@ -130,6 +156,7 @@ def render_kubernetes(
            readiness_path="/health/ready",
            liveness_path="/health",
            probe_host=public_host,
+            extra_environment=_role_database_environment(environment, "API"),
        ),
        _service(
            name=f"{name}-api",
@@ -168,16 +195,25 @@ def render_kubernetes(
            config_name=config_name,
            secret_name=secret_name,
            service_account=service_account,
+            database_environment=_role_database_environment(
+                environment,
+                "MIGRATION",
+            ),
        ),
    ]
-    if spec.replicas.worker:
+    for pool in worker_pools:
+        worker_name = (
+            f"{name}-worker"
+            if len(worker_pools) == 1 and pool.name == "default"
+            else _name_with_suffix(name, f"worker-{pool.name}")
+        )
        items.append(
            _deployment(
-                name=f"{name}-worker",
+                name=worker_name,
                namespace=namespace,
                labels=labels,
                role="worker",
-                replicas=spec.replicas.worker,
+                replicas=pool.replicas,
                image=spec.release.api_image,
                command=(
                    "python",
@@ -187,15 +223,39 @@ def render_kubernetes(
                    "govoplan_core.celery_app:celery",
                    "worker",
                    "--queues",
-                    str(environment["CELERY_QUEUES"]),
+                    ",".join(pool.queues),
+                    "--concurrency",
+                    str(pool.concurrency),
                    "--loglevel",
                    "INFO",
                ),
                config_name=config_name,
                secret_name=secret_name,
                service_account=service_account,
+                extra_environment={
+                    **_role_database_environment(environment, "WORKER"),
+                    "CELERY_QUEUES": ",".join(pool.queues),
+                    "CELERY_WORKER_CONCURRENCY": str(pool.concurrency),
+                    "GOVOPLAN_WORKER_POOL": pool.name,
+                },
+                selector_labels={
+                    "govoplan.add-ideas.de/worker-pool": pool.name,
+                },
            )
        )
+        if pool.replicas > 1:
+            items.append(
+                _pod_disruption_budget(
+                    worker_name,
+                    namespace,
+                    labels,
+                    "worker",
+                    selector_labels={
+                        "govoplan.add-ideas.de/worker-pool": pool.name,
+                    },
+                )
+            )
+    if worker_pools:
        items.append(
            _deployment(
                name=f"{name}-scheduler",
@@ -225,6 +285,10 @@ def render_kubernetes(
                config_name=config_name,
                secret_name=secret_name,
                service_account=service_account,
+                extra_environment=_role_database_environment(
+                    environment,
+                    "SCHEDULER",
+                ),
            )
        )
    if spec.replicas.api > 1:
@@ -248,6 +312,15 @@ def render_kubernetes(
            "annotations": {
                "govoplan.add-ideas.de/profile": "stateless-shared-state",
                "govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
+                "govoplan.add-ideas.de/database-connection-peak": str(
+                    database_capacity["peak"]
+                ),
+                "govoplan.add-ideas.de/database-connection-available": str(
+                    database_capacity["available"]
+                ),
+                "govoplan.add-ideas.de/worker-pools": ",".join(
+                    pool.name for pool in worker_pools
+                ),
            }
        },
        "items": items,
@@ -290,7 +363,13 @@ def _validate_cluster_profile(
            + ", ".join(unpinned_images)
        )
    missing = [
-        key for key in (*_SECRET_KEYS, "CELERY_QUEUES") if not environment.get(key)
+        key
+        for key in (
+            *_SECRET_KEYS,
+            "CELERY_QUEUES",
+            "GOVOPLAN_DB_CONNECTION_LIMIT",
+        )
+        if not environment.get(key)
    ]
    if missing:
        raise ValueError(
@@ -298,6 +377,245 @@ def _validate_cluster_profile(
        )


+def _worker_pools(
+    spec: InstallationSpec,
+    environment: Mapping[str, str],
+) -> tuple[_WorkerPool, ...]:
+    if spec.replicas.worker == 0:
+        if str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip():
+            raise ValueError("Worker pools require at least one worker replica")
+        return ()
+    configured_queues = tuple(
+        item.strip()
+        for item in str(environment.get("CELERY_QUEUES") or "").split(",")
+        if item.strip()
+    )
+    if not configured_queues or any(
+        _QUEUE_NAME.fullmatch(item) is None for item in configured_queues
+    ):
+        raise ValueError("CELERY_QUEUES must contain canonical queue names")
+    raw_pools = str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip()
+    if not raw_pools:
+        return (
+            _WorkerPool(
+                name="default",
+                queues=configured_queues,
+                replicas=spec.replicas.worker,
+                concurrency=_bounded_environment_integer(
+                    environment,
+                    "CELERY_WORKER_CONCURRENCY",
+                    default=2,
+                    minimum=1,
+                    maximum=64,
+                ),
+            ),
+        )
+    try:
+        payload = json.loads(raw_pools)
+    except json.JSONDecodeError as exc:
+        raise ValueError("GOVOPLAN_WORKER_POOLS must be valid JSON") from exc
+    if not isinstance(payload, list) or not payload:
+        raise ValueError("GOVOPLAN_WORKER_POOLS must be a non-empty JSON list")
+    pools: list[_WorkerPool] = []
+    seen_names: set[str] = set()
+    seen_queues: set[str] = set()
+    for index, item in enumerate(payload):
+        if not isinstance(item, dict):
+            raise ValueError(f"Worker pool {index} must be an object")
+        unknown = set(item) - {"name", "queues", "replicas", "concurrency"}
+        if unknown:
+            raise ValueError(
+                f"Worker pool {index} has unsupported keys: "
+                + ", ".join(sorted(unknown))
+            )
+        pool_name = item.get("name")
+        queues = item.get("queues")
+        replicas = item.get("replicas")
+        concurrency = item.get("concurrency")
+        if not isinstance(pool_name, str) or _DNS_LABEL.fullmatch(pool_name) is None:
+            raise ValueError(f"Worker pool {index} has an invalid name")
+        if pool_name in seen_names:
+            raise ValueError(f"Worker pool name is duplicated: {pool_name}")
+        if (
+            not isinstance(queues, list)
+            or not queues
+            or any(
+                not isinstance(queue, str) or _QUEUE_NAME.fullmatch(queue) is None
+                for queue in queues
+            )
+        ):
+            raise ValueError(f"Worker pool {pool_name} has invalid queues")
+        duplicate_queues = seen_queues.intersection(queues)
+        if duplicate_queues:
+            raise ValueError(
+                "Worker queues must have one owning pool: "
+                + ", ".join(sorted(duplicate_queues))
+            )
+        if not isinstance(replicas, int) or not 1 <= replicas <= 128:
+            raise ValueError(
+                f"Worker pool {pool_name} replicas must be between 1 and 128"
+            )
+        if not isinstance(concurrency, int) or not 1 <= concurrency <= 64:
+            raise ValueError(
+                f"Worker pool {pool_name} concurrency must be between 1 and 64"
+            )
+        pools.append(
+            _WorkerPool(
+                name=pool_name,
+                queues=tuple(queues),
+                replicas=replicas,
+                concurrency=concurrency,
+            )
+        )
+        seen_names.add(pool_name)
+        seen_queues.update(queues)
+    if sum(pool.replicas for pool in pools) != spec.replicas.worker:
+        raise ValueError("Worker pool replicas must equal installation replicas.worker")
+    configured_queue_set = set(configured_queues)
+    if seen_queues != configured_queue_set:
+        missing = configured_queue_set - seen_queues
+        unknown = seen_queues - configured_queue_set
+        detail = []
+        if missing:
+            detail.append("missing " + ", ".join(sorted(missing)))
+        if unknown:
+            detail.append("unknown " + ", ".join(sorted(unknown)))
+        raise ValueError(
+            "Worker pools must cover CELERY_QUEUES exactly: " + "; ".join(detail)
+        )
+    return tuple(sorted(pools, key=lambda pool: pool.name))
+
+
+def _database_capacity(
+    spec: InstallationSpec,
+    environment: Mapping[str, str],
+    worker_pools: tuple[_WorkerPool, ...],
+) -> dict[str, int]:
+    limit = _bounded_environment_integer(
+        environment,
+        "GOVOPLAN_DB_CONNECTION_LIMIT",
+        minimum=10,
+        maximum=1_000_000,
+    )
+    reserve = _bounded_environment_integer(
+        environment,
+        "GOVOPLAN_DB_CONNECTION_RESERVE",
+        default=10,
+        minimum=1,
+        maximum=999_999,
+    )
+    if reserve >= limit:
+        raise ValueError("Database connection reserve must be below the limit")
+    api_ceiling = _role_database_ceiling(environment, "API", 5, 2)
+    worker_ceiling = _role_database_ceiling(environment, "WORKER", 1, 1)
+    scheduler_ceiling = _role_database_ceiling(environment, "SCHEDULER", 1, 0)
+    migration_ceiling = _role_database_ceiling(environment, "MIGRATION", 2, 0)
+    worker_processes = sum(
+        pool.replicas * (pool.concurrency + 1) for pool in worker_pools
+    )
+    steady = (
+        spec.replicas.api * api_ceiling
+        + worker_processes * worker_ceiling
+        + (scheduler_ceiling if worker_pools else 0)
+    )
+    rollout_surge = api_ceiling + migration_ceiling
+    if worker_pools:
+        rollout_surge += scheduler_ceiling
+        rollout_surge += sum(
+            (pool.concurrency + 1) * worker_ceiling for pool in worker_pools
+        )
+    peak = steady + rollout_surge
+    available = limit - reserve
+    if peak > available:
+        raise ValueError(
+            "Kubernetes database connection peak exceeds the configured budget: "
+            f"peak={peak}, available={available}, limit={limit}, reserve={reserve}"
+        )
+    return {
+        "limit": limit,
+        "reserve": reserve,
+        "available": available,
+        "steady": steady,
+        "peak": peak,
+    }
+
+
+def _role_database_ceiling(
+    environment: Mapping[str, str],
+    role: str,
+    default_pool_size: int,
+    default_overflow: int,
+) -> int:
+    return _bounded_environment_integer(
+        environment,
+        f"GOVOPLAN_{role}_DB_POOL_SIZE",
+        default=default_pool_size,
+        minimum=1,
+        maximum=100,
+    ) + _bounded_environment_integer(
+        environment,
+        f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
+        default=default_overflow,
+        minimum=0,
+        maximum=200,
+    )
+
+
+def _role_database_environment(
+    environment: Mapping[str, str],
+    role: str,
+) -> dict[str, str]:
+    defaults = {
+        "API": (5, 2),
+        "WORKER": (1, 1),
+        "SCHEDULER": (1, 0),
+        "MIGRATION": (2, 0),
+    }
+    pool_size, overflow = defaults[role]
+    return {
+        "GOVOPLAN_DB_POOL_SIZE": str(
+            _bounded_environment_integer(
+                environment,
+                f"GOVOPLAN_{role}_DB_POOL_SIZE",
+                default=pool_size,
+                minimum=1,
+                maximum=100,
+            )
+        ),
+        "GOVOPLAN_DB_MAX_OVERFLOW": str(
+            _bounded_environment_integer(
+                environment,
+                f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
+                default=overflow,
+                minimum=0,
+                maximum=200,
+            )
+        ),
+    }
+
+
+def _bounded_environment_integer(
+    environment: Mapping[str, str],
+    name: str,
+    *,
+    default: int | None = None,
+    minimum: int,
+    maximum: int,
+) -> int:
+    raw = environment.get(name)
+    if raw is None or not str(raw).strip():
+        if default is None:
+            raise ValueError(f"{name} is required")
+        return default
+    try:
+        value = int(str(raw))
+    except ValueError as exc:
+        raise ValueError(f"{name} must be an integer") from exc
+    if not minimum <= value <= maximum:
+        raise ValueError(f"{name} must be between {minimum} and {maximum}")
+    return value
+
+
 def _deployment(
    *,
    name: str,
@@ -315,8 +633,13 @@ def _deployment(
    liveness_path: str | None = None,
    probe_host: str | None = None,
    extra_environment: Mapping[str, str] | None = None,
+    selector_labels: Mapping[str, str] | None = None,
 ) -> dict[str, Any]:
-    role_labels = {**labels, "app.kubernetes.io/component": role}
+    role_labels = {
+        **labels,
+        "app.kubernetes.io/component": role,
+        **dict(selector_labels or {}),
+    }
    environment: list[dict[str, Any]] = [
        {"name": "TMPDIR", "value": "/tmp"},
        {"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
@@ -400,6 +723,8 @@ def _deployment(
                "env": [
                    {"name": "TMPDIR", "value": "/tmp"},
                    {"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
+                    {"name": "GOVOPLAN_DB_POOL_SIZE", "value": "1"},
+                    {"name": "GOVOPLAN_DB_MAX_OVERFLOW", "value": "0"},
                    *_secret_environment(secret_name),
                ],
                "securityContext": {
@@ -439,6 +764,7 @@ def _migration_job(
    config_name: str,
    secret_name: str,
    service_account: str,
+    database_environment: Mapping[str, str],
 ) -> dict[str, Any]:
    job_labels = {**labels, "app.kubernetes.io/component": "migration"}
    job_name = _name_with_suffix(name, f"migrate-{release_key}")
@@ -481,6 +807,12 @@ def _migration_job(
                            "env": [
                                {"name": "TMPDIR", "value": "/tmp"},
                                {"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
+                                *(
+                                    {"name": key, "value": value}
+                                    for key, value in sorted(
+                                        database_environment.items()
+                                    )
+                                ),
                                {
                                    "name": "GOVOPLAN_NODE_ID",
                                    "valueFrom": {
@@ -541,8 +873,14 @@ def _pod_disruption_budget(
    namespace: str,
    labels: dict[str, str],
    role: str,
+    *,
+    selector_labels: Mapping[str, str] | None = None,
 ) -> dict[str, Any]:
-    role_labels = {**labels, "app.kubernetes.io/component": role}
+    role_labels = {
+        **labels,
+        "app.kubernetes.io/component": role,
+        **dict(selector_labels or {}),
+    }
    return {
        "apiVersion": "policy/v1",
        "kind": "PodDisruptionBudget",