1133 lines
37 KiB
Python
1133 lines
37 KiB
Python
"""Deterministic Kubernetes export for the stateless multi-host profile."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from typing import Any, Mapping
|
|
from urllib.parse import urlsplit
|
|
|
|
from .bundle import BACKUP_RUNTIME_ENV_KEYS
|
|
from .model import InstallationSpec, image_is_digest_pinned
|
|
|
|
|
|
_DNS_LABEL = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$")
|
|
_SECRET_KEYS = (
|
|
"MASTER_KEY_B64",
|
|
"DATABASE_URL",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS",
|
|
"REDIS_URL",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
|
)
|
|
_CONFIG_KEYS = (
|
|
"APP_ENV",
|
|
"GOVOPLAN_INSTALL_PROFILE",
|
|
"ENABLED_MODULES",
|
|
"CELERY_ENABLED",
|
|
"CELERY_QUEUES",
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
|
"GOVOPLAN_API_DB_POOL_SIZE",
|
|
"GOVOPLAN_API_DB_MAX_OVERFLOW",
|
|
"GOVOPLAN_WORKER_DB_POOL_SIZE",
|
|
"GOVOPLAN_WORKER_DB_MAX_OVERFLOW",
|
|
"GOVOPLAN_SCHEDULER_DB_POOL_SIZE",
|
|
"GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW",
|
|
"GOVOPLAN_MIGRATION_DB_POOL_SIZE",
|
|
"GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW",
|
|
"GOVOPLAN_WORKER_POOLS",
|
|
"CORS_ORIGINS",
|
|
"GOVOPLAN_TRUSTED_HOSTS",
|
|
"FORWARDED_ALLOW_IPS",
|
|
"AUTH_COOKIE_SECURE",
|
|
"AUTH_COOKIE_SAMESITE",
|
|
"GOVOPLAN_HTTP_HSTS_SECONDS",
|
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
|
|
"GOVOPLAN_MIGRATION_TRACK",
|
|
"DEV_AUTO_MIGRATE_ENABLED",
|
|
"DEV_BOOTSTRAP_ENABLED",
|
|
"FILE_STORAGE_BACKEND",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
|
"FILE_STORAGE_S3_REGION",
|
|
"FILE_STORAGE_S3_BUCKET",
|
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
|
*BACKUP_RUNTIME_ENV_KEYS,
|
|
)
|
|
_QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _WorkerPool:
|
|
name: str
|
|
queues: tuple[str, ...]
|
|
replicas: int
|
|
concurrency: int
|
|
|
|
|
|
def render_kubernetes(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
*,
|
|
namespace: str = "govoplan",
|
|
secret_name: str = "govoplan-runtime",
|
|
tls_secret_name: str = "govoplan-tls",
|
|
s3_ca_secret_name: str | None = None,
|
|
ingress_class_name: str | None = None,
|
|
backup_required: bool = True,
|
|
backup_evidence: Mapping[str, object] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Render runtime roles only; shared state services stay externally managed."""
|
|
|
|
_validate_cluster_profile(
|
|
spec,
|
|
environment,
|
|
namespace,
|
|
secret_name,
|
|
s3_ca_secret_name=s3_ca_secret_name,
|
|
)
|
|
worker_pools = _worker_pools(spec, environment)
|
|
database_capacity = _database_capacity(spec, environment, worker_pools)
|
|
name = _resource_name(spec.installation_id)
|
|
public_host = urlsplit(spec.public_url).hostname or "localhost"
|
|
labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
|
|
config_name = f"{name}-runtime"
|
|
service_account = f"{name}-runtime"
|
|
config = {
|
|
key: str(environment[key])
|
|
for key in _CONFIG_KEYS
|
|
if environment.get(key) is not None
|
|
}
|
|
config.update(
|
|
{
|
|
"GOVOPLAN_INSTALLATION_ID": spec.installation_id,
|
|
"GOVOPLAN_STATE_PROFILE": "shared",
|
|
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "15",
|
|
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
|
|
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
|
|
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
|
|
"GOVOPLAN_DB_CONNECTION_PEAK": str(database_capacity["peak"]),
|
|
"GOVOPLAN_DB_CONNECTION_AVAILABLE": str(database_capacity["available"]),
|
|
"FILE_STORAGE_BACKEND": "s3",
|
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
|
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
|
|
# Trust no cross-pod proxy headers by default. Operators can bind
|
|
# an exact ingress-controller CIDR through their ConfigMap overlay.
|
|
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
|
}
|
|
)
|
|
items: list[dict[str, Any]] = [
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "Namespace",
|
|
"metadata": {"name": namespace, "labels": labels},
|
|
},
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "ServiceAccount",
|
|
"metadata": {
|
|
"name": service_account,
|
|
"namespace": namespace,
|
|
"labels": labels,
|
|
},
|
|
"automountServiceAccountToken": False,
|
|
},
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {"name": config_name, "namespace": namespace, "labels": labels},
|
|
"data": dict(sorted(config.items())),
|
|
},
|
|
_deployment(
|
|
name=f"{name}-api",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="api",
|
|
replicas=spec.replicas.api,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"uvicorn",
|
|
"govoplan_core.server.app:app",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
"8000",
|
|
"--proxy-headers",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
s3_ca_secret_name=s3_ca_secret_name,
|
|
service_account=service_account,
|
|
container_port=8000,
|
|
readiness_path="/health/ready",
|
|
liveness_path="/health",
|
|
probe_host=public_host,
|
|
graceful_shutdown_seconds=10,
|
|
extra_environment=_role_database_environment(environment, "API"),
|
|
),
|
|
_service(
|
|
name=f"{name}-api",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="api",
|
|
port=8000,
|
|
),
|
|
_deployment(
|
|
name=f"{name}-web",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="web",
|
|
replicas=spec.replicas.web,
|
|
image=spec.release.web_image,
|
|
command=(),
|
|
config_name=None,
|
|
secret_name=None,
|
|
s3_ca_secret_name=None,
|
|
service_account=service_account,
|
|
container_port=8080,
|
|
extra_environment={"GOVOPLAN_API_UPSTREAM": f"http://{name}-api:8000"},
|
|
),
|
|
_service(
|
|
name=f"{name}-web",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="web",
|
|
port=8080,
|
|
),
|
|
_migration_job(
|
|
name=name,
|
|
release_key=_release_key(spec),
|
|
namespace=namespace,
|
|
labels=labels,
|
|
image=spec.release.api_image,
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
s3_ca_secret_name=s3_ca_secret_name,
|
|
service_account=service_account,
|
|
database_environment=_role_database_environment(
|
|
environment,
|
|
"MIGRATION",
|
|
),
|
|
backup_required=backup_required,
|
|
backup_evidence=backup_evidence,
|
|
),
|
|
]
|
|
for pool in worker_pools:
|
|
worker_name = (
|
|
f"{name}-worker"
|
|
if len(worker_pools) == 1 and pool.name == "default"
|
|
else _name_with_suffix(name, f"worker-{pool.name}")
|
|
)
|
|
items.append(
|
|
_deployment(
|
|
name=worker_name,
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="worker",
|
|
replicas=pool.replicas,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"worker",
|
|
"--queues",
|
|
",".join(pool.queues),
|
|
"--concurrency",
|
|
str(pool.concurrency),
|
|
"--loglevel",
|
|
"INFO",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
s3_ca_secret_name=s3_ca_secret_name,
|
|
service_account=service_account,
|
|
extra_environment={
|
|
**_role_database_environment(environment, "WORKER"),
|
|
"CELERY_QUEUES": ",".join(pool.queues),
|
|
"CELERY_WORKER_CONCURRENCY": str(pool.concurrency),
|
|
"GOVOPLAN_WORKER_POOL": pool.name,
|
|
},
|
|
selector_labels={
|
|
"govoplan.add-ideas.de/worker-pool": pool.name,
|
|
},
|
|
)
|
|
)
|
|
if pool.replicas > 1:
|
|
items.append(
|
|
_pod_disruption_budget(
|
|
worker_name,
|
|
namespace,
|
|
labels,
|
|
"worker",
|
|
selector_labels={
|
|
"govoplan.add-ideas.de/worker-pool": pool.name,
|
|
},
|
|
)
|
|
)
|
|
if worker_pools:
|
|
items.append(
|
|
_deployment(
|
|
name=f"{name}-scheduler",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="scheduler",
|
|
replicas=1,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.fenced_run",
|
|
"--resource",
|
|
"scheduler:celery-beat",
|
|
"--wait-seconds",
|
|
"120",
|
|
"--",
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"beat",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--schedule",
|
|
"/tmp/celerybeat-schedule",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
s3_ca_secret_name=s3_ca_secret_name,
|
|
service_account=service_account,
|
|
extra_environment=_role_database_environment(
|
|
environment,
|
|
"SCHEDULER",
|
|
),
|
|
)
|
|
)
|
|
if spec.replicas.api > 1:
|
|
items.append(_pod_disruption_budget(f"{name}-api", namespace, labels, "api"))
|
|
if spec.replicas.web > 1:
|
|
items.append(_pod_disruption_budget(f"{name}-web", namespace, labels, "web"))
|
|
items.append(
|
|
_ingress(
|
|
spec,
|
|
name=name,
|
|
namespace=namespace,
|
|
labels=labels,
|
|
tls_secret_name=tls_secret_name,
|
|
ingress_class_name=ingress_class_name,
|
|
)
|
|
)
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "List",
|
|
"metadata": {
|
|
"annotations": {
|
|
"govoplan.add-ideas.de/profile": "stateless-shared-state",
|
|
"govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
|
|
"govoplan.add-ideas.de/s3-ca-secret": s3_ca_secret_name or "",
|
|
"govoplan.add-ideas.de/database-connection-peak": str(
|
|
database_capacity["peak"]
|
|
),
|
|
"govoplan.add-ideas.de/database-connection-available": str(
|
|
database_capacity["available"]
|
|
),
|
|
"govoplan.add-ideas.de/worker-pools": ",".join(
|
|
pool.name for pool in worker_pools
|
|
),
|
|
}
|
|
},
|
|
"items": items,
|
|
}
|
|
|
|
|
|
def kubernetes_secret_contract() -> tuple[str, ...]:
|
|
return _SECRET_KEYS
|
|
|
|
|
|
def _validate_cluster_profile(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
namespace: str,
|
|
secret_name: str,
|
|
*,
|
|
s3_ca_secret_name: str | None,
|
|
) -> None:
|
|
if not _DNS_LABEL.fullmatch(namespace) or not _DNS_LABEL.fullmatch(secret_name):
|
|
raise ValueError("Kubernetes namespace and secret names must be DNS labels")
|
|
if s3_ca_secret_name is not None and not _DNS_LABEL.fullmatch(s3_ca_secret_name):
|
|
raise ValueError("Kubernetes S3 CA secret name must be a DNS label")
|
|
if spec.installation_id == "govoplan-local":
|
|
raise ValueError(
|
|
"Kubernetes export requires a non-default stable installation id"
|
|
)
|
|
if spec.components.postgres.mode != "external":
|
|
raise ValueError("Kubernetes export requires external shared PostgreSQL")
|
|
if spec.components.redis.mode != "external":
|
|
raise ValueError("Kubernetes export requires external shared Redis")
|
|
if spec.components.storage.mode != "s3":
|
|
raise ValueError("Kubernetes export requires external shared S3 storage")
|
|
unpinned_images = [
|
|
label
|
|
for label, image in (
|
|
("release.api_image", spec.release.api_image),
|
|
("release.web_image", spec.release.web_image),
|
|
)
|
|
if not image_is_digest_pinned(image)
|
|
]
|
|
if unpinned_images:
|
|
raise ValueError(
|
|
"Kubernetes export requires immutable digest-pinned release images: "
|
|
+ ", ".join(unpinned_images)
|
|
)
|
|
missing = [
|
|
key
|
|
for key in (
|
|
*_SECRET_KEYS,
|
|
"CELERY_QUEUES",
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
|
)
|
|
if not environment.get(key)
|
|
]
|
|
if missing:
|
|
raise ValueError(
|
|
"Kubernetes runtime contract is missing: " + ", ".join(missing)
|
|
)
|
|
|
|
|
|
def _worker_pools(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
) -> tuple[_WorkerPool, ...]:
|
|
if spec.replicas.worker == 0:
|
|
if str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip():
|
|
raise ValueError("Worker pools require at least one worker replica")
|
|
return ()
|
|
configured_queues = tuple(
|
|
item.strip()
|
|
for item in str(environment.get("CELERY_QUEUES") or "").split(",")
|
|
if item.strip()
|
|
)
|
|
if not configured_queues or any(
|
|
_QUEUE_NAME.fullmatch(item) is None for item in configured_queues
|
|
):
|
|
raise ValueError("CELERY_QUEUES must contain canonical queue names")
|
|
raw_pools = str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip()
|
|
if not raw_pools:
|
|
return (
|
|
_WorkerPool(
|
|
name="default",
|
|
queues=configured_queues,
|
|
replicas=spec.replicas.worker,
|
|
concurrency=_bounded_environment_integer(
|
|
environment,
|
|
"CELERY_WORKER_CONCURRENCY",
|
|
default=2,
|
|
minimum=1,
|
|
maximum=64,
|
|
),
|
|
),
|
|
)
|
|
try:
|
|
payload = json.loads(raw_pools)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError("GOVOPLAN_WORKER_POOLS must be valid JSON") from exc
|
|
if not isinstance(payload, list) or not payload:
|
|
raise ValueError("GOVOPLAN_WORKER_POOLS must be a non-empty JSON list")
|
|
pools: list[_WorkerPool] = []
|
|
seen_names: set[str] = set()
|
|
seen_queues: set[str] = set()
|
|
for index, item in enumerate(payload):
|
|
if not isinstance(item, dict):
|
|
raise ValueError(f"Worker pool {index} must be an object")
|
|
unknown = set(item) - {"name", "queues", "replicas", "concurrency"}
|
|
if unknown:
|
|
raise ValueError(
|
|
f"Worker pool {index} has unsupported keys: "
|
|
+ ", ".join(sorted(unknown))
|
|
)
|
|
pool_name = item.get("name")
|
|
queues = item.get("queues")
|
|
replicas = item.get("replicas")
|
|
concurrency = item.get("concurrency")
|
|
if not isinstance(pool_name, str) or _DNS_LABEL.fullmatch(pool_name) is None:
|
|
raise ValueError(f"Worker pool {index} has an invalid name")
|
|
if pool_name in seen_names:
|
|
raise ValueError(f"Worker pool name is duplicated: {pool_name}")
|
|
if (
|
|
not isinstance(queues, list)
|
|
or not queues
|
|
or any(
|
|
not isinstance(queue, str) or _QUEUE_NAME.fullmatch(queue) is None
|
|
for queue in queues
|
|
)
|
|
):
|
|
raise ValueError(f"Worker pool {pool_name} has invalid queues")
|
|
duplicate_queues = seen_queues.intersection(queues)
|
|
if duplicate_queues:
|
|
raise ValueError(
|
|
"Worker queues must have one owning pool: "
|
|
+ ", ".join(sorted(duplicate_queues))
|
|
)
|
|
if not isinstance(replicas, int) or not 1 <= replicas <= 128:
|
|
raise ValueError(
|
|
f"Worker pool {pool_name} replicas must be between 1 and 128"
|
|
)
|
|
if not isinstance(concurrency, int) or not 1 <= concurrency <= 64:
|
|
raise ValueError(
|
|
f"Worker pool {pool_name} concurrency must be between 1 and 64"
|
|
)
|
|
pools.append(
|
|
_WorkerPool(
|
|
name=pool_name,
|
|
queues=tuple(queues),
|
|
replicas=replicas,
|
|
concurrency=concurrency,
|
|
)
|
|
)
|
|
seen_names.add(pool_name)
|
|
seen_queues.update(queues)
|
|
if sum(pool.replicas for pool in pools) != spec.replicas.worker:
|
|
raise ValueError("Worker pool replicas must equal installation replicas.worker")
|
|
configured_queue_set = set(configured_queues)
|
|
if seen_queues != configured_queue_set:
|
|
missing = configured_queue_set - seen_queues
|
|
unknown = seen_queues - configured_queue_set
|
|
detail = []
|
|
if missing:
|
|
detail.append("missing " + ", ".join(sorted(missing)))
|
|
if unknown:
|
|
detail.append("unknown " + ", ".join(sorted(unknown)))
|
|
raise ValueError(
|
|
"Worker pools must cover CELERY_QUEUES exactly: " + "; ".join(detail)
|
|
)
|
|
return tuple(sorted(pools, key=lambda pool: pool.name))
|
|
|
|
|
|
def _database_capacity(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
worker_pools: tuple[_WorkerPool, ...],
|
|
) -> dict[str, int]:
|
|
limit = _bounded_environment_integer(
|
|
environment,
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
|
minimum=10,
|
|
maximum=1_000_000,
|
|
)
|
|
reserve = _bounded_environment_integer(
|
|
environment,
|
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
|
default=10,
|
|
minimum=1,
|
|
maximum=999_999,
|
|
)
|
|
if reserve >= limit:
|
|
raise ValueError("Database connection reserve must be below the limit")
|
|
api_ceiling = _role_database_ceiling(environment, "API", 5, 2)
|
|
worker_ceiling = _role_database_ceiling(environment, "WORKER", 1, 1)
|
|
scheduler_ceiling = _role_database_ceiling(environment, "SCHEDULER", 1, 0)
|
|
migration_ceiling = _role_database_ceiling(environment, "MIGRATION", 2, 0)
|
|
worker_processes = sum(
|
|
pool.replicas * (pool.concurrency + 1) for pool in worker_pools
|
|
)
|
|
steady = (
|
|
spec.replicas.api * api_ceiling
|
|
+ worker_processes * worker_ceiling
|
|
+ (scheduler_ceiling if worker_pools else 0)
|
|
)
|
|
rollout_surge = api_ceiling + migration_ceiling
|
|
if worker_pools:
|
|
rollout_surge += scheduler_ceiling
|
|
rollout_surge += sum(
|
|
(pool.concurrency + 1) * worker_ceiling for pool in worker_pools
|
|
)
|
|
peak = steady + rollout_surge
|
|
available = limit - reserve
|
|
if peak > available:
|
|
raise ValueError(
|
|
"Kubernetes database connection peak exceeds the configured budget: "
|
|
f"peak={peak}, available={available}, limit={limit}, reserve={reserve}"
|
|
)
|
|
return {
|
|
"limit": limit,
|
|
"reserve": reserve,
|
|
"available": available,
|
|
"steady": steady,
|
|
"peak": peak,
|
|
}
|
|
|
|
|
|
def _role_database_ceiling(
|
|
environment: Mapping[str, str],
|
|
role: str,
|
|
default_pool_size: int,
|
|
default_overflow: int,
|
|
) -> int:
|
|
return _bounded_environment_integer(
|
|
environment,
|
|
f"GOVOPLAN_{role}_DB_POOL_SIZE",
|
|
default=default_pool_size,
|
|
minimum=1,
|
|
maximum=100,
|
|
) + _bounded_environment_integer(
|
|
environment,
|
|
f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
|
|
default=default_overflow,
|
|
minimum=0,
|
|
maximum=200,
|
|
)
|
|
|
|
|
|
def _role_database_environment(
|
|
environment: Mapping[str, str],
|
|
role: str,
|
|
) -> dict[str, str]:
|
|
defaults = {
|
|
"API": (5, 2),
|
|
"WORKER": (1, 1),
|
|
"SCHEDULER": (1, 0),
|
|
"MIGRATION": (2, 0),
|
|
}
|
|
pool_size, overflow = defaults[role]
|
|
return {
|
|
"GOVOPLAN_DB_POOL_SIZE": str(
|
|
_bounded_environment_integer(
|
|
environment,
|
|
f"GOVOPLAN_{role}_DB_POOL_SIZE",
|
|
default=pool_size,
|
|
minimum=1,
|
|
maximum=100,
|
|
)
|
|
),
|
|
"GOVOPLAN_DB_MAX_OVERFLOW": str(
|
|
_bounded_environment_integer(
|
|
environment,
|
|
f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
|
|
default=overflow,
|
|
minimum=0,
|
|
maximum=200,
|
|
)
|
|
),
|
|
}
|
|
|
|
|
|
def _bounded_environment_integer(
|
|
environment: Mapping[str, str],
|
|
name: str,
|
|
*,
|
|
default: int | None = None,
|
|
minimum: int,
|
|
maximum: int,
|
|
) -> int:
|
|
raw = environment.get(name)
|
|
if raw is None or not str(raw).strip():
|
|
if default is None:
|
|
raise ValueError(f"{name} is required")
|
|
return default
|
|
try:
|
|
value = int(str(raw))
|
|
except ValueError as exc:
|
|
raise ValueError(f"{name} must be an integer") from exc
|
|
if not minimum <= value <= maximum:
|
|
raise ValueError(f"{name} must be between {minimum} and {maximum}")
|
|
return value
|
|
|
|
|
|
def _deployment(
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
replicas: int,
|
|
image: str,
|
|
command: tuple[str, ...],
|
|
config_name: str | None,
|
|
secret_name: str | None,
|
|
s3_ca_secret_name: str | None,
|
|
service_account: str,
|
|
container_port: int | None = None,
|
|
readiness_path: str | None = None,
|
|
liveness_path: str | None = None,
|
|
probe_host: str | None = None,
|
|
graceful_shutdown_seconds: int = 0,
|
|
extra_environment: Mapping[str, str] | None = None,
|
|
selector_labels: Mapping[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
role_labels = {
|
|
**labels,
|
|
"app.kubernetes.io/component": role,
|
|
**dict(selector_labels or {}),
|
|
}
|
|
environment: list[dict[str, Any]] = [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
|
|
{
|
|
"name": "GOVOPLAN_NODE_ID",
|
|
"valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}},
|
|
},
|
|
]
|
|
environment.extend(
|
|
{"name": key, "value": value}
|
|
for key, value in sorted((extra_environment or {}).items())
|
|
)
|
|
if secret_name:
|
|
environment.extend(_secret_environment(secret_name))
|
|
if s3_ca_secret_name:
|
|
environment.append(
|
|
{"name": "AWS_CA_BUNDLE", "value": "/etc/govoplan/trust/s3-ca.crt"}
|
|
)
|
|
container: dict[str, Any] = {
|
|
"name": role,
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"env": environment,
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"resources": {
|
|
"requests": {"cpu": "100m", "memory": "256Mi"},
|
|
"limits": {"memory": "1Gi"},
|
|
},
|
|
}
|
|
if command:
|
|
container["command"] = list(command)
|
|
if container_port is not None:
|
|
container["ports"] = [{"name": "http", "containerPort": container_port}]
|
|
if readiness_path:
|
|
container["readinessProbe"] = _http_probe(
|
|
readiness_path,
|
|
container_port or 8000,
|
|
host=probe_host,
|
|
)
|
|
if liveness_path:
|
|
container["livenessProbe"] = _http_probe(
|
|
liveness_path,
|
|
container_port or 8000,
|
|
host=probe_host,
|
|
)
|
|
if graceful_shutdown_seconds:
|
|
container["lifecycle"] = {
|
|
"preStop": {
|
|
"exec": {
|
|
"command": [
|
|
"/bin/sh",
|
|
"-c",
|
|
f"sleep {graceful_shutdown_seconds}",
|
|
]
|
|
}
|
|
}
|
|
}
|
|
pod_spec: dict[str, Any] = {
|
|
"serviceAccountName": service_account,
|
|
"automountServiceAccountToken": False,
|
|
"securityContext": {
|
|
"runAsNonRoot": True,
|
|
"seccompProfile": {"type": "RuntimeDefault"},
|
|
},
|
|
"containers": [container],
|
|
"volumes": [{"name": "tmp", "emptyDir": {}}],
|
|
"topologySpreadConstraints": [
|
|
{
|
|
"maxSkew": 1,
|
|
"topologyKey": "kubernetes.io/hostname",
|
|
"whenUnsatisfiable": "DoNotSchedule",
|
|
"matchLabelKeys": ["pod-template-hash"],
|
|
"labelSelector": {"matchLabels": role_labels},
|
|
}
|
|
],
|
|
}
|
|
if graceful_shutdown_seconds:
|
|
pod_spec["terminationGracePeriodSeconds"] = max(
|
|
30,
|
|
graceful_shutdown_seconds + 20,
|
|
)
|
|
container["volumeMounts"] = [{"name": "tmp", "mountPath": "/tmp"}]
|
|
if s3_ca_secret_name:
|
|
pod_spec["volumes"].append(_s3_ca_volume(s3_ca_secret_name))
|
|
container["volumeMounts"].append(_s3_ca_volume_mount())
|
|
if config_name:
|
|
pod_spec["containers"][0]["envFrom"] = [{"configMapRef": {"name": config_name}}]
|
|
if config_name and secret_name:
|
|
pod_spec["initContainers"] = [
|
|
{
|
|
"name": "wait-for-database",
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.wait_for_database",
|
|
"--timeout-seconds",
|
|
"900",
|
|
],
|
|
"envFrom": [{"configMapRef": {"name": config_name}}],
|
|
"env": [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
|
|
{"name": "GOVOPLAN_DB_POOL_SIZE", "value": "1"},
|
|
{"name": "GOVOPLAN_DB_MAX_OVERFLOW", "value": "0"},
|
|
*(
|
|
[
|
|
{
|
|
"name": "AWS_CA_BUNDLE",
|
|
"value": "/etc/govoplan/trust/s3-ca.crt",
|
|
}
|
|
]
|
|
if s3_ca_secret_name
|
|
else []
|
|
),
|
|
*_secret_environment(secret_name),
|
|
],
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"volumeMounts": [
|
|
{"name": "tmp", "mountPath": "/tmp"},
|
|
*([_s3_ca_volume_mount()] if s3_ca_secret_name else []),
|
|
],
|
|
}
|
|
]
|
|
return {
|
|
"apiVersion": "apps/v1",
|
|
"kind": "Deployment",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {
|
|
"replicas": replicas,
|
|
"strategy": {
|
|
"type": "RollingUpdate",
|
|
"rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1},
|
|
},
|
|
"selector": {"matchLabels": role_labels},
|
|
"template": {
|
|
"metadata": {"labels": role_labels},
|
|
"spec": pod_spec,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _migration_job(
|
|
*,
|
|
name: str,
|
|
release_key: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
image: str,
|
|
config_name: str,
|
|
secret_name: str,
|
|
s3_ca_secret_name: str | None,
|
|
service_account: str,
|
|
database_environment: Mapping[str, str],
|
|
backup_required: bool,
|
|
backup_evidence: Mapping[str, object] | None,
|
|
) -> dict[str, Any]:
|
|
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
|
|
job_name = _name_with_suffix(name, f"migrate-{release_key}")
|
|
backup_annotations = {
|
|
"govoplan.add-ideas.de/backup-required": str(backup_required).lower(),
|
|
}
|
|
if backup_evidence is not None:
|
|
backup_annotations.update(
|
|
{
|
|
"govoplan.add-ideas.de/backup-evidence-sha256": str(
|
|
backup_evidence["evidence_sha256"]
|
|
),
|
|
"govoplan.add-ideas.de/recovery-point": str(
|
|
backup_evidence["recovery_point_id"]
|
|
),
|
|
"govoplan.add-ideas.de/restore-drill": str(
|
|
backup_evidence["restore_drill_id"]
|
|
),
|
|
}
|
|
)
|
|
return {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"metadata": {
|
|
"name": job_name,
|
|
"namespace": namespace,
|
|
"labels": job_labels,
|
|
"annotations": {
|
|
"govoplan.add-ideas.de/recovery-mode": "forward-recovery",
|
|
**backup_annotations,
|
|
"argocd.argoproj.io/sync-wave": "-1",
|
|
},
|
|
},
|
|
"spec": {
|
|
"backoffLimit": 1,
|
|
"ttlSecondsAfterFinished": 86400,
|
|
"template": {
|
|
"metadata": {"labels": job_labels},
|
|
"spec": {
|
|
"restartPolicy": "Never",
|
|
"serviceAccountName": service_account,
|
|
"automountServiceAccountToken": False,
|
|
"securityContext": {
|
|
"runAsNonRoot": True,
|
|
"seccompProfile": {"type": "RuntimeDefault"},
|
|
},
|
|
"containers": [
|
|
{
|
|
"name": "migration",
|
|
"image": image,
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.init_db",
|
|
],
|
|
"envFrom": [{"configMapRef": {"name": config_name}}],
|
|
"env": [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
|
|
*(
|
|
[
|
|
{
|
|
"name": "AWS_CA_BUNDLE",
|
|
"value": "/etc/govoplan/trust/s3-ca.crt",
|
|
}
|
|
]
|
|
if s3_ca_secret_name
|
|
else []
|
|
),
|
|
*(
|
|
{"name": key, "value": value}
|
|
for key, value in sorted(
|
|
database_environment.items()
|
|
)
|
|
),
|
|
{
|
|
"name": "GOVOPLAN_NODE_ID",
|
|
"valueFrom": {
|
|
"fieldRef": {"fieldPath": "metadata.name"}
|
|
},
|
|
},
|
|
*_secret_environment(secret_name),
|
|
],
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"volumeMounts": [
|
|
{"name": "tmp", "mountPath": "/tmp"},
|
|
*(
|
|
[_s3_ca_volume_mount()]
|
|
if s3_ca_secret_name
|
|
else []
|
|
),
|
|
],
|
|
}
|
|
],
|
|
"volumes": [
|
|
{"name": "tmp", "emptyDir": {}},
|
|
*(
|
|
[_s3_ca_volume(s3_ca_secret_name)]
|
|
if s3_ca_secret_name
|
|
else []
|
|
),
|
|
],
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _secret_environment(secret_name: str) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"name": key,
|
|
"valueFrom": {
|
|
"secretKeyRef": {"name": secret_name, "key": key, "optional": False}
|
|
},
|
|
}
|
|
for key in _SECRET_KEYS
|
|
]
|
|
|
|
|
|
def _s3_ca_volume(secret_name: str) -> dict[str, Any]:
|
|
return {
|
|
"name": "s3-ca",
|
|
"secret": {
|
|
"secretName": secret_name,
|
|
"items": [{"key": "ca.crt", "path": "s3-ca.crt"}],
|
|
},
|
|
}
|
|
|
|
|
|
def _s3_ca_volume_mount() -> dict[str, Any]:
|
|
return {
|
|
"name": "s3-ca",
|
|
"mountPath": "/etc/govoplan/trust",
|
|
"readOnly": True,
|
|
}
|
|
|
|
|
|
def _service(
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
port: int,
|
|
) -> dict[str, Any]:
|
|
role_labels = {**labels, "app.kubernetes.io/component": role}
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "Service",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {
|
|
"selector": role_labels,
|
|
"ports": [{"name": "http", "port": port, "targetPort": "http"}],
|
|
},
|
|
}
|
|
|
|
|
|
def _pod_disruption_budget(
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
*,
|
|
selector_labels: Mapping[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
role_labels = {
|
|
**labels,
|
|
"app.kubernetes.io/component": role,
|
|
**dict(selector_labels or {}),
|
|
}
|
|
return {
|
|
"apiVersion": "policy/v1",
|
|
"kind": "PodDisruptionBudget",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {"minAvailable": 1, "selector": {"matchLabels": role_labels}},
|
|
}
|
|
|
|
|
|
def _ingress(
|
|
spec: InstallationSpec,
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
tls_secret_name: str,
|
|
ingress_class_name: str | None,
|
|
) -> dict[str, Any]:
|
|
public = urlsplit(spec.public_url)
|
|
ingress_spec: dict[str, Any] = {
|
|
"rules": [
|
|
{
|
|
"host": public.hostname,
|
|
"http": {
|
|
"paths": [
|
|
{
|
|
"path": "/",
|
|
"pathType": "Prefix",
|
|
"backend": {
|
|
"service": {
|
|
"name": f"{name}-web",
|
|
"port": {"name": "http"},
|
|
}
|
|
},
|
|
}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
}
|
|
if public.scheme == "https":
|
|
if not tls_secret_name:
|
|
raise ValueError("HTTPS Kubernetes export requires a TLS secret name")
|
|
ingress_spec["tls"] = [
|
|
{"hosts": [public.hostname], "secretName": tls_secret_name}
|
|
]
|
|
if ingress_class_name:
|
|
ingress_spec["ingressClassName"] = ingress_class_name
|
|
return {
|
|
"apiVersion": "networking.k8s.io/v1",
|
|
"kind": "Ingress",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": labels},
|
|
"spec": ingress_spec,
|
|
}
|
|
|
|
|
|
def _http_probe(
|
|
path: str,
|
|
port: int,
|
|
*,
|
|
host: str | None = None,
|
|
) -> dict[str, Any]:
|
|
request: dict[str, Any] = {"path": path, "port": port}
|
|
if host:
|
|
request["httpHeaders"] = [{"name": "Host", "value": host}]
|
|
return {
|
|
"httpGet": request,
|
|
"initialDelaySeconds": 10,
|
|
"periodSeconds": 10,
|
|
"timeoutSeconds": 3,
|
|
"failureThreshold": 6,
|
|
}
|
|
|
|
|
|
def _resource_name(installation_id: str) -> str:
|
|
value = installation_id[:48].rstrip("-")
|
|
if not _DNS_LABEL.fullmatch(value):
|
|
raise ValueError("Installation id cannot be represented as a Kubernetes name")
|
|
return value
|
|
|
|
|
|
def _release_key(spec: InstallationSpec) -> str:
|
|
manifest_hash = str(spec.release.manifest_sha256 or "").strip().lower()
|
|
if re.fullmatch(r"[0-9a-f]{8,}", manifest_hash):
|
|
return manifest_hash[:8]
|
|
payload = "\0".join(
|
|
(
|
|
spec.release.version,
|
|
spec.release.api_image,
|
|
spec.release.web_image,
|
|
)
|
|
).encode("utf-8")
|
|
return sha256(payload).hexdigest()[:8]
|
|
|
|
|
|
def _name_with_suffix(name: str, suffix: str) -> str:
|
|
available = 63 - len(suffix) - 1
|
|
prefix = name[:available].rstrip("-")
|
|
candidate = f"{prefix}-{suffix}"
|
|
if not _DNS_LABEL.fullmatch(candidate):
|
|
raise ValueError("Kubernetes resource name cannot include the release suffix")
|
|
return candidate
|
|
|
|
|
|
def write_secret_creation_hint(
|
|
environment_path: Path,
|
|
*,
|
|
namespace: str,
|
|
secret_name: str,
|
|
) -> str:
|
|
keys = " ".join(f"--from-env-file={environment_path}" for _ in (0,))
|
|
return (
|
|
f"kubectl -n {namespace} create secret generic {secret_name} {keys} "
|
|
"--dry-run=client -o yaml | kubectl apply -f -"
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"kubernetes_secret_contract",
|
|
"render_kubernetes",
|
|
"write_secret_creation_hint",
|
|
]
|