667 lines
21 KiB
Python
667 lines
21 KiB
Python
"""Deterministic Kubernetes export for the stateless multi-host profile."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
import re
|
|
from typing import Any, Mapping
|
|
from urllib.parse import urlsplit
|
|
|
|
from .model import InstallationSpec, image_is_digest_pinned
|
|
|
|
|
|
_DNS_LABEL = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$")
|
|
_SECRET_KEYS = (
|
|
"MASTER_KEY_B64",
|
|
"DATABASE_URL",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS",
|
|
"REDIS_URL",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
|
)
|
|
_CONFIG_KEYS = (
|
|
"APP_ENV",
|
|
"GOVOPLAN_INSTALL_PROFILE",
|
|
"ENABLED_MODULES",
|
|
"CELERY_ENABLED",
|
|
"CELERY_QUEUES",
|
|
"CORS_ORIGINS",
|
|
"GOVOPLAN_TRUSTED_HOSTS",
|
|
"FORWARDED_ALLOW_IPS",
|
|
"AUTH_COOKIE_SECURE",
|
|
"AUTH_COOKIE_SAMESITE",
|
|
"GOVOPLAN_HTTP_HSTS_SECONDS",
|
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
|
|
"GOVOPLAN_MIGRATION_TRACK",
|
|
"DEV_AUTO_MIGRATE_ENABLED",
|
|
"DEV_BOOTSTRAP_ENABLED",
|
|
"FILE_STORAGE_BACKEND",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
|
"FILE_STORAGE_S3_REGION",
|
|
"FILE_STORAGE_S3_BUCKET",
|
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
|
)
|
|
|
|
|
|
def render_kubernetes(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
*,
|
|
namespace: str = "govoplan",
|
|
secret_name: str = "govoplan-runtime",
|
|
tls_secret_name: str = "govoplan-tls",
|
|
ingress_class_name: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Render runtime roles only; shared state services stay externally managed."""
|
|
|
|
_validate_cluster_profile(spec, environment, namespace, secret_name)
|
|
name = _resource_name(spec.installation_id)
|
|
public_host = urlsplit(spec.public_url).hostname or "localhost"
|
|
labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
|
|
config_name = f"{name}-runtime"
|
|
service_account = f"{name}-runtime"
|
|
config = {
|
|
key: str(environment[key])
|
|
for key in _CONFIG_KEYS
|
|
if environment.get(key) is not None
|
|
}
|
|
config.update(
|
|
{
|
|
"GOVOPLAN_INSTALLATION_ID": spec.installation_id,
|
|
"GOVOPLAN_STATE_PROFILE": "shared",
|
|
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "15",
|
|
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
|
|
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
|
|
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
|
|
"FILE_STORAGE_BACKEND": "s3",
|
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
|
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
|
|
# Trust no cross-pod proxy headers by default. Operators can bind
|
|
# an exact ingress-controller CIDR through their ConfigMap overlay.
|
|
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
|
}
|
|
)
|
|
items: list[dict[str, Any]] = [
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "Namespace",
|
|
"metadata": {"name": namespace, "labels": labels},
|
|
},
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "ServiceAccount",
|
|
"metadata": {
|
|
"name": service_account,
|
|
"namespace": namespace,
|
|
"labels": labels,
|
|
},
|
|
"automountServiceAccountToken": False,
|
|
},
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {"name": config_name, "namespace": namespace, "labels": labels},
|
|
"data": dict(sorted(config.items())),
|
|
},
|
|
_deployment(
|
|
name=f"{name}-api",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="api",
|
|
replicas=spec.replicas.api,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"uvicorn",
|
|
"govoplan_core.server.app:app",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
"8000",
|
|
"--proxy-headers",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
service_account=service_account,
|
|
container_port=8000,
|
|
readiness_path="/health/ready",
|
|
liveness_path="/health",
|
|
probe_host=public_host,
|
|
),
|
|
_service(
|
|
name=f"{name}-api",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="api",
|
|
port=8000,
|
|
),
|
|
_deployment(
|
|
name=f"{name}-web",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="web",
|
|
replicas=spec.replicas.web,
|
|
image=spec.release.web_image,
|
|
command=(),
|
|
config_name=None,
|
|
secret_name=None,
|
|
service_account=service_account,
|
|
container_port=8080,
|
|
extra_environment={"GOVOPLAN_API_UPSTREAM": f"http://{name}-api:8000"},
|
|
),
|
|
_service(
|
|
name=f"{name}-web",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="web",
|
|
port=8080,
|
|
),
|
|
_migration_job(
|
|
name=name,
|
|
release_key=_release_key(spec),
|
|
namespace=namespace,
|
|
labels=labels,
|
|
image=spec.release.api_image,
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
service_account=service_account,
|
|
),
|
|
]
|
|
if spec.replicas.worker:
|
|
items.append(
|
|
_deployment(
|
|
name=f"{name}-worker",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="worker",
|
|
replicas=spec.replicas.worker,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"worker",
|
|
"--queues",
|
|
str(environment["CELERY_QUEUES"]),
|
|
"--loglevel",
|
|
"INFO",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
service_account=service_account,
|
|
)
|
|
)
|
|
items.append(
|
|
_deployment(
|
|
name=f"{name}-scheduler",
|
|
namespace=namespace,
|
|
labels=labels,
|
|
role="scheduler",
|
|
replicas=1,
|
|
image=spec.release.api_image,
|
|
command=(
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.fenced_run",
|
|
"--resource",
|
|
"scheduler:celery-beat",
|
|
"--wait-seconds",
|
|
"120",
|
|
"--",
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"beat",
|
|
"--loglevel",
|
|
"INFO",
|
|
),
|
|
config_name=config_name,
|
|
secret_name=secret_name,
|
|
service_account=service_account,
|
|
)
|
|
)
|
|
if spec.replicas.api > 1:
|
|
items.append(_pod_disruption_budget(f"{name}-api", namespace, labels, "api"))
|
|
if spec.replicas.web > 1:
|
|
items.append(_pod_disruption_budget(f"{name}-web", namespace, labels, "web"))
|
|
items.append(
|
|
_ingress(
|
|
spec,
|
|
name=name,
|
|
namespace=namespace,
|
|
labels=labels,
|
|
tls_secret_name=tls_secret_name,
|
|
ingress_class_name=ingress_class_name,
|
|
)
|
|
)
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "List",
|
|
"metadata": {
|
|
"annotations": {
|
|
"govoplan.add-ideas.de/profile": "stateless-shared-state",
|
|
"govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
|
|
}
|
|
},
|
|
"items": items,
|
|
}
|
|
|
|
|
|
def kubernetes_secret_contract() -> tuple[str, ...]:
|
|
return _SECRET_KEYS
|
|
|
|
|
|
def _validate_cluster_profile(
|
|
spec: InstallationSpec,
|
|
environment: Mapping[str, str],
|
|
namespace: str,
|
|
secret_name: str,
|
|
) -> None:
|
|
if not _DNS_LABEL.fullmatch(namespace) or not _DNS_LABEL.fullmatch(secret_name):
|
|
raise ValueError("Kubernetes namespace and secret names must be DNS labels")
|
|
if spec.installation_id == "govoplan-local":
|
|
raise ValueError(
|
|
"Kubernetes export requires a non-default stable installation id"
|
|
)
|
|
if spec.components.postgres.mode != "external":
|
|
raise ValueError("Kubernetes export requires external shared PostgreSQL")
|
|
if spec.components.redis.mode != "external":
|
|
raise ValueError("Kubernetes export requires external shared Redis")
|
|
if spec.components.storage.mode != "s3":
|
|
raise ValueError("Kubernetes export requires external shared S3 storage")
|
|
unpinned_images = [
|
|
label
|
|
for label, image in (
|
|
("release.api_image", spec.release.api_image),
|
|
("release.web_image", spec.release.web_image),
|
|
)
|
|
if not image_is_digest_pinned(image)
|
|
]
|
|
if unpinned_images:
|
|
raise ValueError(
|
|
"Kubernetes export requires immutable digest-pinned release images: "
|
|
+ ", ".join(unpinned_images)
|
|
)
|
|
missing = [
|
|
key for key in (*_SECRET_KEYS, "CELERY_QUEUES") if not environment.get(key)
|
|
]
|
|
if missing:
|
|
raise ValueError(
|
|
"Kubernetes runtime contract is missing: " + ", ".join(missing)
|
|
)
|
|
|
|
|
|
def _deployment(
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
replicas: int,
|
|
image: str,
|
|
command: tuple[str, ...],
|
|
config_name: str | None,
|
|
secret_name: str | None,
|
|
service_account: str,
|
|
container_port: int | None = None,
|
|
readiness_path: str | None = None,
|
|
liveness_path: str | None = None,
|
|
probe_host: str | None = None,
|
|
extra_environment: Mapping[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
role_labels = {**labels, "app.kubernetes.io/component": role}
|
|
environment: list[dict[str, Any]] = [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
|
|
{
|
|
"name": "GOVOPLAN_NODE_ID",
|
|
"valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}},
|
|
},
|
|
]
|
|
environment.extend(
|
|
{"name": key, "value": value}
|
|
for key, value in sorted((extra_environment or {}).items())
|
|
)
|
|
if secret_name:
|
|
environment.extend(_secret_environment(secret_name))
|
|
container: dict[str, Any] = {
|
|
"name": role,
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"env": environment,
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"resources": {
|
|
"requests": {"cpu": "100m", "memory": "256Mi"},
|
|
"limits": {"memory": "1Gi"},
|
|
},
|
|
}
|
|
if command:
|
|
container["command"] = list(command)
|
|
if container_port is not None:
|
|
container["ports"] = [{"name": "http", "containerPort": container_port}]
|
|
if readiness_path:
|
|
container["readinessProbe"] = _http_probe(
|
|
readiness_path,
|
|
container_port or 8000,
|
|
host=probe_host,
|
|
)
|
|
if liveness_path:
|
|
container["livenessProbe"] = _http_probe(
|
|
liveness_path,
|
|
container_port or 8000,
|
|
host=probe_host,
|
|
)
|
|
pod_spec: dict[str, Any] = {
|
|
"serviceAccountName": service_account,
|
|
"automountServiceAccountToken": False,
|
|
"securityContext": {
|
|
"runAsNonRoot": True,
|
|
"seccompProfile": {"type": "RuntimeDefault"},
|
|
},
|
|
"containers": [container],
|
|
"volumes": [{"name": "tmp", "emptyDir": {}}],
|
|
"topologySpreadConstraints": [
|
|
{
|
|
"maxSkew": 1,
|
|
"topologyKey": "kubernetes.io/hostname",
|
|
"whenUnsatisfiable": "ScheduleAnyway",
|
|
"labelSelector": {"matchLabels": role_labels},
|
|
}
|
|
],
|
|
}
|
|
container["volumeMounts"] = [{"name": "tmp", "mountPath": "/tmp"}]
|
|
if config_name:
|
|
pod_spec["containers"][0]["envFrom"] = [{"configMapRef": {"name": config_name}}]
|
|
if config_name and secret_name:
|
|
pod_spec["initContainers"] = [
|
|
{
|
|
"name": "wait-for-database",
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.wait_for_database",
|
|
"--timeout-seconds",
|
|
"900",
|
|
],
|
|
"envFrom": [{"configMapRef": {"name": config_name}}],
|
|
"env": [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
|
|
*_secret_environment(secret_name),
|
|
],
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
|
|
}
|
|
]
|
|
return {
|
|
"apiVersion": "apps/v1",
|
|
"kind": "Deployment",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {
|
|
"replicas": replicas,
|
|
"strategy": {
|
|
"type": "RollingUpdate",
|
|
"rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1},
|
|
},
|
|
"selector": {"matchLabels": role_labels},
|
|
"template": {
|
|
"metadata": {"labels": role_labels},
|
|
"spec": pod_spec,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _migration_job(
|
|
*,
|
|
name: str,
|
|
release_key: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
image: str,
|
|
config_name: str,
|
|
secret_name: str,
|
|
service_account: str,
|
|
) -> dict[str, Any]:
|
|
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
|
|
job_name = _name_with_suffix(name, f"migrate-{release_key}")
|
|
return {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"metadata": {
|
|
"name": job_name,
|
|
"namespace": namespace,
|
|
"labels": job_labels,
|
|
"annotations": {
|
|
"govoplan.add-ideas.de/recovery-mode": "forward-recovery",
|
|
"govoplan.add-ideas.de/backup-required": "true",
|
|
"argocd.argoproj.io/sync-wave": "-1",
|
|
},
|
|
},
|
|
"spec": {
|
|
"backoffLimit": 1,
|
|
"ttlSecondsAfterFinished": 86400,
|
|
"template": {
|
|
"metadata": {"labels": job_labels},
|
|
"spec": {
|
|
"restartPolicy": "Never",
|
|
"serviceAccountName": service_account,
|
|
"automountServiceAccountToken": False,
|
|
"securityContext": {
|
|
"runAsNonRoot": True,
|
|
"seccompProfile": {"type": "RuntimeDefault"},
|
|
},
|
|
"containers": [
|
|
{
|
|
"name": "migration",
|
|
"image": image,
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.init_db",
|
|
],
|
|
"envFrom": [{"configMapRef": {"name": config_name}}],
|
|
"env": [
|
|
{"name": "TMPDIR", "value": "/tmp"},
|
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
|
|
{
|
|
"name": "GOVOPLAN_NODE_ID",
|
|
"valueFrom": {
|
|
"fieldRef": {"fieldPath": "metadata.name"}
|
|
},
|
|
},
|
|
*_secret_environment(secret_name),
|
|
],
|
|
"securityContext": {
|
|
"allowPrivilegeEscalation": False,
|
|
"capabilities": {"drop": ["ALL"]},
|
|
"readOnlyRootFilesystem": True,
|
|
},
|
|
"volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
|
|
}
|
|
],
|
|
"volumes": [{"name": "tmp", "emptyDir": {}}],
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _secret_environment(secret_name: str) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"name": key,
|
|
"valueFrom": {
|
|
"secretKeyRef": {"name": secret_name, "key": key, "optional": False}
|
|
},
|
|
}
|
|
for key in _SECRET_KEYS
|
|
]
|
|
|
|
|
|
def _service(
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
port: int,
|
|
) -> dict[str, Any]:
|
|
role_labels = {**labels, "app.kubernetes.io/component": role}
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "Service",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {
|
|
"selector": role_labels,
|
|
"ports": [{"name": "http", "port": port, "targetPort": "http"}],
|
|
},
|
|
}
|
|
|
|
|
|
def _pod_disruption_budget(
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
role: str,
|
|
) -> dict[str, Any]:
|
|
role_labels = {**labels, "app.kubernetes.io/component": role}
|
|
return {
|
|
"apiVersion": "policy/v1",
|
|
"kind": "PodDisruptionBudget",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
|
|
"spec": {"minAvailable": 1, "selector": {"matchLabels": role_labels}},
|
|
}
|
|
|
|
|
|
def _ingress(
|
|
spec: InstallationSpec,
|
|
*,
|
|
name: str,
|
|
namespace: str,
|
|
labels: dict[str, str],
|
|
tls_secret_name: str,
|
|
ingress_class_name: str | None,
|
|
) -> dict[str, Any]:
|
|
public = urlsplit(spec.public_url)
|
|
ingress_spec: dict[str, Any] = {
|
|
"rules": [
|
|
{
|
|
"host": public.hostname,
|
|
"http": {
|
|
"paths": [
|
|
{
|
|
"path": "/",
|
|
"pathType": "Prefix",
|
|
"backend": {
|
|
"service": {
|
|
"name": f"{name}-web",
|
|
"port": {"name": "http"},
|
|
}
|
|
},
|
|
}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
}
|
|
if public.scheme == "https":
|
|
if not tls_secret_name:
|
|
raise ValueError("HTTPS Kubernetes export requires a TLS secret name")
|
|
ingress_spec["tls"] = [
|
|
{"hosts": [public.hostname], "secretName": tls_secret_name}
|
|
]
|
|
if ingress_class_name:
|
|
ingress_spec["ingressClassName"] = ingress_class_name
|
|
return {
|
|
"apiVersion": "networking.k8s.io/v1",
|
|
"kind": "Ingress",
|
|
"metadata": {"name": name, "namespace": namespace, "labels": labels},
|
|
"spec": ingress_spec,
|
|
}
|
|
|
|
|
|
def _http_probe(
|
|
path: str,
|
|
port: int,
|
|
*,
|
|
host: str | None = None,
|
|
) -> dict[str, Any]:
|
|
request: dict[str, Any] = {"path": path, "port": port}
|
|
if host:
|
|
request["httpHeaders"] = [{"name": "Host", "value": host}]
|
|
return {
|
|
"httpGet": request,
|
|
"initialDelaySeconds": 10,
|
|
"periodSeconds": 10,
|
|
"timeoutSeconds": 3,
|
|
"failureThreshold": 6,
|
|
}
|
|
|
|
|
|
def _resource_name(installation_id: str) -> str:
|
|
value = installation_id[:48].rstrip("-")
|
|
if not _DNS_LABEL.fullmatch(value):
|
|
raise ValueError("Installation id cannot be represented as a Kubernetes name")
|
|
return value
|
|
|
|
|
|
def _release_key(spec: InstallationSpec) -> str:
|
|
manifest_hash = str(spec.release.manifest_sha256 or "").strip().lower()
|
|
if re.fullmatch(r"[0-9a-f]{8,}", manifest_hash):
|
|
return manifest_hash[:8]
|
|
payload = "\0".join(
|
|
(
|
|
spec.release.version,
|
|
spec.release.api_image,
|
|
spec.release.web_image,
|
|
)
|
|
).encode("utf-8")
|
|
return sha256(payload).hexdigest()[:8]
|
|
|
|
|
|
def _name_with_suffix(name: str, suffix: str) -> str:
|
|
available = 63 - len(suffix) - 1
|
|
prefix = name[:available].rstrip("-")
|
|
candidate = f"{prefix}-{suffix}"
|
|
if not _DNS_LABEL.fullmatch(candidate):
|
|
raise ValueError("Kubernetes resource name cannot include the release suffix")
|
|
return candidate
|
|
|
|
|
|
def write_secret_creation_hint(
|
|
environment_path: Path,
|
|
*,
|
|
namespace: str,
|
|
secret_name: str,
|
|
) -> str:
|
|
keys = " ".join(f"--from-env-file={environment_path}" for _ in (0,))
|
|
return (
|
|
f"kubectl -n {namespace} create secret generic {secret_name} {keys} "
|
|
"--dry-run=client -o yaml | kubectl apply -f -"
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"kubernetes_secret_contract",
|
|
"render_kubernetes",
|
|
"write_secret_creation_hint",
|
|
]
|