feat: add bounded Kubernetes runtime verification
This commit is contained in:
@@ -150,8 +150,29 @@ versioning, and lifecycle controls.
|
|||||||
|
|
||||||
## Capacity
|
## Capacity
|
||||||
|
|
||||||
- Scale API replicas only within the PostgreSQL connection budget.
|
- Set `GOVOPLAN_DB_CONNECTION_LIMIT` to the PostgreSQL role's effective
|
||||||
|
connection limit. The Kubernetes export reserves
|
||||||
|
`GOVOPLAN_DB_CONNECTION_RESERVE` connections and rejects a topology whose
|
||||||
|
calculated rolling-update peak would exceed the remainder. The calculation
|
||||||
|
includes API pools, every Celery parent and prefork child, the scheduler,
|
||||||
|
migration, and one surge replica per deployment. Role-specific pool and
|
||||||
|
overflow values are emitted into each workload rather than inherited from one
|
||||||
|
unconstrained global default.
|
||||||
- Scale workers by queue, with upper bounds based on external provider limits.
|
- Scale workers by queue, with upper bounds based on external provider limits.
|
||||||
|
`GOVOPLAN_WORKER_POOLS` may contain a JSON list of exact queue owners, for
|
||||||
|
example:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{"name":"delivery","queues":["send_email","append_sent"],"replicas":2,"concurrency":2},
|
||||||
|
{"name":"platform","queues":["events","workflow","default"],"replicas":2,"concurrency":2}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Pool replica totals must equal `replicas.worker`, and the pools must cover
|
||||||
|
`CELERY_QUEUES` exactly without duplicate ownership. Each pool receives its
|
||||||
|
own Deployment, disruption budget, topology-spread selector, runtime identity,
|
||||||
|
and declared concurrency.
|
||||||
- Keep one fenced scheduler rather than load-balancing schedulers.
|
- Keep one fenced scheduler rather than load-balancing schedulers.
|
||||||
- Increase WebUI replicas for asset/proxy capacity.
|
- Increase WebUI replicas for asset/proxy capacity.
|
||||||
- Measure request latency, database query time and locks, active connections,
|
- Measure request latency, database query time and locks, active connections,
|
||||||
@@ -181,3 +202,28 @@ availability, drill replica loss, rolling replacement, session continuity, job
|
|||||||
redelivery, scheduler failover, migration exclusion, object-store outage, and a
|
redelivery, scheduler failover, migration exclusion, object-store outage, and a
|
||||||
coordinated database/object/key restore. Recovery rules and evidence are
|
coordinated database/object/key restore. Recovery rules and evidence are
|
||||||
defined in [Recovery And Rollback Guarantees](RECOVERY_AND_ROLLBACK_GUARANTEES.md).
|
defined in [Recovery And Rollback Guarantees](RECOVERY_AND_ROLLBACK_GUARANTEES.md).
|
||||||
|
|
||||||
|
## Live Multi-Host Evidence
|
||||||
|
|
||||||
|
After deploying a pinned release on at least two Kubernetes nodes, create an API
|
||||||
|
key with Ops read scope and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export GOVOPLAN_OPS_API_KEY='...'
|
||||||
|
python tools/deployment/govoplan-deploy.py verify-kubernetes \
|
||||||
|
--directory /srv/govoplan/installation \
|
||||||
|
--namespace govoplan
|
||||||
|
```
|
||||||
|
|
||||||
|
The command fails unless API and WebUI pods are ready on at least two nodes,
|
||||||
|
all rendered deployments are available, Ops reports a consistent release and
|
||||||
|
module composition, every declared worker queue is served, and the calculated
|
||||||
|
database peak remains below its budget. It writes a private, sanitized JSON
|
||||||
|
record under the installation evidence directory and never retains the API key.
|
||||||
|
|
||||||
|
Use `--exercise-api-pod-loss` in an approved drill window to delete one API pod,
|
||||||
|
observe the public readiness path continuously, and record its replacement.
|
||||||
|
This proves the bounded stateless-node-loss slice only. Session continuity,
|
||||||
|
accepted-job redelivery, state-service failover, and coordinated restore remain
|
||||||
|
separate target exercises whose signed evidence is governed by
|
||||||
|
`docs/TARGET_MATURITY_EVIDENCE_RUNBOOK.md` and GovOPlaN #37.
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ from govoplan_deploy.bundle import ( # noqa: E402
|
|||||||
)
|
)
|
||||||
from govoplan_deploy.cli import _receipt_uses_direct_web_port, main # noqa: E402
|
from govoplan_deploy.cli import _receipt_uses_direct_web_port, main # noqa: E402
|
||||||
import govoplan_deploy.cli as deployment_cli # noqa: E402
|
import govoplan_deploy.cli as deployment_cli # noqa: E402
|
||||||
|
from govoplan_deploy.cluster_evidence import ( # noqa: E402
|
||||||
|
collect_kubernetes_evidence,
|
||||||
|
)
|
||||||
from govoplan_deploy.kubernetes import render_kubernetes # noqa: E402
|
from govoplan_deploy.kubernetes import render_kubernetes # noqa: E402
|
||||||
from govoplan_deploy.model import ( # noqa: E402
|
from govoplan_deploy.model import ( # noqa: E402
|
||||||
SpecError,
|
SpecError,
|
||||||
@@ -55,7 +58,104 @@ def run_cli(arguments: list[str]) -> tuple[int, str, str]:
|
|||||||
return result, stdout.getvalue(), stderr.getvalue()
|
return result, stdout.getvalue(), stderr.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _kubernetes_test_pod(name: str, component: str, node: str) -> dict:
|
||||||
|
return {
|
||||||
|
"metadata": {
|
||||||
|
"name": name,
|
||||||
|
"uid": f"uid-{name}",
|
||||||
|
"labels": {"app.kubernetes.io/component": component},
|
||||||
|
},
|
||||||
|
"spec": {"nodeName": node},
|
||||||
|
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _kubernetes_test_deployment(component: str, replicas: int) -> dict:
|
||||||
|
return {
|
||||||
|
"metadata": {
|
||||||
|
"name": f"govoplan-cluster-{component}",
|
||||||
|
"labels": {"app.kubernetes.io/component": component},
|
||||||
|
},
|
||||||
|
"spec": {"replicas": replicas},
|
||||||
|
"status": {"availableReplicas": replicas, "updatedReplicas": replicas},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class DeploymentInstallerTests(unittest.TestCase):
|
class DeploymentInstallerTests(unittest.TestCase):
|
||||||
|
def test_kubernetes_evidence_requires_two_node_spread_and_safe_runtime(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
resources = {
|
||||||
|
"nodes": {
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"metadata": {"name": name},
|
||||||
|
"spec": {},
|
||||||
|
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
|
||||||
|
}
|
||||||
|
for name in ("node-a", "node-b")
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"pods": {
|
||||||
|
"items": [
|
||||||
|
_kubernetes_test_pod("api-a", "api", "node-a"),
|
||||||
|
_kubernetes_test_pod("api-b", "api", "node-b"),
|
||||||
|
_kubernetes_test_pod("web-a", "web", "node-a"),
|
||||||
|
_kubernetes_test_pod("web-b", "web", "node-b"),
|
||||||
|
_kubernetes_test_pod("worker-a", "worker", "node-a"),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"deployments": {
|
||||||
|
"items": [
|
||||||
|
_kubernetes_test_deployment("api", 2),
|
||||||
|
_kubernetes_test_deployment("web", 2),
|
||||||
|
_kubernetes_test_deployment("worker", 1),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(arguments):
|
||||||
|
return resources[
|
||||||
|
"nodes"
|
||||||
|
if "nodes" in arguments
|
||||||
|
else "deployments"
|
||||||
|
if "deployments" in arguments
|
||||||
|
else "pods"
|
||||||
|
]
|
||||||
|
|
||||||
|
evidence = collect_kubernetes_evidence(
|
||||||
|
installation_id="govoplan-cluster",
|
||||||
|
namespace="govoplan",
|
||||||
|
ops_url="https://govoplan.example.test/api/v1/ops/status",
|
||||||
|
api_key="not-retained",
|
||||||
|
command_runner=run,
|
||||||
|
json_fetcher=lambda _url, _key: {
|
||||||
|
"readiness": {"ready": True},
|
||||||
|
"runtime_cluster": {
|
||||||
|
"expected": {"api": 2, "worker": 1},
|
||||||
|
"active": {"api": 2, "worker": 1},
|
||||||
|
"composition": {"skewed": False},
|
||||||
|
"software_versions": {"skewed": False},
|
||||||
|
"queues": {"missing": []},
|
||||||
|
},
|
||||||
|
"checks": [
|
||||||
|
{
|
||||||
|
"id": "database_capacity",
|
||||||
|
"state": "ok",
|
||||||
|
"detail": "Within budget",
|
||||||
|
"metrics": {"peak": 30, "available": 90},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("passed", evidence["result"]["state"])
|
||||||
|
self.assertNotIn("not-retained", json.dumps(evidence))
|
||||||
|
self.assertEqual(
|
||||||
|
["node-a", "node-b"],
|
||||||
|
evidence["snapshot"]["ready_node_names"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_pre_migration_failure_restores_checksum_verified_applied_bundle(
|
def test_pre_migration_failure_restores_checksum_verified_applied_bundle(
|
||||||
self,
|
self,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -212,6 +312,7 @@ class DeploymentInstallerTests(unittest.TestCase):
|
|||||||
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
||||||
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
||||||
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -257,6 +358,103 @@ class DeploymentInstallerTests(unittest.TestCase):
|
|||||||
"containers"
|
"containers"
|
||||||
][0]["readinessProbe"]["httpGet"]["httpHeaders"],
|
][0]["readinessProbe"]["httpGet"]["httpHeaders"],
|
||||||
)
|
)
|
||||||
|
worker_command = deployments["govoplan-cluster-worker"]["spec"]["template"][
|
||||||
|
"spec"
|
||||||
|
]["containers"][0]["command"]
|
||||||
|
self.assertIn("--concurrency", worker_command)
|
||||||
|
self.assertEqual(
|
||||||
|
"62",
|
||||||
|
manifest["metadata"]["annotations"][
|
||||||
|
"govoplan.add-ideas.de/database-connection-peak"
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_kubernetes_export_splits_worker_queues_and_rejects_capacity_overrun(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
spec = default_spec(
|
||||||
|
installation_id="govoplan-cluster",
|
||||||
|
postgres_mode="external",
|
||||||
|
redis_mode="external",
|
||||||
|
storage_mode="s3",
|
||||||
|
api_replicas=2,
|
||||||
|
web_replicas=2,
|
||||||
|
worker_replicas=3,
|
||||||
|
api_image="registry.example.test/govoplan-api@sha256:" + "a" * 64,
|
||||||
|
web_image="registry.example.test/govoplan-web@sha256:" + "b" * 64,
|
||||||
|
)
|
||||||
|
environment = initial_secrets(
|
||||||
|
spec,
|
||||||
|
supplied={
|
||||||
|
"DATABASE_URL": "postgresql+psycopg://user:db-secret@postgres.example.test/govoplan",
|
||||||
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:db-secret@postgres.example.test/govoplan",
|
||||||
|
"REDIS_URL": "rediss://:redis-secret@redis.example.test/0",
|
||||||
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
||||||
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
||||||
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
||||||
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
||||||
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
queues = environment["CELERY_QUEUES"].split(",")
|
||||||
|
environment["GOVOPLAN_WORKER_POOLS"] = json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "delivery",
|
||||||
|
"queues": queues[:3],
|
||||||
|
"replicas": 2,
|
||||||
|
"concurrency": 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "platform",
|
||||||
|
"queues": queues[3:],
|
||||||
|
"replicas": 1,
|
||||||
|
"concurrency": 1,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = render_kubernetes(spec, environment)
|
||||||
|
workers = [
|
||||||
|
item
|
||||||
|
for item in manifest["items"]
|
||||||
|
if item["kind"] == "Deployment"
|
||||||
|
and item["metadata"]["labels"].get("app.kubernetes.io/component")
|
||||||
|
== "worker"
|
||||||
|
]
|
||||||
|
|
||||||
|
self.assertEqual(2, len(workers))
|
||||||
|
self.assertEqual(
|
||||||
|
{"delivery", "platform"},
|
||||||
|
{
|
||||||
|
item["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"]
|
||||||
|
for item in workers
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for deployment in workers:
|
||||||
|
container = deployment["spec"]["template"]["spec"]["containers"][0]
|
||||||
|
explicit_environment = {
|
||||||
|
item["name"]: item.get("value")
|
||||||
|
for item in container["env"]
|
||||||
|
if "value" in item
|
||||||
|
}
|
||||||
|
self.assertEqual(
|
||||||
|
deployment["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"],
|
||||||
|
explicit_environment["GOVOPLAN_WORKER_POOL"],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
set(
|
||||||
|
container["command"][
|
||||||
|
container["command"].index("--queues") + 1
|
||||||
|
].split(",")
|
||||||
|
),
|
||||||
|
set(explicit_environment["CELERY_QUEUES"].split(",")),
|
||||||
|
)
|
||||||
|
|
||||||
|
environment["GOVOPLAN_DB_CONNECTION_LIMIT"] = "40"
|
||||||
|
with self.assertRaisesRegex(ValueError, "connection peak"):
|
||||||
|
render_kubernetes(spec, environment)
|
||||||
|
|
||||||
def test_kubernetes_export_rejects_mutable_release_images(self) -> None:
|
def test_kubernetes_export_rejects_mutable_release_images(self) -> None:
|
||||||
spec = default_spec(
|
spec = default_spec(
|
||||||
|
|||||||
@@ -40,6 +40,18 @@ RUNTIME_ENV_KEYS = (
|
|||||||
"ENABLED_MODULES",
|
"ENABLED_MODULES",
|
||||||
"CELERY_ENABLED",
|
"CELERY_ENABLED",
|
||||||
"CELERY_QUEUES",
|
"CELERY_QUEUES",
|
||||||
|
"CELERY_WORKER_CONCURRENCY",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
||||||
|
"GOVOPLAN_API_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_API_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_WORKER_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_WORKER_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_MIGRATION_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_WORKER_POOLS",
|
||||||
"REDIS_URL",
|
"REDIS_URL",
|
||||||
"CORS_ORIGINS",
|
"CORS_ORIGINS",
|
||||||
"GOVOPLAN_TRUSTED_HOSTS",
|
"GOVOPLAN_TRUSTED_HOSTS",
|
||||||
@@ -193,6 +205,46 @@ def reconcile_runtime_environment(
|
|||||||
"CELERY_QUEUES": (
|
"CELERY_QUEUES": (
|
||||||
"send_email,append_sent,notifications,mail,calendar,dataflow,workflow,postbox,events,idm,default"
|
"send_email,append_sent,notifications,mail,calendar,dataflow,workflow,postbox,events,idm,default"
|
||||||
),
|
),
|
||||||
|
"CELERY_WORKER_CONCURRENCY": values.get(
|
||||||
|
"CELERY_WORKER_CONCURRENCY",
|
||||||
|
"2",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE": values.get(
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
||||||
|
"10",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_API_DB_POOL_SIZE": values.get(
|
||||||
|
"GOVOPLAN_API_DB_POOL_SIZE",
|
||||||
|
"5",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_API_DB_MAX_OVERFLOW": values.get(
|
||||||
|
"GOVOPLAN_API_DB_MAX_OVERFLOW",
|
||||||
|
"2",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_WORKER_DB_POOL_SIZE": values.get(
|
||||||
|
"GOVOPLAN_WORKER_DB_POOL_SIZE",
|
||||||
|
"1",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_WORKER_DB_MAX_OVERFLOW": values.get(
|
||||||
|
"GOVOPLAN_WORKER_DB_MAX_OVERFLOW",
|
||||||
|
"1",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_POOL_SIZE": values.get(
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_POOL_SIZE",
|
||||||
|
"1",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW": values.get(
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW",
|
||||||
|
"0",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_MIGRATION_DB_POOL_SIZE": values.get(
|
||||||
|
"GOVOPLAN_MIGRATION_DB_POOL_SIZE",
|
||||||
|
"2",
|
||||||
|
),
|
||||||
|
"GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW": values.get(
|
||||||
|
"GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW",
|
||||||
|
"0",
|
||||||
|
),
|
||||||
"CORS_ORIGINS": spec.public_url,
|
"CORS_ORIGINS": spec.public_url,
|
||||||
"GOVOPLAN_TRUSTED_HOSTS": public.hostname or "",
|
"GOVOPLAN_TRUSTED_HOSTS": public.hostname or "",
|
||||||
"FORWARDED_ALLOW_IPS": spec.network_subnet,
|
"FORWARDED_ALLOW_IPS": spec.network_subnet,
|
||||||
@@ -212,6 +264,8 @@ def reconcile_runtime_environment(
|
|||||||
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true"
|
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true"
|
||||||
else:
|
else:
|
||||||
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false"
|
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false"
|
||||||
|
if postgres.mode == "managed":
|
||||||
|
values.setdefault("GOVOPLAN_DB_CONNECTION_LIMIT", "100")
|
||||||
|
|
||||||
storage = spec.components.storage
|
storage = spec.components.storage
|
||||||
if storage.mode == "local":
|
if storage.mode == "local":
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ from .bundle import (
|
|||||||
service_names,
|
service_names,
|
||||||
write_env,
|
write_env,
|
||||||
)
|
)
|
||||||
|
from .cluster_evidence import collect_kubernetes_evidence
|
||||||
from .model import (
|
from .model import (
|
||||||
ComponentConfig,
|
ComponentConfig,
|
||||||
DEFAULT_GARAGE_IMAGE,
|
DEFAULT_GARAGE_IMAGE,
|
||||||
@@ -141,6 +142,37 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
help="Output JSON path; defaults to <directory>/kubernetes.json.",
|
help="Output JSON path; defaults to <directory>/kubernetes.json.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
verify_kubernetes = subparsers.add_parser(
|
||||||
|
"verify-kubernetes",
|
||||||
|
help="Collect sanitized readiness evidence from a live multi-host profile.",
|
||||||
|
)
|
||||||
|
_directory_argument(verify_kubernetes)
|
||||||
|
verify_kubernetes.add_argument("--namespace", default="govoplan")
|
||||||
|
verify_kubernetes.add_argument(
|
||||||
|
"--ops-url",
|
||||||
|
help="Ops status URL; defaults to <public-url>/api/v1/ops/status.",
|
||||||
|
)
|
||||||
|
verify_kubernetes.add_argument(
|
||||||
|
"--api-key-env",
|
||||||
|
default="GOVOPLAN_OPS_API_KEY",
|
||||||
|
help="Environment variable containing an API key with Ops read scope.",
|
||||||
|
)
|
||||||
|
verify_kubernetes.add_argument(
|
||||||
|
"--exercise-api-pod-loss",
|
||||||
|
action="store_true",
|
||||||
|
help="Delete one ready API pod and prove health-aware replacement.",
|
||||||
|
)
|
||||||
|
verify_kubernetes.add_argument(
|
||||||
|
"--timeout-seconds",
|
||||||
|
type=float,
|
||||||
|
default=180.0,
|
||||||
|
)
|
||||||
|
verify_kubernetes.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
help="Private evidence output path.",
|
||||||
|
)
|
||||||
|
|
||||||
operations = subparsers.add_parser(
|
operations = subparsers.add_parser(
|
||||||
"operations",
|
"operations",
|
||||||
help="List durable deployment and recovery operations.",
|
help="List durable deployment and recovery operations.",
|
||||||
@@ -282,6 +314,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
return _status(args)
|
return _status(args)
|
||||||
if args.command == "render-kubernetes":
|
if args.command == "render-kubernetes":
|
||||||
return _render_kubernetes(args)
|
return _render_kubernetes(args)
|
||||||
|
if args.command == "verify-kubernetes":
|
||||||
|
return _verify_kubernetes(args)
|
||||||
if args.command == "operations":
|
if args.command == "operations":
|
||||||
return _operations(args)
|
return _operations(args)
|
||||||
if args.command == "recover":
|
if args.command == "recover":
|
||||||
@@ -598,9 +632,7 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
|
|||||||
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
|
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
|
||||||
atomic_write(output, canonical_json(manifest), mode=0o600)
|
atomic_write(output, canonical_json(manifest), mode=0o600)
|
||||||
print(f"Wrote stateless Kubernetes runtime manifest to {output}")
|
print(f"Wrote stateless Kubernetes runtime manifest to {output}")
|
||||||
print(
|
print("Required Secret keys: " + ", ".join(kubernetes_secret_contract()))
|
||||||
"Required Secret keys: " + ", ".join(kubernetes_secret_contract())
|
|
||||||
)
|
|
||||||
print(
|
print(
|
||||||
write_secret_creation_hint(
|
write_secret_creation_hint(
|
||||||
paths.env,
|
paths.env,
|
||||||
@@ -611,6 +643,41 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_kubernetes(args: argparse.Namespace) -> int:
|
||||||
|
paths = bundle_paths(args.directory)
|
||||||
|
spec = load_spec(paths.spec)
|
||||||
|
api_key = str(os.environ.get(args.api_key_env) or "").strip()
|
||||||
|
if not api_key:
|
||||||
|
raise ValueError(
|
||||||
|
f"{args.api_key_env} must contain an API key with Ops read scope"
|
||||||
|
)
|
||||||
|
ops_url = args.ops_url or (spec.public_url.rstrip("/") + "/api/v1/ops/status")
|
||||||
|
evidence = collect_kubernetes_evidence(
|
||||||
|
installation_id=spec.installation_id,
|
||||||
|
namespace=args.namespace,
|
||||||
|
ops_url=ops_url,
|
||||||
|
api_key=api_key,
|
||||||
|
exercise_api_pod_loss=args.exercise_api_pod_loss,
|
||||||
|
timeout_seconds=args.timeout_seconds,
|
||||||
|
)
|
||||||
|
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
output = (
|
||||||
|
(args.output or paths.root / "evidence" / f"kubernetes-{timestamp}.json")
|
||||||
|
.expanduser()
|
||||||
|
.resolve()
|
||||||
|
)
|
||||||
|
ensure_private_directory(output.parent)
|
||||||
|
atomic_write(output, canonical_json(evidence), mode=0o600)
|
||||||
|
print(f"Wrote Kubernetes evidence to {output}")
|
||||||
|
if evidence["result"]["state"] != "passed":
|
||||||
|
print(
|
||||||
|
"Failed checks: " + ", ".join(evidence["result"]["failed_checks"]),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _operations(args: argparse.Namespace) -> int:
|
def _operations(args: argparse.Namespace) -> int:
|
||||||
paths = bundle_paths(args.directory)
|
paths = bundle_paths(args.directory)
|
||||||
payload = list_operations(paths)
|
payload = list_operations(paths)
|
||||||
|
|||||||
@@ -0,0 +1,380 @@
|
|||||||
|
"""Collect sanitized evidence for the Kubernetes multi-host profile."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
|
||||||
|
JsonObject = dict[str, Any]
|
||||||
|
CommandRunner = Callable[[Sequence[str]], JsonObject]
|
||||||
|
JsonFetcher = Callable[[str, str], JsonObject]
|
||||||
|
|
||||||
|
|
||||||
|
def collect_kubernetes_evidence(
|
||||||
|
*,
|
||||||
|
installation_id: str,
|
||||||
|
namespace: str,
|
||||||
|
ops_url: str,
|
||||||
|
api_key: str,
|
||||||
|
exercise_api_pod_loss: bool = False,
|
||||||
|
timeout_seconds: float = 180.0,
|
||||||
|
command_runner: CommandRunner | None = None,
|
||||||
|
json_fetcher: JsonFetcher | None = None,
|
||||||
|
) -> JsonObject:
|
||||||
|
"""Inspect a live cluster and optionally exercise one API pod replacement."""
|
||||||
|
|
||||||
|
run_json = command_runner or _kubectl_json
|
||||||
|
fetch_json = json_fetcher or _fetch_json
|
||||||
|
nodes = run_json(("get", "nodes", "-o", "json"))
|
||||||
|
pods = run_json(
|
||||||
|
(
|
||||||
|
"-n",
|
||||||
|
namespace,
|
||||||
|
"get",
|
||||||
|
"pods",
|
||||||
|
"-l",
|
||||||
|
f"app.kubernetes.io/instance={installation_id}",
|
||||||
|
"-o",
|
||||||
|
"json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
deployments = run_json(
|
||||||
|
(
|
||||||
|
"-n",
|
||||||
|
namespace,
|
||||||
|
"get",
|
||||||
|
"deployments",
|
||||||
|
"-l",
|
||||||
|
f"app.kubernetes.io/instance={installation_id}",
|
||||||
|
"-o",
|
||||||
|
"json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ops = fetch_json(ops_url, api_key)
|
||||||
|
snapshot = _evaluate_snapshot(
|
||||||
|
installation_id=installation_id,
|
||||||
|
namespace=namespace,
|
||||||
|
nodes=nodes,
|
||||||
|
pods=pods,
|
||||||
|
deployments=deployments,
|
||||||
|
ops=ops,
|
||||||
|
)
|
||||||
|
replacement: JsonObject | None = None
|
||||||
|
if exercise_api_pod_loss:
|
||||||
|
replacement = _exercise_api_pod_loss(
|
||||||
|
installation_id=installation_id,
|
||||||
|
namespace=namespace,
|
||||||
|
ops_url=ops_url,
|
||||||
|
api_key=api_key,
|
||||||
|
initial_pods=pods,
|
||||||
|
timeout_seconds=timeout_seconds,
|
||||||
|
run_json=run_json,
|
||||||
|
fetch_json=fetch_json,
|
||||||
|
)
|
||||||
|
evidence = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"evidence_kind": "govoplan.kubernetes-multi-host",
|
||||||
|
"collected_at": datetime.now(UTC).isoformat(),
|
||||||
|
"installation_id": installation_id,
|
||||||
|
"namespace": namespace,
|
||||||
|
"snapshot": snapshot,
|
||||||
|
"api_pod_loss": replacement,
|
||||||
|
}
|
||||||
|
failures = [
|
||||||
|
check["id"] for check in snapshot["checks"] if check["state"] != "passed"
|
||||||
|
]
|
||||||
|
if replacement and replacement["state"] != "passed":
|
||||||
|
failures.append("api_pod_loss")
|
||||||
|
evidence["result"] = {
|
||||||
|
"state": "passed" if not failures else "failed",
|
||||||
|
"failed_checks": failures,
|
||||||
|
}
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate_snapshot(
|
||||||
|
*,
|
||||||
|
installation_id: str,
|
||||||
|
namespace: str,
|
||||||
|
nodes: Mapping[str, Any],
|
||||||
|
pods: Mapping[str, Any],
|
||||||
|
deployments: Mapping[str, Any],
|
||||||
|
ops: Mapping[str, Any],
|
||||||
|
) -> JsonObject:
|
||||||
|
ready_nodes = [
|
||||||
|
item
|
||||||
|
for item in _items(nodes)
|
||||||
|
if not bool(item.get("spec", {}).get("unschedulable"))
|
||||||
|
and _condition(item, "Ready") == "True"
|
||||||
|
]
|
||||||
|
pod_rows = [_pod_summary(item) for item in _items(pods)]
|
||||||
|
deployment_rows = [_deployment_summary(item) for item in _items(deployments)]
|
||||||
|
ready_api = [
|
||||||
|
item for item in pod_rows if item["component"] == "api" and item["ready"]
|
||||||
|
]
|
||||||
|
ready_web = [
|
||||||
|
item for item in pod_rows if item["component"] == "web" and item["ready"]
|
||||||
|
]
|
||||||
|
runtime = ops.get("runtime_cluster")
|
||||||
|
runtime = runtime if isinstance(runtime, Mapping) else {}
|
||||||
|
checks = ops.get("checks")
|
||||||
|
check_rows = checks if isinstance(checks, list) else []
|
||||||
|
check_by_id = {
|
||||||
|
str(item.get("id")): item for item in check_rows if isinstance(item, Mapping)
|
||||||
|
}
|
||||||
|
ops_readiness = ops.get("readiness")
|
||||||
|
ops_readiness = ops_readiness if isinstance(ops_readiness, Mapping) else {}
|
||||||
|
snapshot_checks = [
|
||||||
|
_check(
|
||||||
|
"ready_nodes",
|
||||||
|
len(ready_nodes) >= 2,
|
||||||
|
f"{len(ready_nodes)} ready schedulable node(s)",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"api_node_spread",
|
||||||
|
len(ready_api) >= 2 and len({item["node"] for item in ready_api}) >= 2,
|
||||||
|
f"{len(ready_api)} ready API pod(s) on {len({item['node'] for item in ready_api})} node(s)",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"web_node_spread",
|
||||||
|
len(ready_web) >= 2 and len({item["node"] for item in ready_web}) >= 2,
|
||||||
|
f"{len(ready_web)} ready WebUI pod(s) on {len({item['node'] for item in ready_web})} node(s)",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"deployment_availability",
|
||||||
|
bool(deployment_rows)
|
||||||
|
and all(item["available"] >= item["desired"] for item in deployment_rows),
|
||||||
|
f"{len(deployment_rows)} deployment(s) inspected",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"ops_readiness",
|
||||||
|
bool(ops_readiness.get("ready")),
|
||||||
|
str(ops_readiness.get("detail") or "Ops readiness response inspected"),
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"runtime_composition",
|
||||||
|
not bool(runtime.get("composition", {}).get("skewed")),
|
||||||
|
"Active runtime composition matches the loaded module graph",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"runtime_versions",
|
||||||
|
not bool(runtime.get("software_versions", {}).get("skewed")),
|
||||||
|
"Active runtime software versions are consistent",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"worker_queue_coverage",
|
||||||
|
not bool(runtime.get("queues", {}).get("missing")),
|
||||||
|
"Every configured queue has an active worker",
|
||||||
|
),
|
||||||
|
_check(
|
||||||
|
"database_connection_budget",
|
||||||
|
str(check_by_id.get("database_capacity", {}).get("state")) == "ok",
|
||||||
|
str(
|
||||||
|
check_by_id.get("database_capacity", {}).get("detail")
|
||||||
|
or "Database capacity check is unavailable"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"installation_id": installation_id,
|
||||||
|
"namespace": namespace,
|
||||||
|
"ready_node_names": sorted(_name(item) for item in ready_nodes),
|
||||||
|
"pods": sorted(pod_rows, key=lambda item: item["name"]),
|
||||||
|
"deployments": sorted(deployment_rows, key=lambda item: item["name"]),
|
||||||
|
"runtime": {
|
||||||
|
"expected": dict(runtime.get("expected") or {}),
|
||||||
|
"active": dict(runtime.get("active") or {}),
|
||||||
|
"composition": dict(runtime.get("composition") or {}),
|
||||||
|
"software_versions": dict(runtime.get("software_versions") or {}),
|
||||||
|
"queues": dict(runtime.get("queues") or {}),
|
||||||
|
},
|
||||||
|
"database_capacity": dict(
|
||||||
|
check_by_id.get("database_capacity", {}).get("metrics") or {}
|
||||||
|
),
|
||||||
|
"checks": snapshot_checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _exercise_api_pod_loss(
|
||||||
|
*,
|
||||||
|
installation_id: str,
|
||||||
|
namespace: str,
|
||||||
|
ops_url: str,
|
||||||
|
api_key: str,
|
||||||
|
initial_pods: Mapping[str, Any],
|
||||||
|
timeout_seconds: float,
|
||||||
|
run_json: CommandRunner,
|
||||||
|
fetch_json: JsonFetcher,
|
||||||
|
) -> JsonObject:
|
||||||
|
candidates = [
|
||||||
|
_pod_summary(item)
|
||||||
|
for item in _items(initial_pods)
|
||||||
|
if item.get("metadata", {}).get("labels", {}).get("app.kubernetes.io/component")
|
||||||
|
== "api"
|
||||||
|
and _condition(item, "Ready") == "True"
|
||||||
|
]
|
||||||
|
if len(candidates) < 2:
|
||||||
|
return {
|
||||||
|
"state": "failed",
|
||||||
|
"detail": "At least two ready API pods are required for the loss drill.",
|
||||||
|
}
|
||||||
|
victim = sorted(candidates, key=lambda item: item["name"])[0]
|
||||||
|
initial_uids = {item["uid"] for item in candidates}
|
||||||
|
desired_ready = len(candidates)
|
||||||
|
run_json(
|
||||||
|
(
|
||||||
|
"-n",
|
||||||
|
namespace,
|
||||||
|
"delete",
|
||||||
|
"pod",
|
||||||
|
victim["name"],
|
||||||
|
"--wait=false",
|
||||||
|
"-o",
|
||||||
|
"json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
health_failures = 0
|
||||||
|
replacement: JsonObject | None = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
current_ops = fetch_json(ops_url, api_key)
|
||||||
|
if not bool(current_ops.get("readiness", {}).get("ready")):
|
||||||
|
health_failures += 1
|
||||||
|
except (OSError, ValueError):
|
||||||
|
health_failures += 1
|
||||||
|
current = run_json(
|
||||||
|
(
|
||||||
|
"-n",
|
||||||
|
namespace,
|
||||||
|
"get",
|
||||||
|
"pods",
|
||||||
|
"-l",
|
||||||
|
f"app.kubernetes.io/instance={installation_id},app.kubernetes.io/component=api",
|
||||||
|
"-o",
|
||||||
|
"json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ready = [
|
||||||
|
_pod_summary(item)
|
||||||
|
for item in _items(current)
|
||||||
|
if _condition(item, "Ready") == "True"
|
||||||
|
]
|
||||||
|
new_pods = [item for item in ready if item["uid"] not in initial_uids]
|
||||||
|
if len(ready) >= desired_ready and new_pods:
|
||||||
|
replacement = sorted(new_pods, key=lambda item: item["name"])[-1]
|
||||||
|
break
|
||||||
|
time.sleep(2.0)
|
||||||
|
passed = replacement is not None and health_failures == 0
|
||||||
|
return {
|
||||||
|
"state": "passed" if passed else "failed",
|
||||||
|
"victim": victim,
|
||||||
|
"replacement": replacement,
|
||||||
|
"readiness_failures": health_failures,
|
||||||
|
"detail": (
|
||||||
|
"API pod was replaced without an observed readiness outage."
|
||||||
|
if passed
|
||||||
|
else "API replacement timed out or readiness failed during replacement."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _kubectl_json(arguments: Sequence[str]) -> JsonObject:
|
||||||
|
kubectl = shutil.which("kubectl")
|
||||||
|
if kubectl is None:
|
||||||
|
raise ValueError("kubectl is required for Kubernetes evidence collection")
|
||||||
|
result = subprocess.run(
|
||||||
|
(kubectl, *arguments),
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
if result.returncode:
|
||||||
|
detail = result.stderr.strip() or result.stdout.strip()
|
||||||
|
raise ValueError(f"kubectl failed: {detail}")
|
||||||
|
try:
|
||||||
|
payload = json.loads(result.stdout)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ValueError("kubectl did not return JSON") from exc
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError("kubectl returned a non-object JSON payload")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_json(url: str, api_key: str) -> JsonObject:
|
||||||
|
request = Request(
|
||||||
|
url,
|
||||||
|
headers={"Accept": "application/json", "X-API-Key": api_key},
|
||||||
|
)
|
||||||
|
with urlopen(request, timeout=15) as response:
|
||||||
|
payload = json.load(response)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError("Ops API returned a non-object JSON payload")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _items(payload: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||||
|
items = payload.get("items")
|
||||||
|
if not isinstance(items, list):
|
||||||
|
raise ValueError("Kubernetes list response has no items")
|
||||||
|
return [item for item in items if isinstance(item, Mapping)]
|
||||||
|
|
||||||
|
|
||||||
|
def _condition(item: Mapping[str, Any], condition_type: str) -> str | None:
|
||||||
|
conditions = item.get("status", {}).get("conditions", [])
|
||||||
|
for condition in conditions if isinstance(conditions, list) else []:
|
||||||
|
if isinstance(condition, Mapping) and condition.get("type") == condition_type:
|
||||||
|
return str(condition.get("status"))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _name(item: Mapping[str, Any]) -> str:
|
||||||
|
return str(item.get("metadata", {}).get("name") or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _pod_summary(item: Mapping[str, Any]) -> JsonObject:
|
||||||
|
metadata = item.get("metadata", {})
|
||||||
|
labels = metadata.get("labels", {})
|
||||||
|
return {
|
||||||
|
"name": _name(item),
|
||||||
|
"uid": str(metadata.get("uid") or ""),
|
||||||
|
"component": str(labels.get("app.kubernetes.io/component") or ""),
|
||||||
|
"worker_pool": labels.get("govoplan.add-ideas.de/worker-pool"),
|
||||||
|
"node": str(item.get("spec", {}).get("nodeName") or ""),
|
||||||
|
"ready": _condition(item, "Ready") == "True",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _deployment_summary(item: Mapping[str, Any]) -> JsonObject:
|
||||||
|
status = item.get("status", {})
|
||||||
|
return {
|
||||||
|
"name": _name(item),
|
||||||
|
"component": str(
|
||||||
|
item.get("metadata", {})
|
||||||
|
.get("labels", {})
|
||||||
|
.get("app.kubernetes.io/component")
|
||||||
|
or ""
|
||||||
|
),
|
||||||
|
"desired": int(item.get("spec", {}).get("replicas") or 0),
|
||||||
|
"available": int(status.get("availableReplicas") or 0),
|
||||||
|
"updated": int(status.get("updatedReplicas") or 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check(check_id: str, passed: bool, detail: str) -> JsonObject:
|
||||||
|
return {
|
||||||
|
"id": check_id,
|
||||||
|
"state": "passed" if passed else "failed",
|
||||||
|
"detail": detail,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["collect_kubernetes_evidence"]
|
||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
from hashlib import sha256
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
@@ -26,6 +28,17 @@ _CONFIG_KEYS = (
|
|||||||
"ENABLED_MODULES",
|
"ENABLED_MODULES",
|
||||||
"CELERY_ENABLED",
|
"CELERY_ENABLED",
|
||||||
"CELERY_QUEUES",
|
"CELERY_QUEUES",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
||||||
|
"GOVOPLAN_API_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_API_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_WORKER_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_WORKER_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_SCHEDULER_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_MIGRATION_DB_POOL_SIZE",
|
||||||
|
"GOVOPLAN_MIGRATION_DB_MAX_OVERFLOW",
|
||||||
|
"GOVOPLAN_WORKER_POOLS",
|
||||||
"CORS_ORIGINS",
|
"CORS_ORIGINS",
|
||||||
"GOVOPLAN_TRUSTED_HOSTS",
|
"GOVOPLAN_TRUSTED_HOSTS",
|
||||||
"FORWARDED_ALLOW_IPS",
|
"FORWARDED_ALLOW_IPS",
|
||||||
@@ -43,6 +56,15 @@ _CONFIG_KEYS = (
|
|||||||
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
||||||
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
||||||
)
|
)
|
||||||
|
_QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _WorkerPool:
|
||||||
|
name: str
|
||||||
|
queues: tuple[str, ...]
|
||||||
|
replicas: int
|
||||||
|
concurrency: int
|
||||||
|
|
||||||
|
|
||||||
def render_kubernetes(
|
def render_kubernetes(
|
||||||
@@ -57,6 +79,8 @@ def render_kubernetes(
|
|||||||
"""Render runtime roles only; shared state services stay externally managed."""
|
"""Render runtime roles only; shared state services stay externally managed."""
|
||||||
|
|
||||||
_validate_cluster_profile(spec, environment, namespace, secret_name)
|
_validate_cluster_profile(spec, environment, namespace, secret_name)
|
||||||
|
worker_pools = _worker_pools(spec, environment)
|
||||||
|
database_capacity = _database_capacity(spec, environment, worker_pools)
|
||||||
name = _resource_name(spec.installation_id)
|
name = _resource_name(spec.installation_id)
|
||||||
public_host = urlsplit(spec.public_url).hostname or "localhost"
|
public_host = urlsplit(spec.public_url).hostname or "localhost"
|
||||||
labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
|
labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
|
||||||
@@ -75,6 +99,8 @@ def render_kubernetes(
|
|||||||
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
|
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
|
||||||
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
|
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
|
||||||
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
|
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
|
||||||
|
"GOVOPLAN_DB_CONNECTION_PEAK": str(database_capacity["peak"]),
|
||||||
|
"GOVOPLAN_DB_CONNECTION_AVAILABLE": str(database_capacity["available"]),
|
||||||
"FILE_STORAGE_BACKEND": "s3",
|
"FILE_STORAGE_BACKEND": "s3",
|
||||||
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
|
||||||
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
|
||||||
@@ -130,6 +156,7 @@ def render_kubernetes(
|
|||||||
readiness_path="/health/ready",
|
readiness_path="/health/ready",
|
||||||
liveness_path="/health",
|
liveness_path="/health",
|
||||||
probe_host=public_host,
|
probe_host=public_host,
|
||||||
|
extra_environment=_role_database_environment(environment, "API"),
|
||||||
),
|
),
|
||||||
_service(
|
_service(
|
||||||
name=f"{name}-api",
|
name=f"{name}-api",
|
||||||
@@ -168,16 +195,25 @@ def render_kubernetes(
|
|||||||
config_name=config_name,
|
config_name=config_name,
|
||||||
secret_name=secret_name,
|
secret_name=secret_name,
|
||||||
service_account=service_account,
|
service_account=service_account,
|
||||||
|
database_environment=_role_database_environment(
|
||||||
|
environment,
|
||||||
|
"MIGRATION",
|
||||||
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
if spec.replicas.worker:
|
for pool in worker_pools:
|
||||||
|
worker_name = (
|
||||||
|
f"{name}-worker"
|
||||||
|
if len(worker_pools) == 1 and pool.name == "default"
|
||||||
|
else _name_with_suffix(name, f"worker-{pool.name}")
|
||||||
|
)
|
||||||
items.append(
|
items.append(
|
||||||
_deployment(
|
_deployment(
|
||||||
name=f"{name}-worker",
|
name=worker_name,
|
||||||
namespace=namespace,
|
namespace=namespace,
|
||||||
labels=labels,
|
labels=labels,
|
||||||
role="worker",
|
role="worker",
|
||||||
replicas=spec.replicas.worker,
|
replicas=pool.replicas,
|
||||||
image=spec.release.api_image,
|
image=spec.release.api_image,
|
||||||
command=(
|
command=(
|
||||||
"python",
|
"python",
|
||||||
@@ -187,15 +223,39 @@ def render_kubernetes(
|
|||||||
"govoplan_core.celery_app:celery",
|
"govoplan_core.celery_app:celery",
|
||||||
"worker",
|
"worker",
|
||||||
"--queues",
|
"--queues",
|
||||||
str(environment["CELERY_QUEUES"]),
|
",".join(pool.queues),
|
||||||
|
"--concurrency",
|
||||||
|
str(pool.concurrency),
|
||||||
"--loglevel",
|
"--loglevel",
|
||||||
"INFO",
|
"INFO",
|
||||||
),
|
),
|
||||||
config_name=config_name,
|
config_name=config_name,
|
||||||
secret_name=secret_name,
|
secret_name=secret_name,
|
||||||
service_account=service_account,
|
service_account=service_account,
|
||||||
|
extra_environment={
|
||||||
|
**_role_database_environment(environment, "WORKER"),
|
||||||
|
"CELERY_QUEUES": ",".join(pool.queues),
|
||||||
|
"CELERY_WORKER_CONCURRENCY": str(pool.concurrency),
|
||||||
|
"GOVOPLAN_WORKER_POOL": pool.name,
|
||||||
|
},
|
||||||
|
selector_labels={
|
||||||
|
"govoplan.add-ideas.de/worker-pool": pool.name,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if pool.replicas > 1:
|
||||||
|
items.append(
|
||||||
|
_pod_disruption_budget(
|
||||||
|
worker_name,
|
||||||
|
namespace,
|
||||||
|
labels,
|
||||||
|
"worker",
|
||||||
|
selector_labels={
|
||||||
|
"govoplan.add-ideas.de/worker-pool": pool.name,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if worker_pools:
|
||||||
items.append(
|
items.append(
|
||||||
_deployment(
|
_deployment(
|
||||||
name=f"{name}-scheduler",
|
name=f"{name}-scheduler",
|
||||||
@@ -225,6 +285,10 @@ def render_kubernetes(
|
|||||||
config_name=config_name,
|
config_name=config_name,
|
||||||
secret_name=secret_name,
|
secret_name=secret_name,
|
||||||
service_account=service_account,
|
service_account=service_account,
|
||||||
|
extra_environment=_role_database_environment(
|
||||||
|
environment,
|
||||||
|
"SCHEDULER",
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if spec.replicas.api > 1:
|
if spec.replicas.api > 1:
|
||||||
@@ -248,6 +312,15 @@ def render_kubernetes(
|
|||||||
"annotations": {
|
"annotations": {
|
||||||
"govoplan.add-ideas.de/profile": "stateless-shared-state",
|
"govoplan.add-ideas.de/profile": "stateless-shared-state",
|
||||||
"govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
|
"govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
|
||||||
|
"govoplan.add-ideas.de/database-connection-peak": str(
|
||||||
|
database_capacity["peak"]
|
||||||
|
),
|
||||||
|
"govoplan.add-ideas.de/database-connection-available": str(
|
||||||
|
database_capacity["available"]
|
||||||
|
),
|
||||||
|
"govoplan.add-ideas.de/worker-pools": ",".join(
|
||||||
|
pool.name for pool in worker_pools
|
||||||
|
),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"items": items,
|
"items": items,
|
||||||
@@ -290,7 +363,13 @@ def _validate_cluster_profile(
|
|||||||
+ ", ".join(unpinned_images)
|
+ ", ".join(unpinned_images)
|
||||||
)
|
)
|
||||||
missing = [
|
missing = [
|
||||||
key for key in (*_SECRET_KEYS, "CELERY_QUEUES") if not environment.get(key)
|
key
|
||||||
|
for key in (
|
||||||
|
*_SECRET_KEYS,
|
||||||
|
"CELERY_QUEUES",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
||||||
|
)
|
||||||
|
if not environment.get(key)
|
||||||
]
|
]
|
||||||
if missing:
|
if missing:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -298,6 +377,245 @@ def _validate_cluster_profile(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_pools(
|
||||||
|
spec: InstallationSpec,
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
) -> tuple[_WorkerPool, ...]:
|
||||||
|
if spec.replicas.worker == 0:
|
||||||
|
if str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip():
|
||||||
|
raise ValueError("Worker pools require at least one worker replica")
|
||||||
|
return ()
|
||||||
|
configured_queues = tuple(
|
||||||
|
item.strip()
|
||||||
|
for item in str(environment.get("CELERY_QUEUES") or "").split(",")
|
||||||
|
if item.strip()
|
||||||
|
)
|
||||||
|
if not configured_queues or any(
|
||||||
|
_QUEUE_NAME.fullmatch(item) is None for item in configured_queues
|
||||||
|
):
|
||||||
|
raise ValueError("CELERY_QUEUES must contain canonical queue names")
|
||||||
|
raw_pools = str(environment.get("GOVOPLAN_WORKER_POOLS") or "").strip()
|
||||||
|
if not raw_pools:
|
||||||
|
return (
|
||||||
|
_WorkerPool(
|
||||||
|
name="default",
|
||||||
|
queues=configured_queues,
|
||||||
|
replicas=spec.replicas.worker,
|
||||||
|
concurrency=_bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
"CELERY_WORKER_CONCURRENCY",
|
||||||
|
default=2,
|
||||||
|
minimum=1,
|
||||||
|
maximum=64,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
payload = json.loads(raw_pools)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ValueError("GOVOPLAN_WORKER_POOLS must be valid JSON") from exc
|
||||||
|
if not isinstance(payload, list) or not payload:
|
||||||
|
raise ValueError("GOVOPLAN_WORKER_POOLS must be a non-empty JSON list")
|
||||||
|
pools: list[_WorkerPool] = []
|
||||||
|
seen_names: set[str] = set()
|
||||||
|
seen_queues: set[str] = set()
|
||||||
|
for index, item in enumerate(payload):
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
raise ValueError(f"Worker pool {index} must be an object")
|
||||||
|
unknown = set(item) - {"name", "queues", "replicas", "concurrency"}
|
||||||
|
if unknown:
|
||||||
|
raise ValueError(
|
||||||
|
f"Worker pool {index} has unsupported keys: "
|
||||||
|
+ ", ".join(sorted(unknown))
|
||||||
|
)
|
||||||
|
pool_name = item.get("name")
|
||||||
|
queues = item.get("queues")
|
||||||
|
replicas = item.get("replicas")
|
||||||
|
concurrency = item.get("concurrency")
|
||||||
|
if not isinstance(pool_name, str) or _DNS_LABEL.fullmatch(pool_name) is None:
|
||||||
|
raise ValueError(f"Worker pool {index} has an invalid name")
|
||||||
|
if pool_name in seen_names:
|
||||||
|
raise ValueError(f"Worker pool name is duplicated: {pool_name}")
|
||||||
|
if (
|
||||||
|
not isinstance(queues, list)
|
||||||
|
or not queues
|
||||||
|
or any(
|
||||||
|
not isinstance(queue, str) or _QUEUE_NAME.fullmatch(queue) is None
|
||||||
|
for queue in queues
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError(f"Worker pool {pool_name} has invalid queues")
|
||||||
|
duplicate_queues = seen_queues.intersection(queues)
|
||||||
|
if duplicate_queues:
|
||||||
|
raise ValueError(
|
||||||
|
"Worker queues must have one owning pool: "
|
||||||
|
+ ", ".join(sorted(duplicate_queues))
|
||||||
|
)
|
||||||
|
if not isinstance(replicas, int) or not 1 <= replicas <= 128:
|
||||||
|
raise ValueError(
|
||||||
|
f"Worker pool {pool_name} replicas must be between 1 and 128"
|
||||||
|
)
|
||||||
|
if not isinstance(concurrency, int) or not 1 <= concurrency <= 64:
|
||||||
|
raise ValueError(
|
||||||
|
f"Worker pool {pool_name} concurrency must be between 1 and 64"
|
||||||
|
)
|
||||||
|
pools.append(
|
||||||
|
_WorkerPool(
|
||||||
|
name=pool_name,
|
||||||
|
queues=tuple(queues),
|
||||||
|
replicas=replicas,
|
||||||
|
concurrency=concurrency,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
seen_names.add(pool_name)
|
||||||
|
seen_queues.update(queues)
|
||||||
|
if sum(pool.replicas for pool in pools) != spec.replicas.worker:
|
||||||
|
raise ValueError("Worker pool replicas must equal installation replicas.worker")
|
||||||
|
configured_queue_set = set(configured_queues)
|
||||||
|
if seen_queues != configured_queue_set:
|
||||||
|
missing = configured_queue_set - seen_queues
|
||||||
|
unknown = seen_queues - configured_queue_set
|
||||||
|
detail = []
|
||||||
|
if missing:
|
||||||
|
detail.append("missing " + ", ".join(sorted(missing)))
|
||||||
|
if unknown:
|
||||||
|
detail.append("unknown " + ", ".join(sorted(unknown)))
|
||||||
|
raise ValueError(
|
||||||
|
"Worker pools must cover CELERY_QUEUES exactly: " + "; ".join(detail)
|
||||||
|
)
|
||||||
|
return tuple(sorted(pools, key=lambda pool: pool.name))
|
||||||
|
|
||||||
|
|
||||||
|
def _database_capacity(
|
||||||
|
spec: InstallationSpec,
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
worker_pools: tuple[_WorkerPool, ...],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
limit = _bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT",
|
||||||
|
minimum=10,
|
||||||
|
maximum=1_000_000,
|
||||||
|
)
|
||||||
|
reserve = _bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE",
|
||||||
|
default=10,
|
||||||
|
minimum=1,
|
||||||
|
maximum=999_999,
|
||||||
|
)
|
||||||
|
if reserve >= limit:
|
||||||
|
raise ValueError("Database connection reserve must be below the limit")
|
||||||
|
api_ceiling = _role_database_ceiling(environment, "API", 5, 2)
|
||||||
|
worker_ceiling = _role_database_ceiling(environment, "WORKER", 1, 1)
|
||||||
|
scheduler_ceiling = _role_database_ceiling(environment, "SCHEDULER", 1, 0)
|
||||||
|
migration_ceiling = _role_database_ceiling(environment, "MIGRATION", 2, 0)
|
||||||
|
worker_processes = sum(
|
||||||
|
pool.replicas * (pool.concurrency + 1) for pool in worker_pools
|
||||||
|
)
|
||||||
|
steady = (
|
||||||
|
spec.replicas.api * api_ceiling
|
||||||
|
+ worker_processes * worker_ceiling
|
||||||
|
+ (scheduler_ceiling if worker_pools else 0)
|
||||||
|
)
|
||||||
|
rollout_surge = api_ceiling + migration_ceiling
|
||||||
|
if worker_pools:
|
||||||
|
rollout_surge += scheduler_ceiling
|
||||||
|
rollout_surge += sum(
|
||||||
|
(pool.concurrency + 1) * worker_ceiling for pool in worker_pools
|
||||||
|
)
|
||||||
|
peak = steady + rollout_surge
|
||||||
|
available = limit - reserve
|
||||||
|
if peak > available:
|
||||||
|
raise ValueError(
|
||||||
|
"Kubernetes database connection peak exceeds the configured budget: "
|
||||||
|
f"peak={peak}, available={available}, limit={limit}, reserve={reserve}"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"limit": limit,
|
||||||
|
"reserve": reserve,
|
||||||
|
"available": available,
|
||||||
|
"steady": steady,
|
||||||
|
"peak": peak,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _role_database_ceiling(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
role: str,
|
||||||
|
default_pool_size: int,
|
||||||
|
default_overflow: int,
|
||||||
|
) -> int:
|
||||||
|
return _bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
f"GOVOPLAN_{role}_DB_POOL_SIZE",
|
||||||
|
default=default_pool_size,
|
||||||
|
minimum=1,
|
||||||
|
maximum=100,
|
||||||
|
) + _bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
|
||||||
|
default=default_overflow,
|
||||||
|
minimum=0,
|
||||||
|
maximum=200,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _role_database_environment(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
role: str,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
defaults = {
|
||||||
|
"API": (5, 2),
|
||||||
|
"WORKER": (1, 1),
|
||||||
|
"SCHEDULER": (1, 0),
|
||||||
|
"MIGRATION": (2, 0),
|
||||||
|
}
|
||||||
|
pool_size, overflow = defaults[role]
|
||||||
|
return {
|
||||||
|
"GOVOPLAN_DB_POOL_SIZE": str(
|
||||||
|
_bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
f"GOVOPLAN_{role}_DB_POOL_SIZE",
|
||||||
|
default=pool_size,
|
||||||
|
minimum=1,
|
||||||
|
maximum=100,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"GOVOPLAN_DB_MAX_OVERFLOW": str(
|
||||||
|
_bounded_environment_integer(
|
||||||
|
environment,
|
||||||
|
f"GOVOPLAN_{role}_DB_MAX_OVERFLOW",
|
||||||
|
default=overflow,
|
||||||
|
minimum=0,
|
||||||
|
maximum=200,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_environment_integer(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
*,
|
||||||
|
default: int | None = None,
|
||||||
|
minimum: int,
|
||||||
|
maximum: int,
|
||||||
|
) -> int:
|
||||||
|
raw = environment.get(name)
|
||||||
|
if raw is None or not str(raw).strip():
|
||||||
|
if default is None:
|
||||||
|
raise ValueError(f"{name} is required")
|
||||||
|
return default
|
||||||
|
try:
|
||||||
|
value = int(str(raw))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError(f"{name} must be an integer") from exc
|
||||||
|
if not minimum <= value <= maximum:
|
||||||
|
raise ValueError(f"{name} must be between {minimum} and {maximum}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _deployment(
|
def _deployment(
|
||||||
*,
|
*,
|
||||||
name: str,
|
name: str,
|
||||||
@@ -315,8 +633,13 @@ def _deployment(
|
|||||||
liveness_path: str | None = None,
|
liveness_path: str | None = None,
|
||||||
probe_host: str | None = None,
|
probe_host: str | None = None,
|
||||||
extra_environment: Mapping[str, str] | None = None,
|
extra_environment: Mapping[str, str] | None = None,
|
||||||
|
selector_labels: Mapping[str, str] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
role_labels = {**labels, "app.kubernetes.io/component": role}
|
role_labels = {
|
||||||
|
**labels,
|
||||||
|
"app.kubernetes.io/component": role,
|
||||||
|
**dict(selector_labels or {}),
|
||||||
|
}
|
||||||
environment: list[dict[str, Any]] = [
|
environment: list[dict[str, Any]] = [
|
||||||
{"name": "TMPDIR", "value": "/tmp"},
|
{"name": "TMPDIR", "value": "/tmp"},
|
||||||
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
|
||||||
@@ -400,6 +723,8 @@ def _deployment(
|
|||||||
"env": [
|
"env": [
|
||||||
{"name": "TMPDIR", "value": "/tmp"},
|
{"name": "TMPDIR", "value": "/tmp"},
|
||||||
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
|
||||||
|
{"name": "GOVOPLAN_DB_POOL_SIZE", "value": "1"},
|
||||||
|
{"name": "GOVOPLAN_DB_MAX_OVERFLOW", "value": "0"},
|
||||||
*_secret_environment(secret_name),
|
*_secret_environment(secret_name),
|
||||||
],
|
],
|
||||||
"securityContext": {
|
"securityContext": {
|
||||||
@@ -439,6 +764,7 @@ def _migration_job(
|
|||||||
config_name: str,
|
config_name: str,
|
||||||
secret_name: str,
|
secret_name: str,
|
||||||
service_account: str,
|
service_account: str,
|
||||||
|
database_environment: Mapping[str, str],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
|
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
|
||||||
job_name = _name_with_suffix(name, f"migrate-{release_key}")
|
job_name = _name_with_suffix(name, f"migrate-{release_key}")
|
||||||
@@ -481,6 +807,12 @@ def _migration_job(
|
|||||||
"env": [
|
"env": [
|
||||||
{"name": "TMPDIR", "value": "/tmp"},
|
{"name": "TMPDIR", "value": "/tmp"},
|
||||||
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
|
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
|
||||||
|
*(
|
||||||
|
{"name": key, "value": value}
|
||||||
|
for key, value in sorted(
|
||||||
|
database_environment.items()
|
||||||
|
)
|
||||||
|
),
|
||||||
{
|
{
|
||||||
"name": "GOVOPLAN_NODE_ID",
|
"name": "GOVOPLAN_NODE_ID",
|
||||||
"valueFrom": {
|
"valueFrom": {
|
||||||
@@ -541,8 +873,14 @@ def _pod_disruption_budget(
|
|||||||
namespace: str,
|
namespace: str,
|
||||||
labels: dict[str, str],
|
labels: dict[str, str],
|
||||||
role: str,
|
role: str,
|
||||||
|
*,
|
||||||
|
selector_labels: Mapping[str, str] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
role_labels = {**labels, "app.kubernetes.io/component": role}
|
role_labels = {
|
||||||
|
**labels,
|
||||||
|
"app.kubernetes.io/component": role,
|
||||||
|
**dict(selector_labels or {}),
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
"apiVersion": "policy/v1",
|
"apiVersion": "policy/v1",
|
||||||
"kind": "PodDisruptionBudget",
|
"kind": "PodDisruptionBudget",
|
||||||
|
|||||||
Reference in New Issue
Block a user