1805 lines
70 KiB
Python
1805 lines
70 KiB
Python
from __future__ import annotations
|
|
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
from datetime import UTC, datetime, timedelta
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
|
|
META_ROOT = Path(__file__).resolve().parents[1]
|
|
DEPLOYMENT_TOOLS = META_ROOT / "tools" / "deployment"
|
|
if str(DEPLOYMENT_TOOLS) not in sys.path:
|
|
sys.path.insert(0, str(DEPLOYMENT_TOOLS))
|
|
|
|
from govoplan_deploy.bundle import ( # noqa: E402
|
|
atomic_write,
|
|
bundle_paths,
|
|
canonical_json,
|
|
environment_fingerprint,
|
|
initial_secrets,
|
|
read_env,
|
|
reconcile_runtime_environment,
|
|
render_caddy_config,
|
|
render_compose,
|
|
render_existing_proxy_contract,
|
|
render_load_balancer_config,
|
|
write_env,
|
|
)
|
|
from govoplan_deploy.cli import _receipt_uses_direct_web_port, main # noqa: E402
|
|
import govoplan_deploy.cli as deployment_cli # noqa: E402
|
|
from govoplan_deploy.capabilities import ( # noqa: E402
|
|
capability_change_impacts,
|
|
infrastructure_capability_document,
|
|
infrastructure_dependency_inventory_from_mapping,
|
|
)
|
|
from govoplan_deploy.cluster_evidence import ( # noqa: E402
|
|
collect_kubernetes_evidence,
|
|
)
|
|
from govoplan_deploy.kubernetes import render_kubernetes # noqa: E402
|
|
from govoplan_deploy.model import ( # noqa: E402
|
|
SpecError,
|
|
default_spec,
|
|
parse_spec,
|
|
)
|
|
from govoplan_deploy.planning import _endpoint_check, build_plan # noqa: E402
|
|
import govoplan_deploy.planning as deployment_planning # noqa: E402
|
|
from govoplan_deploy.recovery import ( # noqa: E402
|
|
DeploymentOperationJournal,
|
|
load_operation,
|
|
recover_operation,
|
|
write_applied_state,
|
|
)
|
|
|
|
|
|
def run_cli(arguments: list[str]) -> tuple[int, str, str]:
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with redirect_stdout(stdout), redirect_stderr(stderr):
|
|
result = main(arguments)
|
|
return result, stdout.getvalue(), stderr.getvalue()
|
|
|
|
|
|
def _kubernetes_test_pod(name: str, component: str, node: str) -> dict:
|
|
return {
|
|
"metadata": {
|
|
"name": name,
|
|
"uid": f"uid-{name}",
|
|
"labels": {"app.kubernetes.io/component": component},
|
|
},
|
|
"spec": {"nodeName": node},
|
|
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
|
|
}
|
|
|
|
|
|
def _kubernetes_test_deployment(component: str, replicas: int) -> dict:
|
|
return {
|
|
"metadata": {
|
|
"name": f"govoplan-cluster-{component}",
|
|
"labels": {"app.kubernetes.io/component": component},
|
|
},
|
|
"spec": {"replicas": replicas},
|
|
"status": {"availableReplicas": replicas, "updatedReplicas": replicas},
|
|
}
|
|
|
|
|
|
def _dependency_inventory(
|
|
installation_id: str,
|
|
*,
|
|
generated_at: datetime | None = None,
|
|
) -> dict:
|
|
return {
|
|
"schema_version": 1,
|
|
"installation_id": installation_id,
|
|
"generated_at": (generated_at or datetime.now(UTC)).isoformat(),
|
|
"complete": True,
|
|
"inspected_capability_ids": ["coordination.redis", "mail.smtp"],
|
|
"providers": [
|
|
{
|
|
"module_id": "mail",
|
|
"state": "complete",
|
|
"capability_ids": ["mail.smtp"],
|
|
"dependency_count": 1,
|
|
}
|
|
],
|
|
"dependencies": [
|
|
{
|
|
"capability_id": "mail.smtp",
|
|
"module_id": "mail",
|
|
"dependency_type": "smtp_endpoint",
|
|
"dependency_ref": "endpoint:17",
|
|
"state": "active",
|
|
"scope": "system",
|
|
"summary": "Persisted SMTP endpoint has one credential binding.",
|
|
"metrics": {"credential_binding_count": 1},
|
|
"required_action": "Rebind or migrate this SMTP endpoint.",
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
class DeploymentInstallerTests(unittest.TestCase):
|
|
def test_kubernetes_evidence_requires_two_node_spread_and_safe_runtime(
|
|
self,
|
|
) -> None:
|
|
resources = {
|
|
"nodes": {
|
|
"items": [
|
|
{
|
|
"metadata": {"name": name},
|
|
"spec": {},
|
|
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
|
|
}
|
|
for name in ("node-a", "node-b")
|
|
]
|
|
},
|
|
"pods": {
|
|
"items": [
|
|
_kubernetes_test_pod("api-a", "api", "node-a"),
|
|
_kubernetes_test_pod("api-b", "api", "node-b"),
|
|
_kubernetes_test_pod("web-a", "web", "node-a"),
|
|
_kubernetes_test_pod("web-b", "web", "node-b"),
|
|
_kubernetes_test_pod("worker-a", "worker", "node-a"),
|
|
]
|
|
},
|
|
"deployments": {
|
|
"items": [
|
|
_kubernetes_test_deployment("api", 2),
|
|
_kubernetes_test_deployment("web", 2),
|
|
_kubernetes_test_deployment("worker", 1),
|
|
]
|
|
},
|
|
}
|
|
|
|
def run(arguments):
|
|
return resources[
|
|
"nodes"
|
|
if "nodes" in arguments
|
|
else "deployments"
|
|
if "deployments" in arguments
|
|
else "pods"
|
|
]
|
|
|
|
evidence = collect_kubernetes_evidence(
|
|
installation_id="govoplan-cluster",
|
|
namespace="govoplan",
|
|
ops_url="https://govoplan.example.test/api/v1/ops/status",
|
|
api_key="not-retained",
|
|
command_runner=run,
|
|
json_fetcher=lambda _url, _key: {
|
|
"readiness": {"ready": True},
|
|
"runtime_cluster": {
|
|
"expected": {"api": 2, "worker": 1},
|
|
"active": {"api": 2, "worker": 1},
|
|
"composition": {"skewed": False},
|
|
"software_versions": {"skewed": False},
|
|
"queues": {"missing": []},
|
|
},
|
|
"checks": [
|
|
{
|
|
"id": "database_capacity",
|
|
"state": "ok",
|
|
"detail": "Within budget",
|
|
"metrics": {"peak": 30, "available": 90},
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
self.assertEqual("passed", evidence["result"]["state"])
|
|
self.assertNotIn("not-retained", json.dumps(evidence))
|
|
self.assertEqual(
|
|
["node-a", "node-b"],
|
|
evidence["snapshot"]["ready_node_names"],
|
|
)
|
|
|
|
def test_kubernetes_api_loss_uses_a_non_json_mutation_command(self) -> None:
|
|
initial_pods = [
|
|
_kubernetes_test_pod("api-a", "api", "node-a"),
|
|
_kubernetes_test_pod("api-b", "api", "node-b"),
|
|
_kubernetes_test_pod("web-a", "web", "node-a"),
|
|
_kubernetes_test_pod("web-b", "web", "node-b"),
|
|
]
|
|
replacement_pods = [
|
|
_kubernetes_test_pod("api-b", "api", "node-b"),
|
|
_kubernetes_test_pod("api-c", "api", "node-a"),
|
|
]
|
|
deleted = False
|
|
actions: list[tuple[str, ...]] = []
|
|
|
|
def run(arguments):
|
|
if "nodes" in arguments:
|
|
return {
|
|
"items": [
|
|
{
|
|
"metadata": {"name": name},
|
|
"spec": {},
|
|
"status": {
|
|
"conditions": [
|
|
{"type": "Ready", "status": "True"}
|
|
]
|
|
},
|
|
}
|
|
for name in ("node-a", "node-b")
|
|
]
|
|
}
|
|
if "deployments" in arguments:
|
|
return {
|
|
"items": [
|
|
_kubernetes_test_deployment("api", 2),
|
|
_kubernetes_test_deployment("web", 2),
|
|
]
|
|
}
|
|
return {"items": replacement_pods if deleted else initial_pods}
|
|
|
|
def act(arguments):
|
|
nonlocal deleted
|
|
actions.append(tuple(arguments))
|
|
deleted = True
|
|
|
|
evidence = collect_kubernetes_evidence(
|
|
installation_id="govoplan-cluster",
|
|
namespace="govoplan",
|
|
ops_url="https://govoplan.example.test/api/v1/ops/status",
|
|
api_key="not-retained",
|
|
exercise_api_pod_loss=True,
|
|
command_runner=run,
|
|
action_runner=act,
|
|
json_fetcher=lambda _url, _key: {
|
|
"readiness": {"ready": True},
|
|
"runtime_cluster": {
|
|
"composition": {"skewed": False},
|
|
"software_versions": {"skewed": False},
|
|
"queues": {"missing": []},
|
|
},
|
|
"checks": [
|
|
{
|
|
"id": "database_capacity",
|
|
"state": "ok",
|
|
"detail": "Within budget",
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
self.assertEqual("passed", evidence["api_pod_loss"]["state"])
|
|
self.assertEqual(1, len(actions))
|
|
self.assertIn("delete", actions[0])
|
|
self.assertNotIn("-o", actions[0])
|
|
self.assertNotIn("not-retained", json.dumps(evidence))
|
|
|
|
def test_pre_migration_failure_restores_checksum_verified_applied_bundle(
|
|
self,
|
|
) -> None:
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="govoplan-deploy-recovery-"
|
|
) as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
previous_environment = initial_secrets(spec)
|
|
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
|
|
write_env(paths.env, previous_environment)
|
|
atomic_write(
|
|
paths.compose, canonical_json(render_compose(spec)), mode=0o600
|
|
)
|
|
atomic_write(
|
|
paths.receipt, canonical_json({"spec_sha256": "old"}), mode=0o600
|
|
)
|
|
write_applied_state(paths)
|
|
|
|
atomic_write(paths.spec, canonical_json({"changed": True}), mode=0o600)
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan={"installation_id": spec.installation_id},
|
|
)
|
|
journal.failed(RuntimeError("pull failed"))
|
|
|
|
result = recover_operation(paths, operation_id=journal.operation_id)
|
|
|
|
self.assertEqual("configuration_restored", result.action)
|
|
self.assertEqual(spec.to_dict(), json.loads(paths.spec.read_text()))
|
|
self.assertEqual(previous_environment, read_env(paths.env))
|
|
|
|
def test_post_migration_failure_requires_forward_recovery(self) -> None:
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="govoplan-deploy-recovery-"
|
|
) as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
|
|
write_env(paths.env, initial_secrets(spec))
|
|
atomic_write(
|
|
paths.compose, canonical_json(render_compose(spec)), mode=0o600
|
|
)
|
|
write_applied_state(paths)
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan={"installation_id": spec.installation_id},
|
|
)
|
|
journal.migration_started()
|
|
journal.failed(RuntimeError("health failed"))
|
|
|
|
result = recover_operation(paths, operation_id=journal.operation_id)
|
|
|
|
self.assertEqual("forward_recovery", result.action)
|
|
self.assertIn("not restored", result.detail)
|
|
|
|
def test_recovery_refuses_tampered_migration_boundary(self) -> None:
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="govoplan-deploy-recovery-"
|
|
) as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan={"installation_id": spec.installation_id},
|
|
)
|
|
journal.migration_started()
|
|
payload = json.loads(journal.path.joinpath("operation.json").read_text())
|
|
payload["migration_started"] = False
|
|
atomic_write(
|
|
journal.path / "operation.json",
|
|
canonical_json(payload),
|
|
mode=0o600,
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "migration boundary"):
|
|
load_operation(paths, operation_id=journal.operation_id)
|
|
|
|
def test_recovery_verifies_complete_snapshot_before_mutating_bundle(self) -> None:
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="govoplan-deploy-recovery-"
|
|
) as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
|
|
write_env(paths.env, initial_secrets(spec))
|
|
atomic_write(
|
|
paths.compose, canonical_json(render_compose(spec)), mode=0o600
|
|
)
|
|
write_applied_state(paths)
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan={"installation_id": spec.installation_id},
|
|
)
|
|
changed_spec = canonical_json({"changed": True})
|
|
atomic_write(paths.spec, changed_spec, mode=0o600)
|
|
atomic_write(
|
|
journal.path / "before" / "compose.json", b"tampered", mode=0o600
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "checksum failed"):
|
|
recover_operation(paths, operation_id=journal.operation_id)
|
|
|
|
self.assertEqual(changed_spec, paths.spec.read_bytes())
|
|
|
|
def test_recovery_refuses_tampered_desired_plan(self) -> None:
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="govoplan-deploy-recovery-"
|
|
) as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan={"installation_id": spec.installation_id},
|
|
)
|
|
payload = json.loads(journal.path.joinpath("operation.json").read_text())
|
|
payload["desired"]["installation_id"] = "other-installation"
|
|
atomic_write(
|
|
journal.path / "operation.json",
|
|
canonical_json(payload),
|
|
mode=0o600,
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "plan"):
|
|
load_operation(paths, operation_id=journal.operation_id)
|
|
|
|
def test_kubernetes_export_is_stateless_and_uses_exact_secret_contract(
|
|
self,
|
|
) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-cluster",
|
|
postgres_mode="external",
|
|
redis_mode="external",
|
|
storage_mode="s3",
|
|
api_replicas=3,
|
|
web_replicas=2,
|
|
worker_replicas=4,
|
|
api_image="registry.example.test/govoplan-api@sha256:" + "a" * 64,
|
|
web_image="registry.example.test/govoplan-web@sha256:" + "b" * 64,
|
|
)
|
|
environment = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": "postgresql+psycopg://user:db-secret@postgres.example.test/govoplan",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:db-secret@postgres.example.test/govoplan",
|
|
"REDIS_URL": "rediss://:redis-secret@redis.example.test/0",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
|
},
|
|
)
|
|
|
|
manifest = render_kubernetes(
|
|
spec,
|
|
environment,
|
|
backup_required=True,
|
|
backup_evidence={
|
|
"evidence_sha256": "c" * 64,
|
|
"recovery_point_id": "recovery-1",
|
|
"restore_drill_id": "drill-1",
|
|
},
|
|
)
|
|
rendered = json.dumps(manifest, sort_keys=True)
|
|
kinds = [item["kind"] for item in manifest["items"]]
|
|
deployments = {
|
|
item["metadata"]["name"]: item
|
|
for item in manifest["items"]
|
|
if item["kind"] == "Deployment"
|
|
}
|
|
|
|
self.assertNotIn("db-secret", rendered)
|
|
self.assertNotIn("redis-secret", rendered)
|
|
self.assertNotIn("object-secret", rendered)
|
|
capability_config = next(
|
|
item
|
|
for item in manifest["items"]
|
|
if item["kind"] == "ConfigMap"
|
|
and item["metadata"]["name"].endswith("infrastructure-capabilities")
|
|
)
|
|
capability_payload = json.loads(
|
|
capability_config["data"]["infrastructure-capabilities.json"]
|
|
)
|
|
self.assertEqual(1, capability_payload["schema_version"])
|
|
self.assertNotIn("db-secret", json.dumps(capability_payload))
|
|
api_container = deployments["govoplan-cluster-api"]["spec"]["template"]["spec"]["containers"][0]
|
|
self.assertIn(
|
|
{
|
|
"name": "deployment-capabilities",
|
|
"mountPath": "/etc/govoplan/deployment/infrastructure-capabilities.json",
|
|
"subPath": "infrastructure-capabilities.json",
|
|
"readOnly": True,
|
|
},
|
|
api_container["volumeMounts"],
|
|
)
|
|
self.assertNotIn("PersistentVolumeClaim", kinds)
|
|
self.assertNotIn("StatefulSet", kinds)
|
|
self.assertEqual(3, deployments["govoplan-cluster-api"]["spec"]["replicas"])
|
|
self.assertEqual(4, deployments["govoplan-cluster-worker"]["spec"]["replicas"])
|
|
scheduler_command = deployments["govoplan-cluster-scheduler"]["spec"][
|
|
"template"
|
|
]["spec"]["containers"][0]["command"]
|
|
self.assertIn("govoplan_core.commands.fenced_run", scheduler_command)
|
|
self.assertEqual(
|
|
["--schedule", "/tmp/celerybeat-schedule"],
|
|
scheduler_command[-2:],
|
|
)
|
|
api_init_command = deployments["govoplan-cluster-api"]["spec"]["template"][
|
|
"spec"
|
|
]["initContainers"][0]["command"]
|
|
self.assertIn("govoplan_core.commands.wait_for_database", api_init_command)
|
|
migration = next(item for item in manifest["items"] if item["kind"] == "Job")
|
|
self.assertRegex(
|
|
migration["metadata"]["name"],
|
|
r"^govoplan-cluster-migrate-[0-9a-f]{8}$",
|
|
)
|
|
self.assertEqual(
|
|
"forward-recovery",
|
|
migration["metadata"]["annotations"]["govoplan.add-ideas.de/recovery-mode"],
|
|
)
|
|
self.assertEqual(
|
|
"c" * 64,
|
|
migration["metadata"]["annotations"][
|
|
"govoplan.add-ideas.de/backup-evidence-sha256"
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
"recovery-1",
|
|
migration["metadata"]["annotations"][
|
|
"govoplan.add-ideas.de/recovery-point"
|
|
],
|
|
)
|
|
config = next(item for item in manifest["items"] if item["kind"] == "ConfigMap")
|
|
self.assertEqual("shared", config["data"]["GOVOPLAN_STATE_PROFILE"])
|
|
self.assertEqual("3", config["data"]["GOVOPLAN_EXPECTED_API_REPLICAS"])
|
|
self.assertEqual(
|
|
[{"name": "Host", "value": "127.0.0.1"}],
|
|
deployments["govoplan-cluster-api"]["spec"]["template"]["spec"][
|
|
"containers"
|
|
][0]["readinessProbe"]["httpGet"]["httpHeaders"],
|
|
)
|
|
api_pod_spec = deployments["govoplan-cluster-api"]["spec"]["template"][
|
|
"spec"
|
|
]
|
|
self.assertEqual(30, api_pod_spec["terminationGracePeriodSeconds"])
|
|
self.assertEqual(
|
|
["/bin/sh", "-c", "sleep 10"],
|
|
api_pod_spec["containers"][0]["lifecycle"]["preStop"]["exec"][
|
|
"command"
|
|
],
|
|
)
|
|
self.assertNotIn(
|
|
"lifecycle",
|
|
deployments["govoplan-cluster-worker"]["spec"]["template"]["spec"][
|
|
"containers"
|
|
][0],
|
|
)
|
|
worker_command = deployments["govoplan-cluster-worker"]["spec"]["template"][
|
|
"spec"
|
|
]["containers"][0]["command"]
|
|
self.assertIn("--concurrency", worker_command)
|
|
for deployment in deployments.values():
|
|
spread = deployment["spec"]["template"]["spec"][
|
|
"topologySpreadConstraints"
|
|
][0]
|
|
self.assertEqual("DoNotSchedule", spread["whenUnsatisfiable"])
|
|
self.assertEqual(["pod-template-hash"], spread["matchLabelKeys"])
|
|
self.assertEqual(
|
|
"62",
|
|
manifest["metadata"]["annotations"][
|
|
"govoplan.add-ideas.de/database-connection-peak"
|
|
],
|
|
)
|
|
|
|
def test_kubernetes_export_mounts_an_optional_s3_ca_on_backend_roles(
|
|
self,
|
|
) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-cluster",
|
|
postgres_mode="external",
|
|
redis_mode="external",
|
|
storage_mode="s3",
|
|
api_replicas=2,
|
|
web_replicas=2,
|
|
worker_replicas=2,
|
|
api_image="registry.example.test/govoplan-api@sha256:" + "a" * 64,
|
|
web_image="registry.example.test/govoplan-web@sha256:" + "b" * 64,
|
|
)
|
|
environment = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": "postgresql+psycopg://user:secret@postgres.example.test/govoplan",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:secret@postgres.example.test/govoplan",
|
|
"REDIS_URL": "rediss://:secret@redis.example.test/0",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
|
},
|
|
)
|
|
|
|
manifest = render_kubernetes(
|
|
spec,
|
|
environment,
|
|
s3_ca_secret_name="govoplan-s3-ca",
|
|
backup_required=False,
|
|
)
|
|
backend_pods = [
|
|
item["spec"]["template"]["spec"]
|
|
for item in manifest["items"]
|
|
if item["kind"] in {"Deployment", "Job"}
|
|
and item["metadata"]["labels"].get("app.kubernetes.io/component")
|
|
in {"api", "worker", "scheduler", "migration"}
|
|
]
|
|
web = next(
|
|
item
|
|
for item in manifest["items"]
|
|
if item["kind"] == "Deployment"
|
|
and item["metadata"]["labels"].get("app.kubernetes.io/component")
|
|
== "web"
|
|
)
|
|
|
|
self.assertTrue(backend_pods)
|
|
for pod in backend_pods:
|
|
self.assertIn(
|
|
{
|
|
"name": "s3-ca",
|
|
"secret": {
|
|
"secretName": "govoplan-s3-ca",
|
|
"items": [{"key": "ca.crt", "path": "s3-ca.crt"}],
|
|
},
|
|
},
|
|
pod["volumes"],
|
|
)
|
|
for container in [*pod.get("initContainers", []), *pod["containers"]]:
|
|
self.assertIn(
|
|
{
|
|
"name": "AWS_CA_BUNDLE",
|
|
"value": "/etc/govoplan/trust/s3-ca.crt",
|
|
},
|
|
container["env"],
|
|
)
|
|
self.assertIn(
|
|
{
|
|
"name": "s3-ca",
|
|
"mountPath": "/etc/govoplan/trust",
|
|
"readOnly": True,
|
|
},
|
|
container["volumeMounts"],
|
|
)
|
|
self.assertNotIn(
|
|
"s3-ca",
|
|
{
|
|
volume["name"]
|
|
for volume in web["spec"]["template"]["spec"]["volumes"]
|
|
},
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "S3 CA secret"):
|
|
render_kubernetes(
|
|
spec,
|
|
environment,
|
|
s3_ca_secret_name="INVALID_NAME",
|
|
backup_required=False,
|
|
)
|
|
|
|
def test_kubernetes_export_splits_worker_queues_and_rejects_capacity_overrun(
|
|
self,
|
|
) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-cluster",
|
|
postgres_mode="external",
|
|
redis_mode="external",
|
|
storage_mode="s3",
|
|
api_replicas=2,
|
|
web_replicas=2,
|
|
worker_replicas=3,
|
|
api_image="registry.example.test/govoplan-api@sha256:" + "a" * 64,
|
|
web_image="registry.example.test/govoplan-web@sha256:" + "b" * 64,
|
|
)
|
|
environment = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": "postgresql+psycopg://user:db-secret@postgres.example.test/govoplan",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:db-secret@postgres.example.test/govoplan",
|
|
"REDIS_URL": "rediss://:redis-secret@redis.example.test/0",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
|
},
|
|
)
|
|
queues = environment["CELERY_QUEUES"].split(",")
|
|
environment["GOVOPLAN_WORKER_POOLS"] = json.dumps(
|
|
[
|
|
{
|
|
"name": "delivery",
|
|
"queues": queues[:3],
|
|
"replicas": 2,
|
|
"concurrency": 2,
|
|
},
|
|
{
|
|
"name": "platform",
|
|
"queues": queues[3:],
|
|
"replicas": 1,
|
|
"concurrency": 1,
|
|
},
|
|
]
|
|
)
|
|
|
|
manifest = render_kubernetes(spec, environment)
|
|
workers = [
|
|
item
|
|
for item in manifest["items"]
|
|
if item["kind"] == "Deployment"
|
|
and item["metadata"]["labels"].get("app.kubernetes.io/component")
|
|
== "worker"
|
|
]
|
|
|
|
self.assertEqual(2, len(workers))
|
|
self.assertEqual(
|
|
{"delivery", "platform"},
|
|
{
|
|
item["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"]
|
|
for item in workers
|
|
},
|
|
)
|
|
for deployment in workers:
|
|
container = deployment["spec"]["template"]["spec"]["containers"][0]
|
|
explicit_environment = {
|
|
item["name"]: item.get("value")
|
|
for item in container["env"]
|
|
if "value" in item
|
|
}
|
|
self.assertEqual(
|
|
deployment["metadata"]["labels"]["govoplan.add-ideas.de/worker-pool"],
|
|
explicit_environment["GOVOPLAN_WORKER_POOL"],
|
|
)
|
|
self.assertEqual(
|
|
set(
|
|
container["command"][
|
|
container["command"].index("--queues") + 1
|
|
].split(",")
|
|
),
|
|
set(explicit_environment["CELERY_QUEUES"].split(",")),
|
|
)
|
|
|
|
environment["GOVOPLAN_DB_CONNECTION_LIMIT"] = "40"
|
|
with self.assertRaisesRegex(ValueError, "connection peak"):
|
|
render_kubernetes(spec, environment)
|
|
|
|
def test_kubernetes_export_rejects_mutable_release_images(self) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-cluster",
|
|
postgres_mode="external",
|
|
redis_mode="external",
|
|
storage_mode="s3",
|
|
api_image="registry.example.test/govoplan-api:latest",
|
|
web_image="registry.example.test/govoplan-web:latest",
|
|
)
|
|
environment = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": "postgresql+psycopg://user:secret@postgres.example.test/govoplan",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": "postgresql://user:secret@postgres.example.test/govoplan",
|
|
"REDIS_URL": "rediss://:secret@redis.example.test/0",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
},
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "digest-pinned"):
|
|
render_kubernetes(spec, environment)
|
|
|
|
def test_kubernetes_export_rejects_installer_managed_state_services(self) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-cluster",
|
|
storage_mode="garage",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "external shared PostgreSQL"):
|
|
render_kubernetes(spec, initial_secrets(spec))
|
|
|
|
def test_default_bundle_has_base_modules_and_managed_dependencies(self) -> None:
|
|
spec = default_spec()
|
|
compose = render_compose(spec)
|
|
|
|
self.assertEqual("evaluation", spec.profile)
|
|
self.assertIn("access", spec.enabled_modules)
|
|
self.assertIn("postgres", compose["services"])
|
|
self.assertIn("redis", compose["services"])
|
|
self.assertIn("worker", compose["services"])
|
|
self.assertIn("load-balancer", compose["services"])
|
|
self.assertNotIn("test-mail", compose["services"])
|
|
self.assertEqual(
|
|
["127.0.0.1:8080:8080"],
|
|
compose["services"]["load-balancer"]["ports"],
|
|
)
|
|
self.assertNotIn("ports", compose["services"]["web"])
|
|
self.assertIn(
|
|
"./infrastructure-capabilities.json:/etc/govoplan/deployment/infrastructure-capabilities.json:ro",
|
|
compose["services"]["api"]["volumes"],
|
|
)
|
|
self.assertEqual(1, compose["services"]["api"]["scale"])
|
|
self.assertEqual(1, compose["services"]["web"]["scale"])
|
|
|
|
def test_managed_ingress_persists_certificate_state_and_hides_upstream(
|
|
self,
|
|
) -> None:
|
|
spec = default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
ingress_mode="managed",
|
|
acme_email="operator@example.test",
|
|
)
|
|
compose = render_compose(spec)
|
|
ingress = compose["services"]["ingress"]
|
|
|
|
self.assertNotIn("ports", compose["services"]["load-balancer"])
|
|
self.assertEqual(
|
|
["0.0.0.0:80:8080", "0.0.0.0:443:8443"],
|
|
ingress["ports"],
|
|
)
|
|
self.assertIn("caddy-data:/data", ingress["volumes"])
|
|
self.assertIn("caddy-config:/config", ingress["volumes"])
|
|
self.assertEqual(["ALL"], ingress["cap_drop"])
|
|
self.assertEqual(["NET_BIND_SERVICE"], ingress["cap_add"])
|
|
self.assertEqual(["no-new-privileges:true"], ingress["security_opt"])
|
|
self.assertIn("reverse_proxy load-balancer:8080", render_caddy_config(spec))
|
|
self.assertNotIn("operator@example.test", json.dumps(compose))
|
|
|
|
def test_existing_proxy_contract_and_header_trust_are_exact(self) -> None:
|
|
spec = default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
ingress_mode="existing-proxy",
|
|
trusted_proxy_cidrs=("172.20.0.7/32",),
|
|
)
|
|
contract = render_existing_proxy_contract(spec)
|
|
load_balancer = render_load_balancer_config(spec)
|
|
|
|
self.assertEqual("http://127.0.0.1:8080", contract["upstream"])
|
|
self.assertEqual(["172.20.0.7/32"], contract["trusted_proxy_cidrs"])
|
|
self.assertIn("acl trusted_forward_proxy src 172.20.0.7/32", load_balancer)
|
|
self.assertIn("del-header X-Forwarded-Proto", load_balancer)
|
|
self.assertIn(
|
|
"del-header X-Forwarded-For unless trusted_forward_proxy", load_balancer
|
|
)
|
|
|
|
def test_ingress_rejects_unsafe_proxy_ranges_and_managed_ip_hosts(self) -> None:
|
|
with self.assertRaisesRegex(SpecError, "/24 or narrower"):
|
|
default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
ingress_mode="existing-proxy",
|
|
trusted_proxy_cidrs=("10.0.0.0/8",),
|
|
)
|
|
with self.assertRaisesRegex(SpecError, "DNS hostname"):
|
|
default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://192.0.2.10",
|
|
ingress_mode="managed",
|
|
acme_email="operator@example.test",
|
|
)
|
|
|
|
def test_disabled_redis_removes_workers_and_sets_single_process_acknowledgement(
|
|
self,
|
|
) -> None:
|
|
spec = default_spec(redis_mode="disabled")
|
|
values = initial_secrets(spec)
|
|
compose = render_compose(spec)
|
|
|
|
self.assertNotIn("redis", compose["services"])
|
|
self.assertNotIn("worker", compose["services"])
|
|
self.assertNotIn("scheduler", compose["services"])
|
|
self.assertEqual("false", values["CELERY_ENABLED"])
|
|
self.assertEqual("true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"])
|
|
self.assertEqual("local", values["GOVOPLAN_STATE_PROFILE"])
|
|
|
|
def test_self_hosted_rejects_insecure_or_test_only_choices(self) -> None:
|
|
with self.assertRaisesRegex(SpecError, "must use HTTPS"):
|
|
default_spec(profile="self-hosted")
|
|
with self.assertRaisesRegex(SpecError, "require managed or external Redis"):
|
|
default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
redis_mode="disabled",
|
|
)
|
|
with self.assertRaisesRegex(SpecError, "test-mail"):
|
|
default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
mail_mode="test-mail",
|
|
)
|
|
|
|
def test_spec_rejects_unknown_fields_and_duplicate_modules(self) -> None:
|
|
raw = default_spec().to_dict()
|
|
raw["unexpected"] = True
|
|
with self.assertRaisesRegex(SpecError, "unknown fields"):
|
|
parse_spec(raw)
|
|
|
|
raw = default_spec().to_dict()
|
|
raw["enabled_modules"].append(raw["enabled_modules"][0])
|
|
with self.assertRaisesRegex(SpecError, "duplicate module"):
|
|
parse_spec(raw)
|
|
|
|
raw = default_spec().to_dict()
|
|
raw["network_subnet"] = "127.0.0.0/24"
|
|
with self.assertRaisesRegex(SpecError, "RFC1918"):
|
|
parse_spec(raw)
|
|
|
|
def test_generated_secrets_are_stable_across_reconfiguration(self) -> None:
|
|
first = default_spec()
|
|
values = initial_secrets(first)
|
|
master_key = values["MASTER_KEY_B64"]
|
|
postgres_password = values["POSTGRES_PASSWORD"]
|
|
redis_password = values["REDIS_PASSWORD"]
|
|
|
|
second = default_spec(mail_mode="test-mail", module_set="full")
|
|
reconciled = reconcile_runtime_environment(second, values)
|
|
|
|
self.assertEqual(master_key, reconciled["MASTER_KEY_B64"])
|
|
self.assertEqual(postgres_password, reconciled["POSTGRES_PASSWORD"])
|
|
self.assertEqual(redis_password, reconciled["REDIS_PASSWORD"])
|
|
self.assertEqual(
|
|
",".join(second.enabled_modules), reconciled["ENABLED_MODULES"]
|
|
)
|
|
|
|
def test_external_services_require_explicit_valid_urls(self) -> None:
|
|
spec = default_spec(postgres_mode="external", redis_mode="external")
|
|
with self.assertRaisesRegex(ValueError, "external PostgreSQL"):
|
|
initial_secrets(spec)
|
|
with self.assertRaisesRegex(ValueError, "external Redis"):
|
|
initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": (
|
|
"postgresql+psycopg://user:secret@db.example.test/govoplan"
|
|
)
|
|
},
|
|
)
|
|
values = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": (
|
|
"postgresql+psycopg://user:secret@db.example.test/govoplan"
|
|
),
|
|
"REDIS_URL": "rediss://:secret@redis.example.test/0",
|
|
},
|
|
)
|
|
self.assertEqual("rediss://:secret@redis.example.test/0", values["REDIS_URL"])
|
|
|
|
def test_s3_requires_complete_private_configuration_and_https_endpoint(
|
|
self,
|
|
) -> None:
|
|
evaluation = default_spec(storage_mode="s3")
|
|
with self.assertRaisesRegex(ValueError, "S3 storage requires"):
|
|
initial_secrets(evaluation)
|
|
|
|
supplied = {
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "http://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
}
|
|
with self.assertRaisesRegex(ValueError, "must use HTTPS"):
|
|
initial_secrets(evaluation, supplied=supplied)
|
|
supplied["FILE_STORAGE_S3_ENDPOINT_URL"] = "https://s3.example.test"
|
|
values = initial_secrets(evaluation, supplied=supplied)
|
|
self.assertEqual("s3", values["FILE_STORAGE_BACKEND"])
|
|
self.assertEqual("true", values["FILE_STORAGE_S3_ENDPOINT_TRUSTED"])
|
|
production = default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://govoplan.example.test",
|
|
storage_mode="s3",
|
|
)
|
|
self.assertEqual(
|
|
"s3",
|
|
initial_secrets(production, supplied=supplied)["FILE_STORAGE_BACKEND"],
|
|
)
|
|
|
|
def test_managed_garage_bootstraps_private_s3_storage(self) -> None:
|
|
spec = default_spec(storage_mode="garage")
|
|
values = initial_secrets(spec)
|
|
compose = render_compose(spec)
|
|
|
|
self.assertEqual("s3", values["FILE_STORAGE_BACKEND"])
|
|
self.assertEqual("true", values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"])
|
|
self.assertEqual("false", values["FILE_STORAGE_S3_ENDPOINT_TRUSTED"])
|
|
self.assertEqual(
|
|
"http://garage:3900",
|
|
values["FILE_STORAGE_S3_ENDPOINT_URL"],
|
|
)
|
|
self.assertEqual("garage", values["FILE_STORAGE_S3_REGION"])
|
|
self.assertEqual(
|
|
values["GARAGE_DEFAULT_ACCESS_KEY"],
|
|
values["FILE_STORAGE_S3_ACCESS_KEY_ID"],
|
|
)
|
|
self.assertEqual(
|
|
values["GARAGE_DEFAULT_SECRET_KEY"],
|
|
values["FILE_STORAGE_S3_SECRET_ACCESS_KEY"],
|
|
)
|
|
self.assertTrue(values["GARAGE_DEFAULT_ACCESS_KEY"].startswith("GK"))
|
|
self.assertEqual(34, len(values["GARAGE_DEFAULT_ACCESS_KEY"]))
|
|
self.assertEqual(64, len(values["GARAGE_DEFAULT_SECRET_KEY"]))
|
|
self.assertIn("garage", compose["services"])
|
|
self.assertIn("garage-meta", compose["volumes"])
|
|
self.assertIn("garage-data", compose["volumes"])
|
|
self.assertEqual(
|
|
["/garage", "server", "--single-node", "--default-bucket"],
|
|
compose["services"]["garage"]["command"],
|
|
)
|
|
self.assertNotIn(
|
|
values["GARAGE_DEFAULT_SECRET_KEY"],
|
|
json.dumps(compose),
|
|
)
|
|
|
|
reconciled = reconcile_runtime_environment(spec, values)
|
|
self.assertEqual(
|
|
values["GARAGE_DEFAULT_ACCESS_KEY"],
|
|
reconciled["GARAGE_DEFAULT_ACCESS_KEY"],
|
|
)
|
|
self.assertEqual(
|
|
values["GARAGE_RPC_SECRET"],
|
|
reconciled["GARAGE_RPC_SECRET"],
|
|
)
|
|
|
|
def test_infrastructure_capability_document_exposes_refs_not_secrets(self) -> None:
|
|
spec = default_spec(
|
|
installation_id="govoplan-shared",
|
|
postgres_mode="external",
|
|
redis_mode="external",
|
|
storage_mode="s3",
|
|
mail_mode="external-relay",
|
|
module_set="full",
|
|
)
|
|
values = initial_secrets(
|
|
spec,
|
|
supplied={
|
|
"DATABASE_URL": "postgresql+psycopg://user:database-secret@db.example.test/govoplan",
|
|
"REDIS_URL": "rediss://:redis-secret@redis.example.test/0",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test",
|
|
"FILE_STORAGE_S3_REGION": "eu-test-1",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "object-key",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "object-secret",
|
|
"FILE_STORAGE_S3_BUCKET": "govoplan",
|
|
},
|
|
)
|
|
|
|
document = infrastructure_capability_document(spec, values)
|
|
rendered = json.dumps(document, sort_keys=True)
|
|
capabilities = {item["id"]: item for item in document["capabilities"]}
|
|
|
|
self.assertNotIn("database-secret", rendered)
|
|
self.assertNotIn("redis-secret", rendered)
|
|
self.assertNotIn("object-secret", rendered)
|
|
self.assertNotIn("object-key", rendered)
|
|
self.assertEqual("externally_supplied", capabilities["database.postgresql"]["state"])
|
|
self.assertEqual("db.example.test", capabilities["database.postgresql"]["endpoint"]["host"])
|
|
self.assertEqual(["env:DATABASE_URL"], capabilities["database.postgresql"]["secret_refs"])
|
|
self.assertEqual("available_unconfigured", capabilities["mail.smtp"]["state"])
|
|
self.assertEqual("mail.smtp-profile", document["post_install_tasks"][0]["id"])
|
|
|
|
def test_capability_impact_detects_external_endpoint_rebinding(self) -> None:
|
|
spec = default_spec(postgres_mode="external", module_set="full")
|
|
previous = infrastructure_capability_document(
|
|
spec,
|
|
{"DATABASE_URL": "postgresql://user:old-secret@old-db.example.test/govoplan"},
|
|
)
|
|
desired = infrastructure_capability_document(
|
|
spec,
|
|
{"DATABASE_URL": "postgresql://user:new-secret@new-db.example.test/govoplan"},
|
|
)
|
|
|
|
impacts = {
|
|
item.capability_id: item
|
|
for item in capability_change_impacts(previous, desired)
|
|
}
|
|
|
|
self.assertEqual("reconfigure", impacts["database.postgresql"].action)
|
|
self.assertIn("changed endpoint binding", impacts["database.postgresql"].detail)
|
|
self.assertNotIn("old-secret", impacts["database.postgresql"].detail)
|
|
self.assertNotIn("new-secret", impacts["database.postgresql"].detail)
|
|
|
|
def test_capability_impact_includes_provider_dependency_evidence(self) -> None:
|
|
previous_spec = default_spec(mail_mode="test-mail", module_set="full")
|
|
desired_spec = default_spec(mail_mode="disabled", module_set="full")
|
|
inventory = infrastructure_dependency_inventory_from_mapping(
|
|
_dependency_inventory(previous_spec.installation_id)
|
|
)
|
|
|
|
impacts = {
|
|
item.capability_id: item
|
|
for item in capability_change_impacts(
|
|
infrastructure_capability_document(previous_spec, {}),
|
|
infrastructure_capability_document(desired_spec, {}),
|
|
dependency_inventory=inventory,
|
|
)
|
|
}
|
|
|
|
mail = impacts["mail.smtp"]
|
|
self.assertTrue(mail.inventory_inspected)
|
|
self.assertEqual("endpoint:17", mail.actual_dependencies[0].dependency_ref)
|
|
self.assertIn("mail:endpoint:17", mail.detail)
|
|
self.assertIn("Rebind or migrate", mail.required_action)
|
|
|
|
def test_replica_counts_drive_compose_and_load_balancer_discovery(self) -> None:
|
|
spec = default_spec(
|
|
storage_mode="garage",
|
|
api_replicas=3,
|
|
web_replicas=2,
|
|
worker_replicas=4,
|
|
)
|
|
compose = render_compose(spec)
|
|
config = render_load_balancer_config(spec)
|
|
|
|
self.assertEqual(3, compose["services"]["api"]["scale"])
|
|
self.assertEqual(2, compose["services"]["web"]["scale"])
|
|
self.assertEqual(4, compose["services"]["worker"]["scale"])
|
|
self.assertEqual(
|
|
"http://load-balancer:8000",
|
|
compose["services"]["web"]["environment"]["GOVOPLAN_API_UPSTREAM"],
|
|
)
|
|
self.assertIn("server-template web- 2 web:8080", config)
|
|
self.assertIn("server-template api- 3 api:8000", config)
|
|
self.assertIn("option httpchk GET /health/ready", config)
|
|
self.assertIn("http-check send hdr Host 127.0.0.1", config)
|
|
self.assertIn(
|
|
"headers={'Host': '127.0.0.1'}",
|
|
compose["services"]["api"]["healthcheck"]["test"][-1],
|
|
)
|
|
self.assertEqual(
|
|
"host-shared",
|
|
initial_secrets(spec)["GOVOPLAN_STATE_PROFILE"],
|
|
)
|
|
|
|
def test_multi_replica_runtime_requires_shared_redis(self) -> None:
|
|
with self.assertRaisesRegex(SpecError, "multiple API replicas"):
|
|
default_spec(redis_mode="disabled", api_replicas=2)
|
|
with self.assertRaisesRegex(SpecError, "worker must be 0"):
|
|
default_spec(redis_mode="disabled", worker_replicas=1)
|
|
with self.assertRaisesRegex(SpecError, "at least 1"):
|
|
default_spec(redis_mode="managed", worker_replicas=0)
|
|
|
|
def test_legacy_spec_defaults_new_topology_fields(self) -> None:
|
|
raw = default_spec().to_dict()
|
|
raw.pop("replicas")
|
|
raw.pop("ingress")
|
|
raw["components"].pop("load_balancer")
|
|
raw["components"]["storage"].pop("image")
|
|
|
|
parsed = parse_spec(raw)
|
|
|
|
self.assertEqual(1, parsed.replicas.api)
|
|
self.assertEqual(1, parsed.replicas.web)
|
|
self.assertEqual(1, parsed.replicas.worker)
|
|
self.assertEqual("managed", parsed.components.load_balancer.mode)
|
|
self.assertEqual("local", parsed.ingress.mode)
|
|
|
|
def test_compose_contains_no_secret_values(self) -> None:
|
|
spec = default_spec()
|
|
values = initial_secrets(spec)
|
|
rendered = json.dumps(render_compose(spec), sort_keys=True)
|
|
|
|
for key in (
|
|
"MASTER_KEY_B64",
|
|
"POSTGRES_PASSWORD",
|
|
"REDIS_PASSWORD",
|
|
):
|
|
self.assertNotIn(values[key], rendered)
|
|
self.assertNotIn("secrets.env", rendered)
|
|
self.assertNotIn("env_file", rendered)
|
|
self.assertNotIn("./:/etc/govoplan", rendered)
|
|
self.assertNotIn("installer", render_compose(spec)["services"])
|
|
postgres_environment = render_compose(spec)["services"]["postgres"][
|
|
"environment"
|
|
]
|
|
redis_environment = render_compose(spec)["services"]["redis"]["environment"]
|
|
self.assertEqual(
|
|
{"POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"},
|
|
set(postgres_environment),
|
|
)
|
|
self.assertEqual({"REDIS_PASSWORD"}, set(redis_environment))
|
|
self.assertNotIn("MASTER_KEY_B64", postgres_environment)
|
|
self.assertNotIn("MASTER_KEY_B64", redis_environment)
|
|
|
|
def test_secret_environment_round_trips_literal_special_characters(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
path = Path(directory) / "secrets.env"
|
|
values = {
|
|
"PASSWORD": r"dollar$ quote' slash\\ hash# space ",
|
|
"EMPTY_VALUE": "",
|
|
}
|
|
|
|
write_env(path, values)
|
|
|
|
self.assertEqual(values, read_env(path))
|
|
|
|
def test_first_plan_creates_services_and_applied_receipt_becomes_noop(
|
|
self,
|
|
) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
write_env(paths.env, initial_secrets(spec))
|
|
first = build_plan(spec, paths, include_host_checks=False)
|
|
|
|
self.assertIn(
|
|
("create", "installation"),
|
|
{(action.action, action.target) for action in first.actions},
|
|
)
|
|
self.assertIn(
|
|
("start", "api"),
|
|
{(action.action, action.target) for action in first.actions},
|
|
)
|
|
|
|
receipt = {
|
|
"spec_sha256": first.desired_spec_sha256,
|
|
"compose_sha256": first.desired_compose_sha256,
|
|
"environment_fingerprint": (first.desired_environment_fingerprint),
|
|
"services": list(render_compose(spec)["services"]),
|
|
}
|
|
atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
|
|
second = build_plan(spec, paths, include_host_checks=False)
|
|
|
|
self.assertEqual(
|
|
[("noop", "installation")],
|
|
[(action.action, action.target) for action in second.actions],
|
|
)
|
|
|
|
def test_reconfiguration_plans_removed_component_containers(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
first_spec = default_spec(mail_mode="test-mail", module_set="full")
|
|
first_environment = initial_secrets(first_spec)
|
|
write_env(paths.env, first_environment)
|
|
first_plan = build_plan(first_spec, paths, include_host_checks=False)
|
|
first_capabilities = infrastructure_capability_document(
|
|
first_spec,
|
|
first_environment,
|
|
)
|
|
atomic_write(
|
|
paths.receipt,
|
|
canonical_json(
|
|
{
|
|
"spec_sha256": first_plan.desired_spec_sha256,
|
|
"compose_sha256": first_plan.desired_compose_sha256,
|
|
"environment_fingerprint": (
|
|
first_plan.desired_environment_fingerprint
|
|
),
|
|
"services": list(render_compose(first_spec)["services"]),
|
|
"infrastructure_capabilities": first_capabilities,
|
|
}
|
|
),
|
|
mode=0o600,
|
|
)
|
|
|
|
second_spec = default_spec(
|
|
redis_mode="disabled",
|
|
mail_mode="disabled",
|
|
module_set="full",
|
|
)
|
|
write_env(
|
|
paths.env,
|
|
reconcile_runtime_environment(second_spec, first_environment),
|
|
)
|
|
second_plan = build_plan(second_spec, paths, include_host_checks=False)
|
|
removed = {
|
|
action.target
|
|
for action in second_plan.actions
|
|
if action.action == "remove"
|
|
}
|
|
|
|
self.assertEqual(
|
|
{"redis", "worker", "scheduler", "test-mail"},
|
|
removed,
|
|
)
|
|
impacts = {
|
|
item.capability_id: item
|
|
for item in second_plan.capability_impacts
|
|
}
|
|
self.assertEqual("remove", impacts["coordination.redis"].action)
|
|
self.assertEqual("remove", impacts["mail.smtp"].action)
|
|
self.assertIn("mail", impacts["mail.smtp"].dependent_modules)
|
|
self.assertTrue(
|
|
any(
|
|
check.id == "capability.change.mail.smtp"
|
|
and check.level == "warning"
|
|
for check in second_plan.checks
|
|
)
|
|
)
|
|
self.assertTrue(second_plan.blocked)
|
|
self.assertTrue(
|
|
any(
|
|
check.id == "capability.dependency_inventory.missing"
|
|
and check.level == "error"
|
|
for check in second_plan.checks
|
|
)
|
|
)
|
|
|
|
atomic_write(
|
|
paths.dependency_inventory,
|
|
canonical_json(_dependency_inventory(second_spec.installation_id)),
|
|
mode=0o600,
|
|
)
|
|
evidenced_plan = build_plan(
|
|
second_spec,
|
|
paths,
|
|
include_host_checks=False,
|
|
)
|
|
|
|
self.assertFalse(
|
|
any(
|
|
check.level == "error"
|
|
and check.id.startswith("capability.dependency_inventory.")
|
|
for check in evidenced_plan.checks
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
"endpoint:17",
|
|
{
|
|
item.capability_id: item
|
|
for item in evidenced_plan.capability_impacts
|
|
}["mail.smtp"].actual_dependencies[0].dependency_ref,
|
|
)
|
|
self.assertTrue(
|
|
any(
|
|
check.id == "capability.dependency_inventory.current"
|
|
and check.level == "ok"
|
|
for check in evidenced_plan.checks
|
|
)
|
|
)
|
|
|
|
stale = _dependency_inventory(
|
|
second_spec.installation_id,
|
|
generated_at=datetime.now(UTC) - timedelta(minutes=6),
|
|
)
|
|
atomic_write(
|
|
paths.dependency_inventory,
|
|
canonical_json(stale),
|
|
mode=0o600,
|
|
)
|
|
stale_plan = build_plan(second_spec, paths, include_host_checks=False)
|
|
self.assertTrue(stale_plan.blocked)
|
|
self.assertTrue(
|
|
any(
|
|
check.id == "capability.dependency_inventory.stale"
|
|
for check in stale_plan.checks
|
|
)
|
|
)
|
|
|
|
def test_secret_change_is_planned_without_exposing_secret_values(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
paths = bundle_paths(Path(directory))
|
|
paths.root.chmod(0o700)
|
|
spec = default_spec()
|
|
values = initial_secrets(spec)
|
|
write_env(paths.env, values)
|
|
first = build_plan(spec, paths, include_host_checks=False)
|
|
atomic_write(
|
|
paths.receipt,
|
|
canonical_json(
|
|
{
|
|
"spec_sha256": first.desired_spec_sha256,
|
|
"compose_sha256": first.desired_compose_sha256,
|
|
"environment_fingerprint": (
|
|
first.desired_environment_fingerprint
|
|
),
|
|
"services": list(render_compose(spec)["services"]),
|
|
}
|
|
),
|
|
mode=0o600,
|
|
)
|
|
rotated = dict(values)
|
|
rotated["REDIS_PASSWORD"] = "rotated-high-entropy-value"
|
|
write_env(paths.env, rotated)
|
|
|
|
second = build_plan(spec, paths, include_host_checks=False)
|
|
plan_json = json.dumps(second.to_dict())
|
|
|
|
self.assertIn(
|
|
("reconfigure", "environment"),
|
|
{(action.action, action.target) for action in second.actions},
|
|
)
|
|
self.assertNotIn(rotated["REDIS_PASSWORD"], plan_json)
|
|
self.assertEqual(
|
|
environment_fingerprint(rotated),
|
|
second.desired_environment_fingerprint,
|
|
)
|
|
|
|
def test_external_endpoint_preflight_reports_reachability(self) -> None:
|
|
connection = MagicMock()
|
|
connection.__enter__.return_value = connection
|
|
with patch.object(
|
|
deployment_planning.socket,
|
|
"create_connection",
|
|
return_value=connection,
|
|
) as connect:
|
|
check = _endpoint_check(
|
|
"external.redis",
|
|
"External Redis",
|
|
"rediss://redis.example.test/0",
|
|
default_ports={"redis": 6379, "rediss": 6380},
|
|
)
|
|
|
|
self.assertEqual("ok", check.level)
|
|
connect.assert_called_once_with(
|
|
("redis.example.test", 6380),
|
|
timeout=1.5,
|
|
)
|
|
|
|
def test_cli_init_writes_private_idempotent_bundle(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
root = Path(directory) / "installation"
|
|
result, _stdout, _stderr = run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
"--redis",
|
|
"disabled",
|
|
"--mail",
|
|
"test-mail",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(0, result)
|
|
self.assertEqual(0o700, stat.S_IMODE(root.stat().st_mode))
|
|
for name in (
|
|
"installation.json",
|
|
"secrets.env",
|
|
"compose.json",
|
|
"plan.json",
|
|
):
|
|
self.assertEqual(0o600, stat.S_IMODE((root / name).stat().st_mode))
|
|
for name in ("garage.toml", "load-balancer.cfg"):
|
|
self.assertEqual(
|
|
0o644,
|
|
stat.S_IMODE((root / name).stat().st_mode),
|
|
)
|
|
values = read_env(root / "secrets.env")
|
|
self.assertTrue(values["MASTER_KEY_B64"])
|
|
self.assertNotIn(
|
|
values["POSTGRES_PASSWORD"],
|
|
(root / "compose.json").read_text(encoding="utf-8"),
|
|
)
|
|
self.assertEqual(
|
|
1,
|
|
run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
]
|
|
)[0],
|
|
)
|
|
|
|
def test_cli_collects_bounded_private_dependency_inventory(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
root = Path(directory) / "installation"
|
|
self.assertEqual(
|
|
0,
|
|
run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
]
|
|
)[0],
|
|
)
|
|
payload = _dependency_inventory("govoplan-local")
|
|
response = MagicMock()
|
|
response.__enter__.return_value = response
|
|
response.geturl.return_value = "https://ops.example.test/inventory"
|
|
response.read.return_value = json.dumps(payload).encode("utf-8")
|
|
fetch = MagicMock(return_value=response)
|
|
|
|
with (
|
|
patch.dict(os.environ, {"TEST_OPS_KEY": "secret-api-key"}),
|
|
patch.object(deployment_cli, "urlopen", fetch),
|
|
):
|
|
result, stdout, stderr = run_cli(
|
|
[
|
|
"collect-infrastructure-inventory",
|
|
"--directory",
|
|
str(root),
|
|
"--ops-url",
|
|
"https://ops.example.test/inventory",
|
|
"--api-key-env",
|
|
"TEST_OPS_KEY",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(0, result, stderr)
|
|
self.assertIn("1 record(s)", stdout)
|
|
evidence_path = root / "infrastructure-dependency-inventory.json"
|
|
self.assertEqual(0o600, stat.S_IMODE(evidence_path.stat().st_mode))
|
|
self.assertNotIn(
|
|
"secret-api-key",
|
|
evidence_path.read_text(encoding="utf-8"),
|
|
)
|
|
request = fetch.call_args.args[0]
|
|
self.assertEqual("secret-api-key", request.get_header("X-api-key"))
|
|
|
|
def test_cli_requires_external_url_when_switching_from_managed(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
root = Path(directory) / "installation"
|
|
self.assertEqual(
|
|
0,
|
|
run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
]
|
|
)[0],
|
|
)
|
|
self.assertEqual(
|
|
1,
|
|
run_cli(
|
|
[
|
|
"configure",
|
|
"--directory",
|
|
str(root),
|
|
"--postgres",
|
|
"external",
|
|
]
|
|
)[0],
|
|
)
|
|
self.assertEqual(
|
|
"managed",
|
|
json.loads((root / "installation.json").read_text(encoding="utf-8"))[
|
|
"components"
|
|
]["postgres"]["mode"],
|
|
)
|
|
self.assertEqual(
|
|
1,
|
|
run_cli(
|
|
[
|
|
"configure",
|
|
"--directory",
|
|
str(root),
|
|
"--installation-id",
|
|
"renamed-installation",
|
|
]
|
|
)[0],
|
|
)
|
|
|
|
def test_cli_requires_explicit_external_s3_values_after_garage(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
root = Path(directory) / "installation"
|
|
self.assertEqual(
|
|
0,
|
|
run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
"--storage",
|
|
"garage",
|
|
]
|
|
)[0],
|
|
)
|
|
|
|
result, _stdout, stderr = run_cli(
|
|
[
|
|
"configure",
|
|
"--directory",
|
|
str(root),
|
|
"--storage",
|
|
"s3",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(1, result)
|
|
self.assertIn("switching to external S3 requires", stderr)
|
|
self.assertEqual(
|
|
"garage",
|
|
json.loads((root / "installation.json").read_text(encoding="utf-8"))[
|
|
"components"
|
|
]["storage"]["mode"],
|
|
)
|
|
|
|
def test_deployer_builds_and_runs_as_one_zipapp(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
output = Path(directory) / "govoplan-deploy.pyz"
|
|
build = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(DEPLOYMENT_TOOLS / "build-deployer-zipapp.py"),
|
|
"--output",
|
|
str(output),
|
|
],
|
|
cwd=META_ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
help_result = subprocess.run(
|
|
[sys.executable, str(output), "--help"],
|
|
cwd=META_ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
install_root = Path(directory) / "install"
|
|
init_result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(output),
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(install_root),
|
|
],
|
|
cwd=META_ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
render_result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(output),
|
|
"render",
|
|
"--directory",
|
|
str(install_root),
|
|
"--json",
|
|
],
|
|
cwd=META_ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
self.assertEqual(0, build.returncode, build.stderr)
|
|
self.assertTrue(output.exists())
|
|
self.assertEqual(0o755, stat.S_IMODE(output.stat().st_mode))
|
|
self.assertEqual(0, help_result.returncode, help_result.stderr)
|
|
self.assertIn("govoplan-deploy", help_result.stdout)
|
|
self.assertEqual(0, init_result.returncode, init_result.stderr)
|
|
self.assertEqual(1, render_result.returncode, render_result.stderr)
|
|
self.assertTrue(json.loads(render_result.stdout)["blocked"])
|
|
|
|
def test_generated_environment_passes_core_startup_validation_when_available(
|
|
self,
|
|
) -> None:
|
|
try:
|
|
from govoplan_core.core.install_config import (
|
|
validate_runtime_configuration,
|
|
)
|
|
except ModuleNotFoundError:
|
|
self.skipTest("govoplan-core is not installed in this test environment")
|
|
|
|
for profile, public_url in (
|
|
("evaluation", "http://127.0.0.1:8080"),
|
|
("self-hosted", "https://govoplan.example.test"),
|
|
):
|
|
with self.subTest(profile=profile):
|
|
environment = initial_secrets(
|
|
default_spec(profile=profile, public_url=public_url)
|
|
)
|
|
validation = validate_runtime_configuration(environment)
|
|
self.assertEqual(
|
|
(),
|
|
validation.errors,
|
|
validation.to_text(),
|
|
)
|
|
|
|
def test_apply_orders_dependencies_migration_and_runtime_before_receipt(
|
|
self,
|
|
) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
root = Path(directory) / "installation"
|
|
self.assertEqual(
|
|
0,
|
|
run_cli(
|
|
[
|
|
"init",
|
|
"--non-interactive",
|
|
"--directory",
|
|
str(root),
|
|
"--api-image",
|
|
"local/govoplan-api:test",
|
|
"--web-image",
|
|
"local/govoplan-web:test",
|
|
]
|
|
)[0],
|
|
)
|
|
commands: list[list[str]] = []
|
|
|
|
def record_command(argv, *, cwd):
|
|
self.assertEqual(root, cwd)
|
|
commands.append(list(argv))
|
|
|
|
compose_version = subprocess.CompletedProcess(
|
|
args=["docker", "compose", "version"],
|
|
returncode=0,
|
|
stdout="2.30.0\n",
|
|
stderr="",
|
|
)
|
|
with (
|
|
patch.object(
|
|
deployment_cli.shutil,
|
|
"which",
|
|
return_value="/usr/bin/docker",
|
|
),
|
|
patch.object(
|
|
deployment_planning.shutil,
|
|
"which",
|
|
return_value="/usr/bin/docker",
|
|
),
|
|
patch.object(
|
|
deployment_planning,
|
|
"_run_command",
|
|
return_value=compose_version,
|
|
),
|
|
patch.object(deployment_cli, "_run", side_effect=record_command),
|
|
patch.object(deployment_cli, "_wait_for_health") as wait_for_health,
|
|
):
|
|
result, _stdout, _stderr = run_cli(
|
|
[
|
|
"apply",
|
|
"--directory",
|
|
str(root),
|
|
"--skip-pull",
|
|
"--allow-unverified-images",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(0, result)
|
|
migrate_index = next(
|
|
index
|
|
for index, command in enumerate(commands)
|
|
if command[-3:] == ["run", "--rm", "migrate"]
|
|
)
|
|
runtime_index = next(
|
|
index
|
|
for index, command in enumerate(commands)
|
|
if "--remove-orphans" in command
|
|
)
|
|
stop_index = next(
|
|
index
|
|
for index, command in enumerate(commands)
|
|
if "stop" in command and "--timeout" in command
|
|
)
|
|
self.assertLess(stop_index, migrate_index)
|
|
self.assertLess(migrate_index, runtime_index)
|
|
self.assertIn("postgres", commands[0])
|
|
self.assertIn("redis", commands[0])
|
|
wait_for_health.assert_called_once_with(
|
|
"http://127.0.0.1:8080/health/ready",
|
|
timeout_seconds=120.0,
|
|
)
|
|
receipt = json.loads((root / "receipt.json").read_text(encoding="utf-8"))
|
|
self.assertEqual("govoplan-local", receipt["installation_id"])
|
|
self.assertEqual(
|
|
{"address": "127.0.0.1", "port": 8080},
|
|
receipt["listen"],
|
|
)
|
|
self.assertNotIn("installer", receipt["services"])
|
|
self.assertEqual(
|
|
1,
|
|
receipt["infrastructure_capabilities"]["schema_version"],
|
|
)
|
|
self.assertTrue((root / "infrastructure-capabilities.json").is_file())
|
|
|
|
def test_installation_root_symlink_is_rejected(self) -> None:
|
|
if not hasattr(Path, "symlink_to"):
|
|
self.skipTest("symbolic links are unavailable")
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
base = Path(directory)
|
|
target = base / "real"
|
|
target.mkdir()
|
|
link = base / "linked"
|
|
link.symlink_to(target, target_is_directory=True)
|
|
|
|
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
|
bundle_paths(link)
|
|
|
|
def test_legacy_receipt_requests_direct_web_port_handoff(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
|
|
receipt = Path(directory) / "receipt.json"
|
|
receipt.write_text(
|
|
json.dumps({"services": ["api", "web", "worker"]}),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertTrue(_receipt_uses_direct_web_port(receipt))
|
|
|
|
receipt.write_text(
|
|
json.dumps(
|
|
{
|
|
"services": [
|
|
"api",
|
|
"web",
|
|
"load-balancer",
|
|
"worker",
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertFalse(_receipt_uses_direct_web_port(receipt))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|