Add scalable deployment planning and guided releases

2026-07-31 02:49:03 +02:00
parent 3864ce28b1
commit 908090dd0f
13 changed files with 1511 additions and 228 deletions
@@ -26,9 +26,10 @@ from govoplan_deploy.bundle import (  # noqa: E402
    read_env,
    reconcile_runtime_environment,
    render_compose,
+    render_load_balancer_config,
    write_env,
 )
-from govoplan_deploy.cli import main  # noqa: E402
+from govoplan_deploy.cli import _receipt_uses_direct_web_port, main  # noqa: E402
 import govoplan_deploy.cli as deployment_cli  # noqa: E402
 from govoplan_deploy.model import (  # noqa: E402
    SpecError,
@@ -57,11 +58,15 @@ class DeploymentInstallerTests(unittest.TestCase):
        self.assertIn("postgres", compose["services"])
        self.assertIn("redis", compose["services"])
        self.assertIn("worker", compose["services"])
+        self.assertIn("load-balancer", compose["services"])
        self.assertNotIn("test-mail", compose["services"])
        self.assertEqual(
            ["127.0.0.1:8080:8080"],
-            compose["services"]["web"]["ports"],
+            compose["services"]["load-balancer"]["ports"],
        )
+        self.assertNotIn("ports", compose["services"]["web"])
+        self.assertEqual(1, compose["services"]["api"]["scale"])
+        self.assertEqual(1, compose["services"]["web"]["scale"])

    def test_disabled_redis_removes_workers_and_sets_single_process_acknowledgement(
        self,
@@ -74,9 +79,7 @@ class DeploymentInstallerTests(unittest.TestCase):
        self.assertNotIn("worker", compose["services"])
        self.assertNotIn("scheduler", compose["services"])
        self.assertEqual("false", values["CELERY_ENABLED"])
-        self.assertEqual(
-            "true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"]
-        )
+        self.assertEqual("true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"])

    def test_self_hosted_rejects_insecure_or_test_only_choices(self) -> None:
        with self.assertRaisesRegex(SpecError, "must use HTTPS"):
@@ -149,9 +152,7 @@ class DeploymentInstallerTests(unittest.TestCase):
                "REDIS_URL": "rediss://:secret@redis.example.test/0",
            },
        )
-        self.assertEqual(
-            "rediss://:secret@redis.example.test/0", values["REDIS_URL"]
-        )
+        self.assertEqual("rediss://:secret@redis.example.test/0", values["REDIS_URL"])

    def test_s3_requires_complete_private_configuration_and_https_in_production(
        self,
@@ -169,9 +170,7 @@ class DeploymentInstallerTests(unittest.TestCase):
        }
        self.assertEqual(
            "s3",
-            initial_secrets(evaluation, supplied=supplied)[
-                "FILE_STORAGE_BACKEND"
-            ],
+            initial_secrets(evaluation, supplied=supplied)["FILE_STORAGE_BACKEND"],
        )
        production = default_spec(
            profile="self-hosted",
@@ -181,6 +180,92 @@ class DeploymentInstallerTests(unittest.TestCase):
        with self.assertRaisesRegex(ValueError, "must use HTTPS"):
            initial_secrets(production, supplied=supplied)

+    def test_managed_garage_bootstraps_private_s3_storage(self) -> None:
+        spec = default_spec(storage_mode="garage")
+        values = initial_secrets(spec)
+        compose = render_compose(spec)
+
+        self.assertEqual("s3", values["FILE_STORAGE_BACKEND"])
+        self.assertEqual("true", values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"])
+        self.assertEqual(
+            "http://garage:3900",
+            values["FILE_STORAGE_S3_ENDPOINT_URL"],
+        )
+        self.assertEqual("garage", values["FILE_STORAGE_S3_REGION"])
+        self.assertEqual(
+            values["GARAGE_DEFAULT_ACCESS_KEY"],
+            values["FILE_STORAGE_S3_ACCESS_KEY_ID"],
+        )
+        self.assertEqual(
+            values["GARAGE_DEFAULT_SECRET_KEY"],
+            values["FILE_STORAGE_S3_SECRET_ACCESS_KEY"],
+        )
+        self.assertTrue(values["GARAGE_DEFAULT_ACCESS_KEY"].startswith("GK"))
+        self.assertEqual(34, len(values["GARAGE_DEFAULT_ACCESS_KEY"]))
+        self.assertEqual(64, len(values["GARAGE_DEFAULT_SECRET_KEY"]))
+        self.assertIn("garage", compose["services"])
+        self.assertIn("garage-meta", compose["volumes"])
+        self.assertIn("garage-data", compose["volumes"])
+        self.assertEqual(
+            ["/garage", "server", "--single-node", "--default-bucket"],
+            compose["services"]["garage"]["command"],
+        )
+        self.assertNotIn(
+            values["GARAGE_DEFAULT_SECRET_KEY"],
+            json.dumps(compose),
+        )
+
+        reconciled = reconcile_runtime_environment(spec, values)
+        self.assertEqual(
+            values["GARAGE_DEFAULT_ACCESS_KEY"],
+            reconciled["GARAGE_DEFAULT_ACCESS_KEY"],
+        )
+        self.assertEqual(
+            values["GARAGE_RPC_SECRET"],
+            reconciled["GARAGE_RPC_SECRET"],
+        )
+
+    def test_replica_counts_drive_compose_and_load_balancer_discovery(self) -> None:
+        spec = default_spec(
+            storage_mode="garage",
+            api_replicas=3,
+            web_replicas=2,
+            worker_replicas=4,
+        )
+        compose = render_compose(spec)
+        config = render_load_balancer_config(spec)
+
+        self.assertEqual(3, compose["services"]["api"]["scale"])
+        self.assertEqual(2, compose["services"]["web"]["scale"])
+        self.assertEqual(4, compose["services"]["worker"]["scale"])
+        self.assertEqual(
+            "http://load-balancer:8000",
+            compose["services"]["web"]["environment"]["GOVOPLAN_API_UPSTREAM"],
+        )
+        self.assertIn("server-template web- 2 web:8080", config)
+        self.assertIn("server-template api- 3 api:8000", config)
+
+    def test_multi_replica_runtime_requires_shared_redis(self) -> None:
+        with self.assertRaisesRegex(SpecError, "multiple API replicas"):
+            default_spec(redis_mode="disabled", api_replicas=2)
+        with self.assertRaisesRegex(SpecError, "worker must be 0"):
+            default_spec(redis_mode="disabled", worker_replicas=1)
+        with self.assertRaisesRegex(SpecError, "at least 1"):
+            default_spec(redis_mode="managed", worker_replicas=0)
+
+    def test_legacy_spec_defaults_new_topology_fields(self) -> None:
+        raw = default_spec().to_dict()
+        raw.pop("replicas")
+        raw["components"].pop("load_balancer")
+        raw["components"]["storage"].pop("image")
+
+        parsed = parse_spec(raw)
+
+        self.assertEqual(1, parsed.replicas.api)
+        self.assertEqual(1, parsed.replicas.web)
+        self.assertEqual(1, parsed.replicas.worker)
+        self.assertEqual("managed", parsed.components.load_balancer.mode)
+
    def test_compose_contains_no_secret_values(self) -> None:
        spec = default_spec()
        values = initial_secrets(spec)
@@ -199,9 +284,7 @@ class DeploymentInstallerTests(unittest.TestCase):
        postgres_environment = render_compose(spec)["services"]["postgres"][
            "environment"
        ]
-        redis_environment = render_compose(spec)["services"]["redis"][
-            "environment"
-        ]
+        redis_environment = render_compose(spec)["services"]["redis"]["environment"]
        self.assertEqual(
            {"POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"},
            set(postgres_environment),
@@ -244,9 +327,7 @@ class DeploymentInstallerTests(unittest.TestCase):
            receipt = {
                "spec_sha256": first.desired_spec_sha256,
                "compose_sha256": first.desired_compose_sha256,
-                "environment_fingerprint": (
-                    first.desired_environment_fingerprint
-                ),
+                "environment_fingerprint": (first.desired_environment_fingerprint),
                "services": list(render_compose(spec)["services"]),
            }
            atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
@@ -264,9 +345,7 @@ class DeploymentInstallerTests(unittest.TestCase):
            first_spec = default_spec(mail_mode="test-mail")
            first_environment = initial_secrets(first_spec)
            write_env(paths.env, first_environment)
-            first_plan = build_plan(
-                first_spec, paths, include_host_checks=False
-            )
+            first_plan = build_plan(first_spec, paths, include_host_checks=False)
            atomic_write(
                paths.receipt,
                canonical_json(
@@ -287,9 +366,7 @@ class DeploymentInstallerTests(unittest.TestCase):
                paths.env,
                reconcile_runtime_environment(second_spec, first_environment),
            )
-            second_plan = build_plan(
-                second_spec, paths, include_host_checks=False
-            )
+            second_plan = build_plan(second_spec, paths, include_host_checks=False)
            removed = {
                action.target
                for action in second_plan.actions
@@ -385,8 +462,11 @@ class DeploymentInstallerTests(unittest.TestCase):
                "compose.json",
                "plan.json",
            ):
+                self.assertEqual(0o600, stat.S_IMODE((root / name).stat().st_mode))
+            for name in ("garage.toml", "load-balancer.cfg"):
                self.assertEqual(
-                    0o600, stat.S_IMODE((root / name).stat().st_mode)
+                    0o644,
+                    stat.S_IMODE((root / name).stat().st_mode),
                )
            values = read_env(root / "secrets.env")
            self.assertTrue(values["MASTER_KEY_B64"])
@@ -434,9 +514,9 @@ class DeploymentInstallerTests(unittest.TestCase):
            )
            self.assertEqual(
                "managed",
-                json.loads(
-                    (root / "installation.json").read_text(encoding="utf-8")
-                )["components"]["postgres"]["mode"],
+                json.loads((root / "installation.json").read_text(encoding="utf-8"))[
+                    "components"
+                ]["postgres"]["mode"],
            )
            self.assertEqual(
                1,
@@ -451,6 +531,42 @@ class DeploymentInstallerTests(unittest.TestCase):
                )[0],
            )

+    def test_cli_requires_explicit_external_s3_values_after_garage(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
+            root = Path(directory) / "installation"
+            self.assertEqual(
+                0,
+                run_cli(
+                    [
+                        "init",
+                        "--non-interactive",
+                        "--directory",
+                        str(root),
+                        "--storage",
+                        "garage",
+                    ]
+                )[0],
+            )
+
+            result, _stdout, stderr = run_cli(
+                [
+                    "configure",
+                    "--directory",
+                    str(root),
+                    "--storage",
+                    "s3",
+                ]
+            )
+
+            self.assertEqual(1, result)
+            self.assertIn("switching to external S3 requires", stderr)
+            self.assertEqual(
+                "garage",
+                json.loads((root / "installation.json").read_text(encoding="utf-8"))[
+                    "components"
+                ]["storage"]["mode"],
+            )
+
    def test_deployer_builds_and_runs_as_one_zipapp(self) -> None:
        with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
            output = Path(directory) / "govoplan-deploy.pyz"
@@ -616,9 +732,7 @@ class DeploymentInstallerTests(unittest.TestCase):
                "http://127.0.0.1:8080/health",
                timeout_seconds=120.0,
            )
-            receipt = json.loads(
-                (root / "receipt.json").read_text(encoding="utf-8")
-            )
+            receipt = json.loads((root / "receipt.json").read_text(encoding="utf-8"))
            self.assertEqual("govoplan-local", receipt["installation_id"])
            self.assertEqual(
                {"address": "127.0.0.1", "port": 8080},
@@ -639,6 +753,30 @@ class DeploymentInstallerTests(unittest.TestCase):
            with self.assertRaisesRegex(ValueError, "symbolic link"):
                bundle_paths(link)

+    def test_legacy_receipt_requests_direct_web_port_handoff(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory:
+            receipt = Path(directory) / "receipt.json"
+            receipt.write_text(
+                json.dumps({"services": ["api", "web", "worker"]}),
+                encoding="utf-8",
+            )
+            self.assertTrue(_receipt_uses_direct_web_port(receipt))
+
+            receipt.write_text(
+                json.dumps(
+                    {
+                        "services": [
+                            "api",
+                            "web",
+                            "load-balancer",
+                            "worker",
+                        ]
+                    }
+                ),
+                encoding="utf-8",
+            )
+            self.assertFalse(_receipt_uses_direct_web_port(receipt))
+

 if __name__ == "__main__":
    unittest.main()