Add scalable deployment planning and guided releases

tools/deployment/govoplan_deploy/model.py

@@ -12,6 +12,8 @@ from urllib.parse import urlsplit
 
 
 SCHEMA_VERSION = 1
+DEFAULT_GARAGE_IMAGE = "dxflrs/garage:v2.3.0"
+DEFAULT_LOAD_BALANCER_IMAGE = "haproxy:3.2.21-alpine"
 INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$")
 ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$")
 SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
@@ -87,6 +89,7 @@ class ServiceConfig:
 @dataclass(frozen=True, slots=True)
 class StorageConfig:
     mode: str
+    image: str = ""
 
 
 @dataclass(frozen=True, slots=True)
@@ -95,6 +98,14 @@ class ComponentConfig:
     redis: ServiceConfig
     mail: ServiceConfig
     storage: StorageConfig
+    load_balancer: ServiceConfig
+
+
+@dataclass(frozen=True, slots=True)
+class ReplicaConfig:
+    api: int
+    web: int
+    worker: int
 
 
 @dataclass(frozen=True, slots=True)
@@ -107,6 +118,7 @@ class InstallationSpec:
     network_subnet: str
     release: ReleaseConfig
     components: ComponentConfig
+    replicas: ReplicaConfig
     enabled_modules: tuple[str, ...]
 
     def to_dict(self) -> dict[str, Any]:
@@ -126,6 +138,11 @@ def default_spec(
     redis_mode: str = "managed",
     mail_mode: str = "disabled",
     storage_mode: str = "local",
+    garage_image: str = DEFAULT_GARAGE_IMAGE,
+    load_balancer_image: str = DEFAULT_LOAD_BALANCER_IMAGE,
+    api_replicas: int = 1,
+    web_replicas: int = 1,
+    worker_replicas: int | None = None,
     module_set: str = "base",
     api_image: str = "govoplan-api:unpublished",
     web_image: str = "govoplan-web:unpublished",
@@ -141,6 +158,11 @@ def default_spec(
     }.get(module_set)
     if modules is None:
         raise SpecError("module_set must be core, base, or full")
+    effective_worker_replicas = (
+        (0 if redis_mode == "disabled" else 1)
+        if worker_replicas is None
+        else worker_replicas
+    )
     subnet_octet = 32 + (sum(installation_id.encode("utf-8")) % 192)
     raw = {
         "schema_version": SCHEMA_VERSION,
@@ -173,7 +195,20 @@ def default_spec(
                 "image": "greenmail/standalone:2.1.9",
                 "url_env": "",
             },
-            "storage": {"mode": storage_mode},
+            "storage": {
+                "mode": storage_mode,
+                "image": garage_image if storage_mode == "garage" else "",
+            },
+            "load_balancer": {
+                "mode": "managed",
+                "image": load_balancer_image,
+                "url_env": "",
+            },
+        },
+        "replicas": {
+            "api": api_replicas,
+            "web": web_replicas,
+            "worker": effective_worker_replicas,
         },
         "enabled_modules": list(modules),
     }
@@ -203,6 +238,7 @@ def parse_spec(raw: object) -> InstallationSpec:
             "network_subnet",
             "release",
             "components",
+            "replicas",
             "enabled_modules",
         },
         "installation",
@@ -233,11 +269,10 @@ def parse_spec(raw: object) -> InstallationSpec:
         port=_port(_integer(listen_raw, "port"), "listen.port"),
     )
 
-    network_subnet = _network(
-        _string(root, "network_subnet"), "network_subnet"
-    )
+    network_subnet = _network(_string(root, "network_subnet"), "network_subnet")
     release = _release(root.get("release"))
     components = _components(root.get("components"), profile=profile)
+    replicas = _replicas(root.get("replicas"), components=components)
 
     enabled_raw = root.get("enabled_modules")
     if not isinstance(enabled_raw, list):
@@ -248,9 +283,7 @@ def parse_spec(raw: object) -> InstallationSpec:
         if not isinstance(value, str) or not re.fullmatch(
             r"[a-z][a-z0-9_]{1,63}", value
         ):
-            raise SpecError(
-                f"enabled_modules[{index}] must be a canonical module id"
-            )
+            raise SpecError(f"enabled_modules[{index}] must be a canonical module id")
         if value in seen_modules:
             raise SpecError(f"enabled_modules contains duplicate module {value!r}")
         enabled_modules.append(value)
@@ -265,6 +298,7 @@ def parse_spec(raw: object) -> InstallationSpec:
         network_subnet=network_subnet,
         release=release,
         components=components,
+        replicas=replicas,
         enabled_modules=tuple(enabled_modules),
     )
 
@@ -296,7 +330,9 @@ def _release(raw: object) -> ReleaseConfig:
             raise SpecError("release.manifest_url must use HTTPS")
     manifest_sha256 = _optional_string(value, "manifest_sha256").lower()
     if manifest_sha256 and not SHA256_PATTERN.fullmatch(manifest_sha256):
-        raise SpecError("release.manifest_sha256 must be a lowercase SHA-256 hex digest")
+        raise SpecError(
+            "release.manifest_sha256 must be a lowercase SHA-256 hex digest"
+        )
     api_image = _image(_string(value, "api_image"), "release.api_image")
     web_image = _image(_string(value, "web_image"), "release.web_image")
     return ReleaseConfig(
@@ -311,7 +347,11 @@ def _release(raw: object) -> ReleaseConfig:
 
 def _components(raw: object, *, profile: str) -> ComponentConfig:
     value = _mapping(raw, "components")
-    _only_keys(value, {"postgres", "redis", "mail", "storage"}, "components")
+    _only_keys(
+        value,
+        {"postgres", "redis", "mail", "storage", "load_balancer"},
+        "components",
+    )
     postgres = _service(
         value.get("postgres"),
         "components.postgres",
@@ -334,14 +374,41 @@ def _components(raw: object, *, profile: str) -> ComponentConfig:
         url_env_required_for=set(),
     )
     storage_raw = _mapping(value.get("storage"), "components.storage")
-    _only_keys(storage_raw, {"mode"}, "components.storage")
-    storage = StorageConfig(mode=_choice(storage_raw, "mode", {"local", "s3"}))
+    _only_keys(storage_raw, {"mode", "image"}, "components.storage")
+    storage_mode = _choice(storage_raw, "mode", {"local", "s3", "garage"})
+    storage_image = _optional_string(storage_raw, "image")
+    if storage_mode == "garage":
+        storage_image = _image(
+            storage_image or DEFAULT_GARAGE_IMAGE,
+            "components.storage.image",
+        )
+    elif storage_image:
+        raise SpecError(
+            "components.storage.image is only valid for managed Garage storage"
+        )
+    storage = StorageConfig(mode=storage_mode, image=storage_image)
+    load_balancer = _service(
+        value.get(
+            "load_balancer",
+            {
+                "mode": "managed",
+                "image": DEFAULT_LOAD_BALANCER_IMAGE,
+                "url_env": "",
+            },
+        ),
+        "components.load_balancer",
+        modes={"managed"},
+        image_required_for={"managed"},
+        url_env_required_for=set(),
+    )
     if postgres.url_env != "DATABASE_URL":
         raise SpecError("components.postgres.url_env must be DATABASE_URL")
     if redis.url_env != "REDIS_URL":
         raise SpecError("components.redis.url_env must be REDIS_URL")
     if mail.url_env:
         raise SpecError("components.mail.url_env must be empty")
+    if load_balancer.url_env:
+        raise SpecError("components.load_balancer.url_env must be empty")
     if profile == "self-hosted" and redis.mode == "disabled":
         raise SpecError("self-hosted installations require managed or external Redis")
     if profile == "self-hosted" and mail.mode == "test-mail":
@@ -351,9 +418,36 @@ def _components(raw: object, *, profile: str) -> ComponentConfig:
         redis=redis,
         mail=mail,
         storage=storage,
+        load_balancer=load_balancer,
     )
 
 
+def _replicas(raw: object, *, components: ComponentConfig) -> ReplicaConfig:
+    if raw is None:
+        return ReplicaConfig(
+            api=1,
+            web=1,
+            worker=0 if components.redis.mode == "disabled" else 1,
+        )
+    value = _mapping(raw, "replicas")
+    _only_keys(value, {"api", "web", "worker"}, "replicas")
+    replicas = ReplicaConfig(
+        api=_bounded_integer(value, "api", minimum=1, maximum=64),
+        web=_bounded_integer(value, "web", minimum=1, maximum=64),
+        worker=_bounded_integer(value, "worker", minimum=0, maximum=128),
+    )
+    if components.redis.mode == "disabled":
+        if replicas.worker:
+            raise SpecError("replicas.worker must be 0 when Redis is disabled")
+        if replicas.api > 1:
+            raise SpecError(
+                "multiple API replicas require Redis for shared throttling and queues"
+            )
+    elif replicas.worker < 1:
+        raise SpecError("replicas.worker must be at least 1 when Redis is enabled")
+    return replicas
+
+
 def _service(
     raw: object,
     label: str,
@@ -418,6 +512,19 @@ def _integer(value: Mapping[str, Any], key: str) -> int:
     return result
 
 
+def _bounded_integer(
+    value: Mapping[str, Any],
+    key: str,
+    *,
+    minimum: int,
+    maximum: int,
+) -> int:
+    result = _integer(value, key)
+    if result < minimum or result > maximum:
+        raise SpecError(f"{key} must be between {minimum} and {maximum}")
+    return result
+
+
 def _choice(value: Mapping[str, Any], key: str, choices: set[str]) -> str:
     result = _string(value, key)
     if result not in choices:
@@ -434,9 +541,7 @@ def _http_url(value: str, label: str) -> str:
     if parsed.scheme not in {"http", "https"} or not parsed.netloc:
         raise SpecError(f"{label} must be an absolute HTTP(S) URL")
     if parsed.username or parsed.password or parsed.fragment or parsed.query:
-        raise SpecError(
-            f"{label} must not contain credentials, a query, or a fragment"
-        )
+        raise SpecError(f"{label} must not contain credentials, a query, or a fragment")
     if label == "public_url" and parsed.path not in {"", "/"}:
         raise SpecError("public_url must address the site root without a path")
     return value.rstrip("/")
@@ -470,9 +575,7 @@ def _network(value: str, label: str) -> str:
         or not any(network.subnet_of(parent) for parent in RFC1918_NETWORKS)
         or not 24 <= network.prefixlen <= 28
     ):
-        raise SpecError(
-            f"{label} must be an RFC1918 IPv4 /24 to /28 network"
-        )
+        raise SpecError(f"{label} must be an RFC1918 IPv4 /24 to /28 network")
     return str(network)