Add scalable deployment planning and guided releases

tools/deployment/govoplan_deploy/cli.py

@@ -30,13 +30,18 @@ from .bundle import (
     read_env,
     reconcile_runtime_environment,
     render_compose,
+    render_garage_config,
+    render_load_balancer_config,
     service_names,
     write_env,
 )
 from .model import (
     ComponentConfig,
+    DEFAULT_GARAGE_IMAGE,
+    DEFAULT_LOAD_BALANCER_IMAGE,
     InstallationSpec,
     ListenConfig,
+    ReplicaConfig,
     SpecError,
     default_spec,
     load_spec,
@@ -117,7 +122,12 @@ def _directory_argument(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--directory",
         type=Path,
-        default=Path.home() / ".local" / "share" / "govoplan" / "installations" / "default",
+        default=Path.home()
+        / ".local"
+        / "share"
+        / "govoplan"
+        / "installations"
+        / "default",
         help="Private installation state directory.",
     )
 
@@ -140,7 +150,9 @@ def _configuration_arguments(
         choices=("managed", "external"),
         default=default("managed"),
     )
-    parser.add_argument("--database-url", help="External PostgreSQL URL; stored privately.")
+    parser.add_argument(
+        "--database-url", help="External PostgreSQL URL; stored privately."
+    )
     parser.add_argument(
         "--redis",
         choices=("managed", "external", "disabled"),
@@ -154,17 +166,47 @@ def _configuration_arguments(
     )
     parser.add_argument(
         "--storage",
-        choices=("local", "s3"),
+        choices=("local", "garage", "s3"),
         default=default("local"),
     )
+    parser.add_argument(
+        "--garage-image",
+        default=default(DEFAULT_GARAGE_IMAGE),
+        help="Garage image used by managed object storage.",
+    )
     parser.add_argument("--s3-endpoint-url", help="S3-compatible endpoint URL.")
     parser.add_argument("--s3-region", help="S3 region.")
-    parser.add_argument("--s3-access-key-id", help="S3 access key id; stored privately.")
+    parser.add_argument(
+        "--s3-access-key-id", help="S3 access key id; stored privately."
+    )
     parser.add_argument(
         "--s3-secret-access-key",
         help="S3 secret access key; stored privately.",
     )
     parser.add_argument("--s3-bucket", help="S3 bucket for managed files.")
+    parser.add_argument(
+        "--load-balancer-image",
+        default=default(DEFAULT_LOAD_BALANCER_IMAGE),
+        help="HAProxy image used by the managed local load balancer.",
+    )
+    parser.add_argument(
+        "--api-replicas",
+        type=int,
+        default=default(1),
+        help="API containers on this Compose host (1-64).",
+    )
+    parser.add_argument(
+        "--web-replicas",
+        type=int,
+        default=default(1),
+        help="WebUI containers on this Compose host (1-64).",
+    )
+    parser.add_argument(
+        "--worker-replicas",
+        type=int,
+        default=None,
+        help="Worker containers on this Compose host (1-128, or 0 without Redis).",
+    )
     parser.add_argument(
         "--module-set",
         choices=("core", "base", "full"),
@@ -216,6 +258,11 @@ def _init(args: argparse.Namespace) -> int:
         redis_mode=args.redis,
         mail_mode=args.mail,
         storage_mode=args.storage,
+        garage_image=args.garage_image,
+        load_balancer_image=args.load_balancer_image,
+        api_replicas=args.api_replicas,
+        web_replicas=args.web_replicas,
+        worker_replicas=args.worker_replicas,
         module_set=args.module_set,
         api_image=args.api_image,
         web_image=args.web_image,
@@ -260,9 +307,23 @@ def _configure(args: argparse.Namespace) -> int:
         and spec.components.redis.mode == "external"
         and not args.redis_url
     ):
-        raise ValueError(
-            "switching Redis to external requires --redis-url"
-        )
+        raise ValueError("switching Redis to external requires --redis-url")
+    if current.components.storage.mode != "s3" and spec.components.storage.mode == "s3":
+        missing_s3_options = [
+            flag
+            for flag, value in (
+                ("--s3-endpoint-url", args.s3_endpoint_url),
+                ("--s3-region", args.s3_region),
+                ("--s3-access-key-id", args.s3_access_key_id),
+                ("--s3-secret-access-key", args.s3_secret_access_key),
+                ("--s3-bucket", args.s3_bucket),
+            )
+            if not value
+        ]
+        if missing_s3_options:
+            raise ValueError(
+                "switching to external S3 requires: " + ", ".join(missing_s3_options)
+            )
     secrets = read_env(paths.env)
     secrets.update(_supplied_secret_values(args))
     secrets = reconcile_runtime_environment(spec, secrets)
@@ -318,6 +379,8 @@ def _apply(args: argparse.Namespace) -> int:
                         "components.postgres.image.",
                         "components.redis.image.",
                         "components.mail.image.",
+                        "components.storage.image.",
+                        "components.load_balancer.image.",
                         "release.manifest",
                         "modules.image_composition",
                     )
@@ -344,15 +407,17 @@ def _apply(args: argparse.Namespace) -> int:
             _run([*compose, "pull"], cwd=paths.root)
         dependencies = [
             name
-            for name in ("postgres", "redis", "test-mail")
+            for name in ("postgres", "redis", "garage", "test-mail")
             if name in service_names(spec)
         ]
         if dependencies:
             _run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
         _run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
+        if _receipt_uses_direct_web_port(paths.receipt):
+            _run([*compose, "stop", "web"], cwd=paths.root)
         runtime_services = [
             name
-            for name in ("api", "web", "worker", "scheduler")
+            for name in ("api", "web", "load-balancer", "worker", "scheduler")
             if name in service_names(spec)
         ]
         _run(
@@ -378,6 +443,11 @@ def _apply(args: argparse.Namespace) -> int:
                 "web_image": spec.release.web_image,
             },
             "services": list(service_names(spec)),
+            "replicas": {
+                "api": spec.replicas.api,
+                "web": spec.replicas.web,
+                "worker": spec.replicas.worker,
+            },
             "listen": {
                 "address": spec.listen.address,
                 "port": spec.listen.port,
@@ -470,6 +540,7 @@ def _updated_spec(
         api_image=args.api_image or current.release.api_image,
         web_image=args.web_image or current.release.web_image,
     )
+    storage_mode = args.storage or current.components.storage.mode
     components = ComponentConfig(
         postgres=replace(
             current.components.postgres,
@@ -485,7 +556,35 @@ def _updated_spec(
         ),
         storage=replace(
             current.components.storage,
-            mode=args.storage or current.components.storage.mode,
+            mode=storage_mode,
+            image=(
+                args.garage_image
+                or current.components.storage.image
+                or DEFAULT_GARAGE_IMAGE
+            )
+            if storage_mode == "garage"
+            else "",
+        ),
+        load_balancer=replace(
+            current.components.load_balancer,
+            image=(args.load_balancer_image or current.components.load_balancer.image),
+        ),
+    )
+    replicas = ReplicaConfig(
+        api=(
+            args.api_replicas if args.api_replicas is not None else current.replicas.api
+        ),
+        web=(
+            args.web_replicas if args.web_replicas is not None else current.replicas.web
+        ),
+        worker=(
+            args.worker_replicas
+            if args.worker_replicas is not None
+            else (
+                0
+                if components.redis.mode == "disabled"
+                else max(current.replicas.worker, 1)
+            )
         ),
     )
     value = replace(
@@ -499,6 +598,7 @@ def _updated_spec(
         ),
         release=release,
         components=components,
+        replicas=replicas,
         enabled_modules=modules,
     )
     return parse_spec(value.to_dict())
@@ -514,9 +614,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]:
     s3_values = {
         "FILE_STORAGE_S3_ENDPOINT_URL": getattr(args, "s3_endpoint_url", None),
         "FILE_STORAGE_S3_REGION": getattr(args, "s3_region", None),
-        "FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(
-            args, "s3_access_key_id", None
-        ),
+        "FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(args, "s3_access_key_id", None),
         "FILE_STORAGE_S3_SECRET_ACCESS_KEY": getattr(
             args, "s3_secret_access_key", None
         ),
@@ -528,9 +626,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]:
 
 def _pgtools_url(database_url: str) -> str:
     if database_url.startswith("postgresql+psycopg://"):
-        return "postgresql://" + database_url.removeprefix(
-            "postgresql+psycopg://"
-        )
+        return "postgresql://" + database_url.removeprefix("postgresql+psycopg://")
     return database_url
 
 
@@ -543,6 +639,16 @@ def _write_bundle(
     atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
     write_env(paths.env, secrets)
     atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
+    atomic_write(
+        paths.load_balancer_config,
+        render_load_balancer_config(spec).encode("utf-8"),
+        mode=0o644,
+    )
+    atomic_write(
+        paths.garage_config,
+        render_garage_config().encode("utf-8"),
+        mode=0o644,
+    )
 
 
 def _write_plan(path: Path, plan: DeploymentPlan) -> None:
@@ -570,9 +676,7 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
     if args.profile == "self-hosted" and args.public_url.startswith("http://"):
         args.public_url = "https://govoplan.example.org"
     args.public_url = _prompt("Public URL", args.public_url)
-    args.postgres = _prompt_choice(
-        "PostgreSQL", args.postgres, ("managed", "external")
-    )
+    args.postgres = _prompt_choice("PostgreSQL", args.postgres, ("managed", "external"))
     if args.postgres == "external" and not args.database_url:
         args.database_url = getpass.getpass(
             "External PostgreSQL URL (input hidden): "
@@ -582,13 +686,9 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
         if args.profile == "self-hosted"
         else ("managed", "external", "disabled")
     )
-    args.redis = _prompt_choice(
-        "Redis", args.redis, redis_choices
-    )
+    args.redis = _prompt_choice("Redis", args.redis, redis_choices)
     if args.redis == "external" and not args.redis_url:
-        args.redis_url = getpass.getpass(
-            "External Redis URL (input hidden): "
-        ).strip()
+        args.redis_url = getpass.getpass("External Redis URL (input hidden): ").strip()
     mail_choices = (
         ("disabled", "external-relay")
         if args.profile == "self-hosted"
@@ -599,20 +699,43 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
         args.mail,
         mail_choices,
     )
-    args.storage = _prompt_choice("File storage", args.storage, ("local", "s3"))
+    args.storage = _prompt_choice(
+        "File storage",
+        args.storage,
+        ("local", "garage", "s3"),
+    )
     if args.storage == "s3":
         args.s3_endpoint_url = args.s3_endpoint_url or _prompt(
             "S3 endpoint URL", "https://s3.example.org"
         )
         args.s3_region = args.s3_region or _prompt("S3 region", "eu-central-1")
-        args.s3_access_key_id = args.s3_access_key_id or _prompt(
-            "S3 access key id", ""
-        )
+        args.s3_access_key_id = args.s3_access_key_id or _prompt("S3 access key id", "")
         args.s3_secret_access_key = (
             args.s3_secret_access_key
             or getpass.getpass("S3 secret access key (input hidden): ").strip()
         )
         args.s3_bucket = args.s3_bucket or _prompt("S3 bucket", "govoplan-files")
+    args.api_replicas = _prompt_integer(
+        "API replicas on this host",
+        args.api_replicas,
+        minimum=1,
+        maximum=64,
+    )
+    args.web_replicas = _prompt_integer(
+        "WebUI replicas on this host",
+        args.web_replicas,
+        minimum=1,
+        maximum=64,
+    )
+    if args.redis != "disabled":
+        args.worker_replicas = _prompt_integer(
+            "Worker replicas on this host",
+            args.worker_replicas or 1,
+            minimum=1,
+            maximum=128,
+        )
+    else:
+        args.worker_replicas = 0
     args.module_set = _prompt_choice(
         "Initial module set", args.module_set, ("core", "base", "full")
     )
@@ -631,6 +754,24 @@ def _prompt_choice(label: str, default: str, choices: tuple[str, ...]) -> str:
         print(f"Choose one of: {', '.join(choices)}")
 
 
+def _prompt_integer(
+    label: str,
+    default: int,
+    *,
+    minimum: int,
+    maximum: int,
+) -> int:
+    while True:
+        raw = _prompt(label, str(default))
+        try:
+            value = int(raw)
+        except ValueError:
+            value = minimum - 1
+        if minimum <= value <= maximum:
+            return value
+        print(f"Choose a number between {minimum} and {maximum}.")
+
+
 @contextmanager
 def _deployment_lock(path: Path) -> Iterator[None]:
     descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
@@ -638,7 +779,9 @@ def _deployment_lock(path: Path) -> Iterator[None]:
         try:
             fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
         except BlockingIOError as exc:
-            raise ValueError("another deployment operation owns the installation lock") from exc
+            raise ValueError(
+                "another deployment operation owns the installation lock"
+            ) from exc
         os.ftruncate(descriptor, 0)
         os.write(descriptor, f"{os.getpid()}\n".encode("ascii"))
         os.fsync(descriptor)
@@ -686,7 +829,22 @@ def _parse_compose_ps(output: str) -> list[object]:
     return value if isinstance(value, list) else [value]
 
 
+def _receipt_uses_direct_web_port(path: Path) -> bool:
+    if not path.exists():
+        return False
+    try:
+        receipt = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return False
+    services = receipt.get("services") if isinstance(receipt, dict) else None
+    return (
+        isinstance(services, list)
+        and "web" in services
+        and "load-balancer" not in services
+    )
+
+
 def _now() -> str:
-    return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace(
-        "+00:00", "Z"
+    return (
+        datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
     )