From 2c515f73c2aa7e8480d5ddc107e67218e6f5f7c6 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Mon, 3 Aug 2026 01:15:06 +0200 Subject: [PATCH] Implement supported ingress and TLS profiles --- .gitea/workflows/runtime-distribution.yml | 15 + ...NSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md | 60 ++- docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md | 2 +- docs/installation-spec.schema.json | 49 +++ tests/test_deployment_installer.py | 60 +++ tools/checks/managed-ingress-drill.py | 239 ++++++++++++ tools/deployment/govoplan_deploy/bundle.py | 145 +++++++- tools/deployment/govoplan_deploy/cli.py | 131 ++++++- tools/deployment/govoplan_deploy/model.py | 153 +++++++- tools/deployment/govoplan_deploy/planning.py | 343 +++++++++++++++++- tools/deployment/govoplan_deploy/recovery.py | 4 + tools/release/runtime/nginx.conf | 10 +- 12 files changed, 1174 insertions(+), 37 deletions(-) create mode 100644 tools/checks/managed-ingress-drill.py diff --git a/.gitea/workflows/runtime-distribution.yml b/.gitea/workflows/runtime-distribution.yml index 89c6734..422fcf3 100644 --- a/.gitea/workflows/runtime-distribution.yml +++ b/.gitea/workflows/runtime-distribution.yml @@ -27,6 +27,10 @@ on: description: Digest-pinned HAProxy image required: true type: string + managed_ingress_image: + description: Digest-pinned Caddy image + required: true + type: string garage_image: description: Digest-pinned Garage image required: true @@ -169,6 +173,7 @@ jobs: POSTGRES_IMAGE: ${{ inputs.postgres_image }} REDIS_IMAGE: ${{ inputs.redis_image }} LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }} + MANAGED_INGRESS_IMAGE: ${{ inputs.managed_ingress_image }} GARAGE_IMAGE: ${{ inputs.garage_image }} TEST_MAIL_IMAGE: ${{ inputs.test_mail_image }} run: | @@ -192,6 +197,7 @@ jobs: --dependency "postgres=$POSTGRES_IMAGE" \ --dependency "redis=$REDIS_IMAGE" \ --dependency "load_balancer=$LOAD_BALANCER_IMAGE" \ + --dependency "managed_ingress=$MANAGED_INGRESS_IMAGE" \ --dependency "garage=$GARAGE_IMAGE" \ --dependency "test_mail=$TEST_MAIL_IMAGE" \ --output-directory runtime-output/evidence \ @@ -221,6 +227,15 @@ jobs: --manifest-sha256 "$MANIFEST_SHA256" \ --trusted-keyring runtime-output/distribution-keyring.json \ --adopt + - name: Exercise the managed ingress boundary + working-directory: govoplan + env: + MANAGED_INGRESS_IMAGE: ${{ inputs.managed_ingress_image }} + LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }} + run: >- + python tools/checks/managed-ingress-drill.py + --caddy-image "$MANAGED_INGRESS_IMAGE" + --load-balancer-image "$LOAD_BALANCER_IMAGE" - name: Publish immutable Gitea release assets working-directory: govoplan env: diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index 81e073c..9eabe93 100644 --- a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -91,6 +91,8 @@ The private installation directory contains: | `compose.json` | Deterministic generated Compose definition | | `garage.toml` | Non-secret managed Garage server configuration | | `load-balancer.cfg` | Non-secret HAProxy WebUI/API discovery configuration | +| `Caddyfile` | Non-secret managed-ingress route and ACME policy | +| `existing-proxy.json` | Exact upstream, trusted-source, header, and health contract for an operator-owned proxy | | `plan.json` | Latest desired-state diff and readiness findings | | `receipt.json` | Last successfully applied immutable identities | | `distribution-manifest.json` | Canonical signed runtime/image selection adopted by the installer | @@ -184,12 +186,11 @@ a production distribution: 5. **Deployment agent.** Web updates need a separate privileged reconciler with a typed command allowlist. The API and browser must never receive the Docker socket or arbitrary shell access. -6. **Ingress and certificates.** The managed HAProxy service provides HTTP - load balancing inside the deployment boundary; it does not issue or renew - certificates. A self-hosted profile still needs an explicit choice - between an existing reverse proxy and a supported managed ingress, including - trusted-proxy boundaries, TLS certificate issuance, renewal, and health - probing through the public route. +6. **Ingress reachability evidence.** Managed Caddy ingress and the + existing-proxy contract are implemented. A production claim still requires + running `doctor` from the target host after public DNS/firewall changes and + retaining the first successful container drill and public TLS/readiness + evidence. `apply --allow-unverified-images` is therefore restricted to the evaluation profile. It explicitly acknowledges both mutable image identities and @@ -263,7 +264,9 @@ must use a tested multi-node Garage cluster or another external S3 service. ### Load Balancing And Replicas -The generated Compose topology publishes only `load-balancer`. HAProxy uses +The generated Compose topology publishes only `load-balancer` for local or +existing-proxy profiles. With managed ingress, only Caddy publishes host ports +and HAProxy remains private. HAProxy uses Docker DNS service discovery to distribute public traffic across WebUI replicas and WebUI API proxy traffic across API replicas. The WebUI and API services do not publish host ports. HAProxy has no Docker socket and discovers only the @@ -287,6 +290,49 @@ PostgreSQL advisory lock. The Celery scheduler is run under a renewable, fencing-token lease. Multiple API replicas are rejected when Redis is disabled because distributed throttling and queued work cannot then be shared correctly. +### Public Ingress And TLS + +A self-hosted installation is fail-closed until one of these boundaries is +selected: + +- `existing-proxy` publishes HAProxy at `listen.address:listen.port` and emits + `existing-proxy.json`. The operator-owned proxy must use the recorded host, + upstream, and health paths. Only the exact CIDRs listed with repeated + `--trusted-proxy-cidr` values may supply `X-Forwarded-*` headers. Public + proxy addresses must be `/32` or `/128`; private ranges are limited to `/24` + or narrower for IPv4 and `/64` or narrower for IPv6. +- `managed` publishes Caddy on the selected HTTP/HTTPS ports, redirects HTTP to + HTTPS, obtains and renews certificates through ACME, and keeps certificate + material exclusively in the private `caddy-data` and `caddy-config` volumes. + The application containers receive no ACME account or TLS private keys. + +Example existing-proxy configuration: + +```sh +python govoplan-deploy.py configure \ + --directory /srv/govoplan \ + --ingress existing-proxy \ + --trusted-proxy-cidr 172.20.0.7/32 +``` + +Example managed configuration: + +```sh +python govoplan-deploy.py configure \ + --directory /srv/govoplan \ + --ingress managed \ + --acme-email operator@example.org +``` + +Before managed ingress starts, public A/AAAA records must resolve to the target +and inbound TCP 80/443 must reach it. Existing-proxy mode additionally requires +the public proxy and valid certificate to be reachable before apply. After a +successful receipt, `doctor` reports DNS resolution, certificate validity and +remaining lifetime, public `/health/ready`, and the private HAProxy/WebUI path +as separate checks. Reconfiguration retains the certificate volumes; bundle +rollback never deletes or exposes their contents. Include both Caddy volumes +in coordinated backup and restore evidence. + This is same-host scaling. Docker Compose uses a bridge network and does not place containers on another machine. See [Scaling And Multi-Host Deployment](SCALING_AND_MULTI_HOST_DEPLOYMENT.md) for diff --git a/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md index f5523dc..01a27e4 100644 --- a/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md +++ b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md @@ -193,7 +193,7 @@ fencing. It does not by itself provide: - automatic PostgreSQL backup, point-in-time recovery, or restore verification; - autoscaling policy; - central logs, metrics, traces, or alert routing; -- managed ingress certificates; +- certificate portability between independently managed ingress providers; - automatic reconciliation of every possible module side effect; - a service-level availability guarantee. diff --git a/docs/installation-spec.schema.json b/docs/installation-spec.schema.json index 0cff46d..eeba452 100644 --- a/docs/installation-spec.schema.json +++ b/docs/installation-spec.schema.json @@ -177,6 +177,55 @@ } } }, + "ingress": { + "type": "object", + "additionalProperties": false, + "required": [ + "mode", + "image", + "trusted_proxy_cidrs", + "http_port", + "https_port", + "acme_email" + ], + "properties": { + "mode": { + "enum": [ + "local", + "existing-proxy", + "managed", + "unconfigured" + ] + }, + "image": { + "type": "string", + "maxLength": 300 + }, + "trusted_proxy_cidrs": { + "type": "array", + "maxItems": 16, + "uniqueItems": true, + "items": { + "type": "string", + "maxLength": 64 + } + }, + "http_port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "https_port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "acme_email": { + "type": "string", + "maxLength": 254 + } + } + }, "enabled_modules": { "type": "array", "uniqueItems": true, diff --git a/tests/test_deployment_installer.py b/tests/test_deployment_installer.py index 9397a16..a0e1d90 100644 --- a/tests/test_deployment_installer.py +++ b/tests/test_deployment_installer.py @@ -25,7 +25,9 @@ from govoplan_deploy.bundle import ( # noqa: E402 initial_secrets, read_env, reconcile_runtime_environment, + render_caddy_config, render_compose, + render_existing_proxy_contract, render_load_balancer_config, write_env, ) @@ -509,6 +511,62 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertEqual(1, compose["services"]["api"]["scale"]) self.assertEqual(1, compose["services"]["web"]["scale"]) + def test_managed_ingress_persists_certificate_state_and_hides_upstream( + self, + ) -> None: + spec = default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + ingress_mode="managed", + acme_email="operator@example.test", + ) + compose = render_compose(spec) + ingress = compose["services"]["ingress"] + + self.assertNotIn("ports", compose["services"]["load-balancer"]) + self.assertEqual( + ["0.0.0.0:80:8080", "0.0.0.0:443:8443"], + ingress["ports"], + ) + self.assertIn("caddy-data:/data", ingress["volumes"]) + self.assertIn("caddy-config:/config", ingress["volumes"]) + self.assertIn("reverse_proxy load-balancer:8080", render_caddy_config(spec)) + self.assertNotIn("operator@example.test", json.dumps(compose)) + + def test_existing_proxy_contract_and_header_trust_are_exact(self) -> None: + spec = default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + ingress_mode="existing-proxy", + trusted_proxy_cidrs=("172.20.0.7/32",), + ) + contract = render_existing_proxy_contract(spec) + load_balancer = render_load_balancer_config(spec) + + self.assertEqual("http://127.0.0.1:8080", contract["upstream"]) + self.assertEqual(["172.20.0.7/32"], contract["trusted_proxy_cidrs"]) + self.assertIn("acl trusted_forward_proxy src 172.20.0.7/32", load_balancer) + self.assertIn("del-header X-Forwarded-Proto", load_balancer) + self.assertIn( + "del-header X-Forwarded-For unless trusted_forward_proxy", load_balancer + ) + + def test_ingress_rejects_unsafe_proxy_ranges_and_managed_ip_hosts(self) -> None: + with self.assertRaisesRegex(SpecError, "/24 or narrower"): + default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + ingress_mode="existing-proxy", + trusted_proxy_cidrs=("10.0.0.0/8",), + ) + with self.assertRaisesRegex(SpecError, "DNS hostname"): + default_spec( + profile="self-hosted", + public_url="https://192.0.2.10", + ingress_mode="managed", + acme_email="operator@example.test", + ) + def test_disabled_redis_removes_workers_and_sets_single_process_acknowledgement( self, ) -> None: @@ -713,6 +771,7 @@ class DeploymentInstallerTests(unittest.TestCase): def test_legacy_spec_defaults_new_topology_fields(self) -> None: raw = default_spec().to_dict() raw.pop("replicas") + raw.pop("ingress") raw["components"].pop("load_balancer") raw["components"]["storage"].pop("image") @@ -722,6 +781,7 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertEqual(1, parsed.replicas.web) self.assertEqual(1, parsed.replicas.worker) self.assertEqual("managed", parsed.components.load_balancer.mode) + self.assertEqual("local", parsed.ingress.mode) def test_compose_contains_no_secret_values(self) -> None: spec = default_spec() diff --git a/tools/checks/managed-ingress-drill.py b/tools/checks/managed-ingress-drill.py new file mode 100644 index 0000000..1d39aa9 --- /dev/null +++ b/tools/checks/managed-ingress-drill.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Exercise generated managed ingress with a real Caddy container.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +from uuid import uuid4 + + +META_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(META_ROOT / "tools" / "deployment")) + +from govoplan_deploy.bundle import ( # noqa: E402 + render_caddy_config, + render_load_balancer_config, +) +from govoplan_deploy.model import default_spec # noqa: E402 + + +DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") + + +def _run(argv: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + argv, + check=check, + capture_output=True, + text=True, + timeout=60, + ) + + +def _published_port(container: str, target: int) -> int: + output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip() + try: + return int(output.rsplit(":", 1)[1]) + except (IndexError, ValueError) as exc: + raise RuntimeError(f"cannot determine published port from {output!r}") from exc + + +def _curl(url: str, *, headers: bool = False) -> str: + argv = ["curl", "--silent", "--show-error", "--insecure"] + if headers: + argv.extend(["--head"]) + argv.append(url) + return _run(argv).stdout + + +def _wait_for_https(port: int) -> str: + deadline = time.monotonic() + 30 + last_error = "" + while time.monotonic() < deadline: + try: + return _curl(f"https://localhost:{port}/health") + except subprocess.CalledProcessError as exc: + last_error = exc.stderr.strip() + time.sleep(0.5) + raise RuntimeError(f"managed ingress did not become ready: {last_error}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--caddy-image", required=True) + parser.add_argument("--load-balancer-image", required=True) + args = parser.parse_args() + for label, image in ( + ("--caddy-image", args.caddy_image), + ("--load-balancer-image", args.load_balancer_image), + ): + if DIGEST_IMAGE.fullmatch(image) is None: + parser.error(f"{label} must be pinned by sha256 digest") + if shutil.which("docker") is None or shutil.which("curl") is None: + parser.error("docker and curl are required") + + suffix = uuid4().hex[:10] + network = f"govoplan-ingress-drill-{suffix}" + ingress = f"govoplan-ingress-{suffix}" + backend = f"govoplan-ingress-backend-{suffix}" + data_volume = f"govoplan-ingress-data-{suffix}" + config_volume = f"govoplan-ingress-config-{suffix}" + cleanup = [ + ["docker", "rm", "--force", ingress, backend], + ["docker", "network", "rm", network], + ["docker", "volume", "rm", data_volume, config_volume], + ] + try: + _run(["docker", "network", "create", network]) + _run(["docker", "volume", "create", data_volume]) + _run(["docker", "volume", "create", config_volume]) + with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory: + root = Path(directory) + backend_config = root / "backend.Caddyfile" + backend_config.write_text( + """{ + admin off + auto_https off +} + +:8080 { + respond /health "proto={http.request.header.X-Forwarded-Proto}" +} +""", + encoding="utf-8", + ) + backend_config.chmod(0o644) + spec = default_spec( + profile="self-hosted", + public_url="https://localhost:8443", + ingress_mode="managed", + ingress_image=args.caddy_image, + ingress_https_port=8443, + acme_email="operator@example.test", + ) + ingress_config = root / "Caddyfile" + ingress_config.write_text(render_caddy_config(spec), encoding="utf-8") + ingress_config.chmod(0o644) + load_balancer_config = root / "haproxy.cfg" + load_balancer_config.write_text( + render_load_balancer_config(spec), + encoding="utf-8", + ) + load_balancer_config.chmod(0o644) + _run( + [ + "docker", + "run", + "--rm", + "--read-only", + "--cap-drop", + "ALL", + "--mount", + f"type=bind,src={load_balancer_config},dst=/usr/local/etc/haproxy/haproxy.cfg,readonly", + args.load_balancer_image, + "haproxy", + "-c", + "-f", + "/usr/local/etc/haproxy/haproxy.cfg", + ] + ) + + _run( + [ + "docker", + "run", + "--detach", + "--name", + backend, + "--network", + network, + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=16m", + "--mount", + f"type=bind,src={backend_config},dst=/etc/caddy/Caddyfile,readonly", + args.caddy_image, + "caddy", + "run", + "--config", + "/etc/caddy/Caddyfile", + ] + ) + ingress_command = [ + "docker", + "run", + "--detach", + "--name", + ingress, + "--network", + network, + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=16m", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--publish", + "127.0.0.1::8080", + "--publish", + "127.0.0.1::8443", + "--mount", + f"type=bind,src={ingress_config},dst=/etc/caddy/Caddyfile,readonly", + "--mount", + f"type=volume,src={data_volume},dst=/data", + "--mount", + f"type=volume,src={config_volume},dst=/config", + args.caddy_image, + "caddy", + "run", + "--config", + "/etc/caddy/Caddyfile", + ] + _run(ingress_command) + http_port = _published_port(ingress, 8080) + https_port = _published_port(ingress, 8443) + body = _wait_for_https(https_port) + if body.strip() != "proto=https": + raise RuntimeError(f"forwarded protocol was not normalized: {body!r}") + redirect = _curl(f"http://localhost:{http_port}/health", headers=True) + if not redirect.startswith("HTTP/1.1 308"): + raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}") + + _run(["docker", "rm", "--force", ingress]) + _run(ingress_command) + https_port = _published_port(ingress, 8443) + if _wait_for_https(https_port).strip() != "proto=https": + raise RuntimeError( + "managed ingress did not recover with persistent state" + ) + _run( + [ + "docker", + "run", + "--rm", + "--mount", + f"type=volume,src={data_volume},dst=/data,readonly", + "--entrypoint", + "sh", + args.caddy_image, + "-c", + 'test -n "$(find /data -type f -print -quit)"', + ] + ) + finally: + for command in cleanup: + _run(command, check=False) + print("Managed ingress container drill passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deployment/govoplan_deploy/bundle.py b/tools/deployment/govoplan_deploy/bundle.py index afbbad3..d2100ea 100644 --- a/tools/deployment/govoplan_deploy/bundle.py +++ b/tools/deployment/govoplan_deploy/bundle.py @@ -22,6 +22,8 @@ ENV_FILENAME = "secrets.env" COMPOSE_FILENAME = "compose.json" GARAGE_CONFIG_FILENAME = "garage.toml" LOAD_BALANCER_CONFIG_FILENAME = "load-balancer.cfg" +CADDY_CONFIG_FILENAME = "Caddyfile" +EXISTING_PROXY_FILENAME = "existing-proxy.json" PLAN_FILENAME = "plan.json" RECEIPT_FILENAME = "receipt.json" MANIFEST_FILENAME = "distribution-manifest.json" @@ -87,6 +89,8 @@ class BundlePaths: compose: Path garage_config: Path load_balancer_config: Path + caddy_config: Path + existing_proxy: Path plan: Path receipt: Path manifest: Path @@ -106,6 +110,8 @@ def bundle_paths(root: Path) -> BundlePaths: compose=resolved / COMPOSE_FILENAME, garage_config=resolved / GARAGE_CONFIG_FILENAME, load_balancer_config=resolved / LOAD_BALANCER_CONFIG_FILENAME, + caddy_config=resolved / CADDY_CONFIG_FILENAME, + existing_proxy=resolved / EXISTING_PROXY_FILENAME, plan=resolved / PLAN_FILENAME, receipt=resolved / RECEIPT_FILENAME, manifest=resolved / MANIFEST_FILENAME, @@ -455,6 +461,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: "restart": "unless-stopped", "volumes": data_mounts, "networks": ["internal"], + "read_only": True, + "tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"], + "security_opt": ["no-new-privileges:true"], + "cap_drop": ["ALL"], } if dependency_conditions: common_runtime["depends_on"] = dependency_conditions @@ -470,6 +480,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: "restart": "no", "volumes": data_mounts, "networks": ["internal"], + "read_only": True, + "tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"], + "security_opt": ["no-new-privileges:true"], + "cap_drop": ["ALL"], **({"depends_on": dependency_conditions} if dependency_conditions else {}), } services["api"] = { @@ -515,9 +529,13 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: "restart": "unless-stopped", "scale": spec.replicas.web, "environment": {"GOVOPLAN_API_UPSTREAM": "http://load-balancer:8000"}, + "read_only": True, + "tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"], + "security_opt": ["no-new-privileges:true"], + "cap_drop": ["ALL"], "networks": ["internal"], } - services["load-balancer"] = { + load_balancer: dict[str, object] = { "image": spec.components.load_balancer.image, "restart": "unless-stopped", "healthcheck": { @@ -537,7 +555,6 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: render_load_balancer_config(spec).encode("utf-8") ).hexdigest() }, - "ports": [_published_port(spec.listen.address, spec.listen.port, 8080)], "read_only": True, "security_opt": ["no-new-privileges:true"], "volumes": [ @@ -545,6 +562,53 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: ], "networks": ["internal"], } + if spec.ingress.mode != "managed": + load_balancer["ports"] = [ + _published_port(spec.listen.address, spec.listen.port, 8080) + ] + services["load-balancer"] = load_balancer + if spec.ingress.mode == "managed": + services["ingress"] = { + "image": spec.ingress.image, + "restart": "unless-stopped", + "command": [ + "caddy", + "run", + "--config", + "/etc/caddy/Caddyfile", + "--adapter", + "caddyfile", + ], + "healthcheck": { + "test": [ + "CMD", + "caddy", + "validate", + "--config", + "/etc/caddy/Caddyfile", + "--adapter", + "caddyfile", + ], + "interval": "30s", + "timeout": "5s", + "retries": 3, + }, + "ports": [ + _published_port("0.0.0.0", spec.ingress.http_port, 8080), + _published_port("0.0.0.0", spec.ingress.https_port, 8443), + ], + "read_only": True, + "tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"], + "security_opt": ["no-new-privileges:true"], + "cap_drop": ["ALL"], + "volumes": [ + f"./{CADDY_CONFIG_FILENAME}:/etc/caddy/Caddyfile:ro", + "caddy-data:/data", + "caddy-config:/config", + ], + "networks": ["internal"], + "depends_on": {"load-balancer": {"condition": "service_healthy"}}, + } if spec.components.redis.mode != "disabled": services["worker"] = { **common_runtime, @@ -606,6 +670,9 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: if spec.components.storage.mode == "garage": volumes["garage-meta"] = {} volumes["garage-data"] = {} + if spec.ingress.mode == "managed": + volumes["caddy-data"] = {} + volumes["caddy-config"] = {} return { "name": spec.installation_id, @@ -642,6 +709,31 @@ api_bind_addr = "[::]:3903" def render_load_balancer_config(spec: InstallationSpec) -> str: health_host = urlsplit(spec.public_url).hostname or "localhost" + trusted_proxy_cidrs = ( + (spec.network_subnet,) + if spec.ingress.mode == "managed" + else spec.ingress.trusted_proxy_cidrs + if spec.ingress.mode == "existing-proxy" + else () + ) + trusted_acl = ( + " acl trusted_forward_proxy src " + " ".join(trusted_proxy_cidrs) + "\n" + if trusted_proxy_cidrs + else "" + ) + forwarded_rules = ( + " http-request set-var(txn.forwarded_proto) req.hdr(X-Forwarded-Proto) if trusted_forward_proxy\n" + " http-request del-header X-Forwarded-Proto\n" + " http-request set-header X-Forwarded-Proto https if trusted_forward_proxy { var(txn.forwarded_proto) -m str https }\n" + " http-request set-header X-Forwarded-Proto http unless { var(txn.forwarded_proto) -m str https }\n" + " http-request del-header X-Forwarded-For unless trusted_forward_proxy\n" + if trusted_proxy_cidrs + else ( + " http-request del-header X-Forwarded-Proto\n" + " http-request set-header X-Forwarded-Proto http\n" + " http-request del-header X-Forwarded-For\n" + ) + ) return f"""global log stdout format raw local0 maxconn 4096 @@ -669,6 +761,9 @@ resolvers docker frontend public_web bind :8080 +{trusted_acl}{forwarded_rules} option forwardfor + http-request del-header X-Forwarded-Host + http-request set-header X-Forwarded-Host %[req.hdr(host)] default_backend web_replicas backend web_replicas @@ -690,6 +785,52 @@ backend api_replicas """ +def render_caddy_config(spec: InstallationSpec) -> str: + if spec.ingress.mode != "managed": + return "# Managed ingress is not selected.\n" + hostname = urlsplit(spec.public_url).hostname or "" + return f"""{{ + admin off + email {spec.ingress.acme_email} + http_port 8080 + https_port 8443 +}} + +{hostname} {{ + encode zstd gzip + header {{ + Strict-Transport-Security "max-age=31536000; includeSubDomains" + }} + reverse_proxy load-balancer:8080 {{ + header_up X-Forwarded-Proto https + }} +}} +""" + + +def render_existing_proxy_contract(spec: InstallationSpec) -> dict[str, object]: + return { + "schema_version": 1, + "mode": spec.ingress.mode, + "public_url": spec.public_url, + "upstream": ( + f"http://{spec.listen.address}:{spec.listen.port}" + if spec.ingress.mode == "existing-proxy" + else None + ), + "trusted_proxy_cidrs": list(spec.ingress.trusted_proxy_cidrs), + "required_headers": { + "Host": urlsplit(spec.public_url).hostname or "", + "X-Forwarded-Proto": "https", + "X-Forwarded-For": "client, proxy chain", + }, + "health_paths": { + "load_balancer": "/health", + "api_readiness": "/health/ready", + }, + } + + def service_names(spec: InstallationSpec) -> tuple[str, ...]: return tuple(render_compose(spec)["services"].keys()) diff --git a/tools/deployment/govoplan_deploy/cli.py b/tools/deployment/govoplan_deploy/cli.py index 2727504..1efe09b 100644 --- a/tools/deployment/govoplan_deploy/cli.py +++ b/tools/deployment/govoplan_deploy/cli.py @@ -30,7 +30,9 @@ from .bundle import ( initial_secrets, read_env, reconcile_runtime_environment, + render_caddy_config, render_compose, + render_existing_proxy_contract, render_garage_config, render_load_balancer_config, service_names, @@ -54,7 +56,9 @@ from .distribution import ( from .model import ( ComponentConfig, DEFAULT_GARAGE_IMAGE, + DEFAULT_INGRESS_IMAGE, DEFAULT_LOAD_BALANCER_IMAGE, + IngressConfig, InstallationSpec, ListenConfig, ReplicaConfig, @@ -320,6 +324,26 @@ def _configuration_arguments( default=default(DEFAULT_LOAD_BALANCER_IMAGE), help="HAProxy image used by the managed local load balancer.", ) + parser.add_argument( + "--ingress", + choices=("local", "existing-proxy", "managed", "unconfigured"), + default=default(None), + help="Public route boundary; self-hosted requires existing-proxy or managed.", + ) + parser.add_argument( + "--ingress-image", + default=default(DEFAULT_INGRESS_IMAGE), + help="Caddy image used by managed ingress.", + ) + parser.add_argument( + "--trusted-proxy-cidr", + action="append", + default=None, + help="Exact source CIDR trusted to supply forwarded headers; repeatable.", + ) + parser.add_argument("--acme-email", default=default("")) + parser.add_argument("--ingress-http-port", type=int, default=default(80)) + parser.add_argument("--ingress-https-port", type=int, default=default(443)) parser.add_argument( "--api-replicas", type=int, @@ -409,6 +433,12 @@ def _init(args: argparse.Namespace) -> int: storage_mode=args.storage, garage_image=args.garage_image, load_balancer_image=args.load_balancer_image, + ingress_mode=args.ingress, + ingress_image=args.ingress_image, + trusted_proxy_cidrs=tuple(args.trusted_proxy_cidr or ()), + ingress_http_port=args.ingress_http_port, + ingress_https_port=args.ingress_https_port, + acme_email=args.acme_email, api_replicas=args.api_replicas, web_replicas=args.web_replicas, worker_replicas=args.worker_replicas, @@ -530,6 +560,7 @@ def _apply(args: argparse.Namespace) -> int: "components.mail.image.", "components.storage.image.", "components.load_balancer.image.", + "ingress.image.", "release.manifest", "modules.image_composition", ) @@ -600,7 +631,14 @@ def _apply(args: argparse.Namespace) -> int: _run([*compose, "stop", "web"], cwd=paths.root) runtime_services = [ name - for name in ("api", "web", "load-balancer", "worker", "scheduler") + for name in ( + "api", + "web", + "load-balancer", + "worker", + "scheduler", + "ingress", + ) if name in service_names(spec) ] _run( @@ -794,7 +832,22 @@ def _verify_release(args: argparse.Namespace) -> int: image=str(dependency_images["load_balancer"]), ), ) - adopted = parse_spec(replace(spec, release=release, components=components).to_dict()) + ingress = replace( + spec.ingress, + image=( + str(dependency_images["managed_ingress"]) + if spec.ingress.mode == "managed" + else spec.ingress.image + ), + ) + adopted = parse_spec( + replace( + spec, + release=release, + components=components, + ingress=ingress, + ).to_dict() + ) ensure_private_directory(paths.root) atomic_write(paths.manifest, encoded_manifest, mode=0o644) atomic_write( @@ -862,11 +915,12 @@ def _selected_dependency_images( names.append("test_mail") if spec.components.storage.mode == "garage": names.append("garage") + if spec.ingress.mode == "managed": + names.append("managed_ingress") missing = [name for name in names if not isinstance(available.get(name), str)] if missing: raise DistributionError( - "distribution is missing selected dependency images: " - + ", ".join(missing) + "distribution is missing selected dependency images: " + ", ".join(missing) ) return {name: str(available[name]) for name in names} @@ -997,6 +1051,11 @@ def _deployment_receipt( "address": spec.listen.address, "port": spec.listen.port, }, + "ingress": { + "mode": spec.ingress.mode, + "http_port": spec.ingress.http_port, + "https_port": spec.ingress.https_port, + }, "management": { "mode": "govoplan-deploy", "agent": "cli", @@ -1080,6 +1139,41 @@ def _updated_spec( ) ), ) + ingress_mode = args.ingress or current.ingress.mode + ingress = IngressConfig( + mode=ingress_mode, + image=(args.ingress_image or current.ingress.image or DEFAULT_INGRESS_IMAGE) + if ingress_mode == "managed" + else "", + trusted_proxy_cidrs=tuple( + ( + args.trusted_proxy_cidr + if args.trusted_proxy_cidr is not None + else current.ingress.trusted_proxy_cidrs + ) + if ingress_mode == "existing-proxy" + else () + ), + http_port=( + args.ingress_http_port + if args.ingress_http_port is not None + else current.ingress.http_port + ), + https_port=( + args.ingress_https_port + if args.ingress_https_port is not None + else current.ingress.https_port + ), + acme_email=( + ( + args.acme_email + if args.acme_email is not None + else current.ingress.acme_email + ) + if ingress_mode == "managed" + else "" + ), + ) value = replace( current, installation_id=args.installation_id or current.installation_id, @@ -1092,6 +1186,7 @@ def _updated_spec( release=release, components=components, replicas=replicas, + ingress=ingress, enabled_modules=modules, ) return parse_spec(value.to_dict()) @@ -1137,6 +1232,16 @@ def _write_bundle( render_load_balancer_config(spec).encode("utf-8"), mode=0o644, ) + atomic_write( + paths.caddy_config, + render_caddy_config(spec).encode("utf-8"), + mode=0o644, + ) + atomic_write( + paths.existing_proxy, + canonical_json(render_existing_proxy_contract(spec)), + mode=0o644, + ) atomic_write( paths.garage_config, render_garage_config().encode("utf-8"), @@ -1169,6 +1274,24 @@ def _prompt_configuration(args: argparse.Namespace) -> None: if args.profile == "self-hosted" and args.public_url.startswith("http://"): args.public_url = "https://govoplan.example.org" args.public_url = _prompt("Public URL", args.public_url) + if args.profile == "self-hosted": + args.ingress = _prompt_choice( + "Public ingress", + args.ingress or "existing-proxy", + ("existing-proxy", "managed"), + ) + if args.ingress == "existing-proxy": + current = (args.trusted_proxy_cidr or ["127.0.0.1/32"])[0] + args.trusted_proxy_cidr = [ + _prompt("Trusted reverse-proxy source CIDR", current) + ] + else: + args.acme_email = args.acme_email or _prompt( + "ACME account email", + "admin@example.org", + ) + else: + args.ingress = args.ingress or "local" args.postgres = _prompt_choice("PostgreSQL", args.postgres, ("managed", "external")) if args.postgres == "external" and not args.database_url: args.database_url = getpass.getpass( diff --git a/tools/deployment/govoplan_deploy/model.py b/tools/deployment/govoplan_deploy/model.py index 3ad70d7..2b4c271 100644 --- a/tools/deployment/govoplan_deploy/model.py +++ b/tools/deployment/govoplan_deploy/model.py @@ -14,6 +14,7 @@ from urllib.parse import urlsplit SCHEMA_VERSION = 1 DEFAULT_GARAGE_IMAGE = "dxflrs/garage:v2.3.0" DEFAULT_LOAD_BALANCER_IMAGE = "haproxy:3.2.21-alpine" +DEFAULT_INGRESS_IMAGE = "caddy:2.10.2-alpine" INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$") ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$") SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") @@ -113,6 +114,16 @@ class ReplicaConfig: worker: int +@dataclass(frozen=True, slots=True) +class IngressConfig: + mode: str + image: str + trusted_proxy_cidrs: tuple[str, ...] + http_port: int + https_port: int + acme_email: str + + @dataclass(frozen=True, slots=True) class InstallationSpec: schema_version: int @@ -124,11 +135,13 @@ class InstallationSpec: release: ReleaseConfig components: ComponentConfig replicas: ReplicaConfig + ingress: IngressConfig enabled_modules: tuple[str, ...] def to_dict(self) -> dict[str, Any]: value = asdict(self) value["enabled_modules"] = list(self.enabled_modules) + value["ingress"]["trusted_proxy_cidrs"] = list(self.ingress.trusted_proxy_cidrs) return value @@ -145,6 +158,12 @@ def default_spec( storage_mode: str = "local", garage_image: str = DEFAULT_GARAGE_IMAGE, load_balancer_image: str = DEFAULT_LOAD_BALANCER_IMAGE, + ingress_mode: str | None = None, + ingress_image: str = DEFAULT_INGRESS_IMAGE, + trusted_proxy_cidrs: tuple[str, ...] = (), + ingress_http_port: int = 80, + ingress_https_port: int = 443, + acme_email: str = "", api_replicas: int = 1, web_replicas: int = 1, worker_replicas: int | None = None, @@ -218,6 +237,15 @@ def default_spec( "web": web_replicas, "worker": effective_worker_replicas, }, + "ingress": { + "mode": ingress_mode + or ("unconfigured" if profile == "self-hosted" else "local"), + "image": ingress_image if ingress_mode == "managed" else "", + "trusted_proxy_cidrs": list(trusted_proxy_cidrs), + "http_port": ingress_http_port, + "https_port": ingress_https_port, + "acme_email": acme_email, + }, "enabled_modules": list(modules), } return parse_spec(raw) @@ -247,6 +275,7 @@ def parse_spec(raw: object) -> InstallationSpec: "release", "components", "replicas", + "ingress", "enabled_modules", }, "installation", @@ -281,6 +310,17 @@ def parse_spec(raw: object) -> InstallationSpec: release = _release(root.get("release")) components = _components(root.get("components"), profile=profile) replicas = _replicas(root.get("replicas"), components=components) + ingress = _ingress(root.get("ingress"), profile=profile) + if ingress.mode == "managed": + try: + ipaddress.ip_address(public_parts.hostname or "") + except ValueError: + pass + else: + raise SpecError("managed ingress requires a DNS hostname in public_url") + public_port = public_parts.port or 443 + if public_port != ingress.https_port: + raise SpecError("managed ingress HTTPS port must match the public_url port") enabled_raw = root.get("enabled_modules") if not isinstance(enabled_raw, list): @@ -307,6 +347,7 @@ def parse_spec(raw: object) -> InstallationSpec: release=release, components=components, replicas=replicas, + ingress=ingress, enabled_modules=tuple(enabled_modules), ) @@ -344,18 +385,14 @@ def _release(raw: object) -> ReleaseConfig: raise SpecError( "release.manifest_sha256 must be a lowercase SHA-256 hex digest" ) - manifest_keyring_sha256 = _optional_string( - value, "manifest_keyring_sha256" - ).lower() + manifest_keyring_sha256 = _optional_string(value, "manifest_keyring_sha256").lower() if manifest_keyring_sha256 and not SHA256_PATTERN.fullmatch( manifest_keyring_sha256 ): raise SpecError( "release.manifest_keyring_sha256 must be a lowercase SHA-256 hex digest" ) - manifest_signature_key_id = _optional_string( - value, "manifest_signature_key_id" - ) + manifest_signature_key_id = _optional_string(value, "manifest_signature_key_id") if manifest_signature_key_id and not re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", manifest_signature_key_id ): @@ -483,6 +520,84 @@ def _replicas(raw: object, *, components: ComponentConfig) -> ReplicaConfig: return replicas +def _ingress(raw: object, *, profile: str) -> IngressConfig: + if raw is None: + return IngressConfig( + mode="unconfigured" if profile == "self-hosted" else "local", + image="", + trusted_proxy_cidrs=(), + http_port=80, + https_port=443, + acme_email="", + ) + value = _mapping(raw, "ingress") + _only_keys( + value, + { + "mode", + "image", + "trusted_proxy_cidrs", + "http_port", + "https_port", + "acme_email", + }, + "ingress", + ) + mode = _choice( + value, + "mode", + {"local", "existing-proxy", "managed", "unconfigured"}, + ) + image = _optional_string(value, "image") + raw_cidrs = value.get("trusted_proxy_cidrs", []) + if not isinstance(raw_cidrs, list) or len(raw_cidrs) > 16: + raise SpecError( + "ingress.trusted_proxy_cidrs must be an array of at most 16 networks" + ) + cidrs: list[str] = [] + for item in raw_cidrs: + if not isinstance(item, str): + raise SpecError("ingress.trusted_proxy_cidrs must contain strings") + cidrs.append(_trusted_proxy_network(item)) + if len(set(cidrs)) != len(cidrs): + raise SpecError("ingress.trusted_proxy_cidrs contains duplicates") + http_port = _port(_integer(value, "http_port"), "ingress.http_port") + https_port = _port(_integer(value, "https_port"), "ingress.https_port") + if http_port == https_port: + raise SpecError("ingress HTTP and HTTPS ports must differ") + acme_email = _optional_string(value, "acme_email") + if acme_email and ( + len(acme_email) > 254 or re.fullmatch(r"[^@\s]+@[^@\s]+", acme_email) is None + ): + raise SpecError("ingress.acme_email must be a valid email address") + if profile == "self-hosted" and mode == "local": + raise SpecError( + "self-hosted installations require existing-proxy or managed ingress" + ) + if profile == "evaluation" and mode == "unconfigured": + raise SpecError("evaluation installations cannot use unconfigured ingress") + if mode == "managed": + image = _image(image or DEFAULT_INGRESS_IMAGE, "ingress.image") + if not acme_email: + raise SpecError("managed ingress requires ingress.acme_email") + elif image: + raise SpecError("ingress.image is only valid for managed ingress") + if mode == "existing-proxy" and not cidrs: + raise SpecError("existing-proxy ingress requires a trusted proxy CIDR") + if mode != "existing-proxy" and cidrs: + raise SpecError("trusted proxy CIDRs are only valid for existing-proxy ingress") + if mode != "managed" and acme_email: + raise SpecError("ingress.acme_email is only valid for managed ingress") + return IngressConfig( + mode=mode, + image=image, + trusted_proxy_cidrs=tuple(cidrs), + http_port=http_port, + https_port=https_port, + acme_email=acme_email, + ) + + def _service( raw: object, label: str, @@ -614,6 +729,32 @@ def _network(value: str, label: str) -> str: return str(network) +def _trusted_proxy_network(value: str) -> str: + try: + network = ipaddress.ip_network(value.strip(), strict=True) + except ValueError as exc: + raise SpecError( + "ingress.trusted_proxy_cidrs must contain canonical IP networks" + ) from exc + if network.is_unspecified or network.is_multicast: + raise SpecError("trusted proxy CIDR cannot be unspecified or multicast") + if network.version == 4: + if network.is_global and network.prefixlen != 32: + raise SpecError("a public trusted proxy must be an exact IPv4 address") + if not network.is_global and network.prefixlen < 24: + raise SpecError( + "a private trusted IPv4 proxy network must be /24 or narrower" + ) + else: + if network.is_global and network.prefixlen != 128: + raise SpecError("a public trusted proxy must be an exact IPv6 address") + if not network.is_global and network.prefixlen < 64: + raise SpecError( + "a private trusted IPv6 proxy network must be /64 or narrower" + ) + return str(network) + + def _port(value: int, label: str) -> int: if value < 1 or value > 65535: raise SpecError(f"{label} must be between 1 and 65535") diff --git a/tools/deployment/govoplan_deploy/planning.py b/tools/deployment/govoplan_deploy/planning.py index 5db2ff1..c163f9b 100644 --- a/tools/deployment/govoplan_deploy/planning.py +++ b/tools/deployment/govoplan_deploy/planning.py @@ -9,17 +9,24 @@ from pathlib import Path import platform import shutil import socket +import ssl import stat import subprocess +import time from typing import Callable, Mapping, Sequence +from urllib.error import HTTPError, URLError from urllib.parse import urlsplit +from urllib.request import Request, urlopen from .bundle import ( BundlePaths, + canonical_json, digest_json, environment_fingerprint, read_env, + render_caddy_config, render_compose, + render_existing_proxy_contract, service_names, ) from .distribution import ( @@ -178,6 +185,7 @@ def build_plan( def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ...]: checks: list[Check] = [] + checks.extend(_ingress_configuration_checks(spec, paths)) images = { "release.api_image": spec.release.api_image, "release.web_image": spec.release.web_image, @@ -191,6 +199,8 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. if spec.components.storage.mode == "garage": images["components.storage.image"] = spec.components.storage.image images["components.load_balancer.image"] = spec.components.load_balancer.image + if spec.ingress.mode == "managed": + images["ingress.image"] = spec.ingress.image for label, image in images.items(): if image_is_unpublished(image): @@ -340,6 +350,88 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. return tuple(checks) +def _ingress_configuration_checks( + spec: InstallationSpec, + paths: BundlePaths, +) -> tuple[Check, ...]: + if spec.ingress.mode == "unconfigured": + return ( + Check( + "ingress.configuration", + "error" if spec.profile == "self-hosted" else "warning", + "No supported public ingress boundary is configured.", + "Select managed ingress or an existing reverse proxy before apply.", + ), + ) + checks = [ + Check( + "ingress.configuration", + "ok", + f"Ingress mode {spec.ingress.mode!r} has a bounded configuration.", + ) + ] + if spec.ingress.mode == "managed": + checks.append( + _artifact_check( + "ingress.managed_config", + paths.caddy_config, + render_caddy_config(spec).encode("utf-8"), + "Managed ingress configuration", + ) + ) + ingress = render_compose(spec)["services"].get("ingress", {}) + volumes = ingress.get("volumes", []) if isinstance(ingress, dict) else [] + persistent = "caddy-data:/data" in volumes and "caddy-config:/config" in volumes + checks.append( + Check( + "ingress.certificate_state", + "ok" if persistent else "error", + ( + "Managed certificate and renewal state uses persistent private volumes." + if persistent + else "Managed certificate state is not persistent." + ), + "Restore the caddy-data and caddy-config volume bindings." + if not persistent + else "", + ) + ) + elif spec.ingress.mode == "existing-proxy": + checks.append( + _artifact_check( + "ingress.existing_proxy_contract", + paths.existing_proxy, + canonical_json(render_existing_proxy_contract(spec)), + "Existing reverse-proxy contract", + ) + ) + return tuple(checks) + + +def _artifact_check( + check_id: str, + path: Path, + expected: bytes, + label: str, +) -> Check: + try: + actual = path.read_bytes() + except OSError as exc: + return Check( + check_id, + "error", + f"{label} is unavailable: {exc}", + "Re-render the installation bundle.", + ) + matches = actual == expected + return Check( + check_id, + "ok" if matches else "error", + f"{label} {'matches' if matches else 'does not match'} the installation specification.", + "Re-render the installation bundle before apply." if not matches else "", + ) + + def _distribution_checks( spec: InstallationSpec, paths: BundlePaths, @@ -379,7 +471,9 @@ def _distribution_checks( maximum_bytes=MAX_MANIFEST_BYTES, ) if manifest_digest != spec.release.manifest_sha256: - raise DistributionError("stored manifest digest does not match installation") + raise DistributionError( + "stored manifest digest does not match installation" + ) keyring_digest = file_sha256( paths.keyring, maximum_bytes=MAX_KEYRING_BYTES, @@ -400,7 +494,9 @@ def _distribution_checks( expected_channel=spec.release.channel, ) if key_id != spec.release.manifest_signature_key_id: - raise DistributionError("verified signature key does not match installation") + raise DistributionError( + "verified signature key does not match installation" + ) verify_manifest_binding( manifest, channel=spec.release.channel, @@ -461,6 +557,8 @@ def _selected_dependency_images(spec: InstallationSpec) -> dict[str, str]: values["test_mail"] = spec.components.mail.image if spec.components.storage.mode == "garage": values["garage"] = spec.components.storage.image + if spec.ingress.mode == "managed": + values["managed_ingress"] = spec.ingress.image return values @@ -471,6 +569,7 @@ def host_checks( command_runner: CommandRunner | None = None, ) -> tuple[Check, ...]: checks: list[Check] = [] + runner = command_runner or _run_command machine = platform.machine().lower() supported = machine in {"x86_64", "amd64", "aarch64", "arm64"} checks.append( @@ -543,7 +642,6 @@ def host_checks( ) ) else: - runner = command_runner or _run_command result = runner((docker, "compose", "version", "--short"), paths.root) checks.append( Check( @@ -608,30 +706,245 @@ def host_checks( ) receipt = _read_receipt(paths.receipt) + checks.extend( + _ingress_host_checks( + spec, + paths, + receipt=receipt, + docker=docker, + command_runner=runner, + ) + ) + return tuple(checks) + + +def _ingress_host_checks( + spec: InstallationSpec, + paths: BundlePaths, + *, + receipt: Mapping[str, object], + docker: str | None, + command_runner: CommandRunner, +) -> tuple[Check, ...]: + checks: list[Check] = [] + applied = bool(receipt) + if spec.ingress.mode in {"managed", "existing-proxy"}: + public = urlsplit(spec.public_url) + host = public.hostname or "" + port = public.port or 443 + checks.append(_dns_resolution_check(host, port)) + if spec.ingress.mode == "managed" and not applied: + checks.append( + Check( + "ingress.tls", + "warning", + "TLS issuance will be verified after managed ingress starts.", + "Ensure public DNS resolves to this host and ports 80/443 are reachable.", + ) + ) + checks.append( + Check( + "ingress.public_route", + "warning", + "Public-route health will be verified after managed ingress starts.", + "Permit inbound HTTP and HTTPS through the host firewall and upstream NAT.", + ) + ) + else: + checks.append(_tls_validity_check(host, port)) + checks.append( + _public_route_check( + spec.public_url, + require_ready=applied, + ) + ) + + previous_ingress = receipt.get("ingress") + desired_ingress = { + "mode": spec.ingress.mode, + "http_port": spec.ingress.http_port, + "https_port": spec.ingress.https_port, + } previous_listen = receipt.get("listen") desired_listen = { "address": spec.listen.address, "port": spec.listen.port, } - if not receipt or previous_listen != desired_listen: - available = _port_available(spec.listen.address, spec.listen.port) + if spec.ingress.mode == "managed": + if not applied or previous_ingress != desired_ingress: + for label, port in ( + ("http", spec.ingress.http_port), + ("https", spec.ingress.https_port), + ): + checks.append( + _available_port_check( + f"host.ingress_{label}_port", + "0.0.0.0", + port, + ) + ) + elif not applied or previous_listen != desired_listen: checks.append( - Check( + _available_port_check( "host.listen_port", - "ok" if available else "error", - ( - f"Listen endpoint {spec.listen.address}:{spec.listen.port} is available." - if available - else f"Listen endpoint {spec.listen.address}:{spec.listen.port} is already in use." - ), - "Choose another listen port or stop the conflicting service." - if not available - else "", + spec.listen.address, + spec.listen.port, + ) + ) + + if applied: + checks.append( + _local_upstream_check( + spec, + paths, + docker=docker, + command_runner=command_runner, ) ) return tuple(checks) +def _available_port_check(check_id: str, address: str, port: int) -> Check: + available = _port_available(address, port) + return Check( + check_id, + "ok" if available else "error", + ( + f"Listen endpoint {address}:{port} is available." + if available + else f"Listen endpoint {address}:{port} is already in use." + ), + "Choose another port or stop the conflicting service." if not available else "", + ) + + +def _dns_resolution_check(host: str, port: int) -> Check: + try: + results = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + addresses = sorted({str(item[4][0]) for item in results}) + except OSError as exc: + return Check( + "ingress.dns", + "error", + f"Public hostname {host!r} does not resolve: {exc}", + "Publish public A/AAAA records before apply.", + ) + return Check( + "ingress.dns", + "ok", + f"Public hostname {host!r} resolves to {', '.join(addresses[:8])}.", + ) + + +def _tls_validity_check(host: str, port: int) -> Check: + try: + context = ssl.create_default_context() + with socket.create_connection((host, port), timeout=3.0) as connection: + with context.wrap_socket(connection, server_hostname=host) as secured: + certificate = secured.getpeercert() + expires = str(certificate.get("notAfter") or "") + remaining_seconds = ssl.cert_time_to_seconds(expires) - time.time() + except (OSError, ValueError, ssl.SSLError) as exc: + return Check( + "ingress.tls", + "error", + f"Public TLS validation failed for {host}:{port}: {exc}", + "Correct certificate issuance, trust chain, hostname, and public routing.", + ) + remaining_days = int(remaining_seconds // 86400) + level = "ok" if remaining_days >= 21 else "warning" + return Check( + "ingress.tls", + level, + f"Public TLS certificate is valid for approximately {remaining_days} more day(s).", + "Verify automated certificate renewal immediately." + if level == "warning" + else "", + ) + + +def _public_route_check(public_url: str, *, require_ready: bool) -> Check: + target = public_url.rstrip("/") + "/health/ready" + status: int | None = None + try: + request = Request(target, headers={"User-Agent": "govoplan-deploy/doctor"}) + with urlopen(request, timeout=4.0) as response: + status = response.status + except HTTPError as exc: + status = exc.code + except (OSError, URLError, ValueError) as exc: + return Check( + "ingress.public_route", + "error", + f"Public route is unreachable: {exc}", + "Check external DNS, firewall/NAT, reverse-proxy routing, and TLS.", + ) + acceptable = status == 200 if require_ready else status not in {400, 404, 421} + return Check( + "ingress.public_route", + "ok" if acceptable else "error", + f"Public readiness route returned HTTP {status}.", + ( + "Route the configured hostname and /health/ready path to the generated upstream." + if not acceptable + else "" + ), + ) + + +def _local_upstream_check( + spec: InstallationSpec, + paths: BundlePaths, + *, + docker: str | None, + command_runner: CommandRunner, +) -> Check: + if docker is None: + return Check( + "ingress.local_upstream", + "error", + "Local upstream health cannot be checked without Docker.", + "Restore Docker access and rerun doctor.", + ) + argv = ( + docker, + "compose", + "--env-file", + str(paths.env), + "--project-name", + spec.installation_id, + "--file", + str(paths.compose), + "exec", + "--no-TTY", + "load-balancer", + "wget", + "-qO-", + "http://127.0.0.1:8080/health", + ) + try: + result = command_runner(argv, paths.root) + except (OSError, subprocess.SubprocessError) as exc: + return Check( + "ingress.local_upstream", + "error", + f"Local upstream health probe failed: {exc}", + "Inspect the load-balancer and WebUI service health.", + ) + return Check( + "ingress.local_upstream", + "ok" if result.returncode == 0 else "error", + ( + "The generated local upstream is healthy." + if result.returncode == 0 + else "The generated local upstream health probe failed." + ), + "Inspect load-balancer and WebUI health before exposing the route." + if result.returncode != 0 + else "", + ) + + def _read_receipt(path: Path) -> Mapping[str, object]: if not path.exists(): return {} diff --git a/tools/deployment/govoplan_deploy/recovery.py b/tools/deployment/govoplan_deploy/recovery.py index 117dd63..1887e28 100644 --- a/tools/deployment/govoplan_deploy/recovery.py +++ b/tools/deployment/govoplan_deploy/recovery.py @@ -25,6 +25,10 @@ _BUNDLE_FILES = ( "compose.json", "garage.toml", "load-balancer.cfg", + "Caddyfile", + "existing-proxy.json", + "distribution-manifest.json", + "distribution-keyring.json", "receipt.json", ) diff --git a/tools/release/runtime/nginx.conf b/tools/release/runtime/nginx.conf index e0821fc..2ddf20f 100644 --- a/tools/release/runtime/nginx.conf +++ b/tools/release/runtime/nginx.conf @@ -15,6 +15,12 @@ http { client_body_temp_path /tmp/client_temp; proxy_temp_path /tmp/proxy_temp; + map $http_x_forwarded_proto $govoplan_forwarded_proto { + default $scheme; + http http; + https https; + } + server { listen 8080; root /usr/share/nginx/html; @@ -30,8 +36,8 @@ http { proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $govoplan_forwarded_proto; + proxy_set_header X-Forwarded-For $http_x_forwarded_for; } location / {