#!/usr/bin/env python3 """Exercise generated managed ingress with a real Caddy container.""" from __future__ import annotations import argparse import json from pathlib import Path import re import shutil import socket import subprocess import sys import tempfile from uuid import uuid4 META_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(META_ROOT / "tools" / "deployment")) from govoplan_deploy.bundle import ( # noqa: E402 render_caddy_config, render_load_balancer_config, ) from govoplan_deploy.model import default_spec # noqa: E402 DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") def _run( argv: list[str], *, check: bool = True, input_text: str | None = None, ) -> subprocess.CompletedProcess[str]: try: return subprocess.run( argv, check=check, capture_output=True, input=input_text, text=True, timeout=60, ) except subprocess.CalledProcessError as exc: stderr = exc.stderr.strip() if stderr: print(stderr, file=sys.stderr) raise def _write_volume_file( *, image: str, volume: str, filename: str, content: str, ) -> None: if not re.fullmatch(r"[A-Za-z0-9_.-]+", filename): raise ValueError(f"invalid config filename: {filename!r}") _run( [ "docker", "run", "--rm", "--interactive", "--user", "0:0", "--mount", f"type=volume,src={volume},dst=/govoplan-config", "--entrypoint", "sh", image, "-c", f"umask 022; cat > /govoplan-config/{filename}", ], input_text=content, ) def _published_port(container: str, target: int) -> int: output = _run( [ "docker", "inspect", "--format", "{{json .HostConfig.PortBindings}}", container, ] ).stdout.strip() try: bindings = json.loads(output)[f"{target}/tcp"] if not isinstance(bindings, list) or len(bindings) != 1: raise ValueError("expected exactly one published binding") binding = bindings[0] if binding.get("HostIp") != "127.0.0.1": raise ValueError("published binding is not loopback-only") return int(binding["HostPort"]) except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: raise RuntimeError( f"cannot determine loopback binding for {target}/tcp from {output!r}" ) from exc def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int: for _attempt in range(10): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: listener.bind(("127.0.0.1", 0)) port = int(listener.getsockname()[1]) if port not in exclude: return port raise RuntimeError("cannot allocate distinct loopback ports for ingress drill") def _probe_ingress(*, image: str, network: str, container: str) -> None: probe = r''' import socket import ssl import time def request(port, payload, *, tls): connection = socket.create_connection(("ingress", port), timeout=3) if tls: connection = ssl._create_unverified_context().wrap_socket( connection, server_hostname="localhost" ) with connection: connection.sendall(payload) chunks = [] while True: chunk = connection.recv(65536) if not chunk: break chunks.append(chunk) return b"".join(chunks) deadline = time.monotonic() + 30 last_error = "" while time.monotonic() < deadline: try: response = request( 8443, b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", tls=True, ) head, body_bytes = response.split(b"\r\n\r\n", 1) status = int(head.split(b" ", 2)[1]) if status != 200: raise RuntimeError(f"HTTPS returned {status}, expected 200") body = body_bytes.decode("utf-8").strip() if body != "proto=https": raise RuntimeError(f"forwarded protocol was not normalized: {body!r}") break except Exception as exc: last_error = f"{type(exc).__name__}: {exc}" time.sleep(0.5) else: raise SystemExit(f"managed ingress did not become ready: {last_error}") response = request( 8080, b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", tls=False, ) head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1") lines = head.split("\r\n") status = int(lines[0].split(" ", 2)[1]) if status != 308: raise SystemExit(f"HTTP returned {status}, expected redirect 308") headers = { key.lower(): value.strip() for key, separator, value in (line.partition(":") for line in lines[1:]) if separator } location = headers.get("location", "") if not location.startswith("https://localhost"): raise SystemExit(f"HTTP redirect had unexpected location: {location!r}") ''' try: _run( [ "docker", "run", "--rm", "--network", network, "--read-only", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", "--entrypoint", "python", image, "-c", probe, ] ) except subprocess.CalledProcessError: logs = _run(["docker", "logs", container], check=False).stdout.strip() if logs: print(f"managed ingress logs:\n{logs}", file=sys.stderr) raise def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--caddy-image", required=True) parser.add_argument("--load-balancer-image", required=True) parser.add_argument("--probe-image", required=True) args = parser.parse_args() for label, image in ( ("--caddy-image", args.caddy_image), ("--load-balancer-image", args.load_balancer_image), ("--probe-image", args.probe_image), ): if DIGEST_IMAGE.fullmatch(image) is None: parser.error(f"{label} must be pinned by sha256 digest") if shutil.which("docker") is None: parser.error("docker is required") suffix = uuid4().hex[:10] network = f"govoplan-ingress-drill-{suffix}" ingress = f"govoplan-ingress-{suffix}" backend = f"govoplan-ingress-backend-{suffix}" data_volume = f"govoplan-ingress-data-{suffix}" config_volume = f"govoplan-ingress-config-{suffix}" backend_config_volume = f"govoplan-ingress-backend-config-{suffix}" ingress_config_volume = f"govoplan-ingress-caddy-config-{suffix}" load_balancer_config_volume = f"govoplan-ingress-haproxy-config-{suffix}" cleanup = [ ["docker", "rm", "--force", ingress, backend], ["docker", "network", "rm", network], [ "docker", "volume", "rm", data_volume, config_volume, backend_config_volume, ingress_config_volume, load_balancer_config_volume, ], ] try: _run(["docker", "network", "create", network]) for volume in ( data_volume, config_volume, backend_config_volume, ingress_config_volume, load_balancer_config_volume, ): _run(["docker", "volume", "create", volume]) with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory: root = Path(directory) backend_config = root / "backend.Caddyfile" backend_config.write_text( """{ admin off auto_https off } :8080 { respond /health "proto={http.request.header.X-Forwarded-Proto}" } """, encoding="utf-8", ) backend_config.chmod(0o644) spec = default_spec( profile="self-hosted", public_url="https://localhost:8443", ingress_mode="managed", ingress_image=args.caddy_image, ingress_https_port=8443, acme_email="operator@example.test", ) ingress_config = root / "Caddyfile" ingress_config.write_text(render_caddy_config(spec), encoding="utf-8") ingress_config.chmod(0o644) load_balancer_config = root / "haproxy.cfg" load_balancer_config.write_text( render_load_balancer_config(spec), encoding="utf-8", ) load_balancer_config.chmod(0o644) _write_volume_file( image=args.load_balancer_image, volume=load_balancer_config_volume, filename="haproxy.cfg", content=load_balancer_config.read_text(encoding="utf-8"), ) _write_volume_file( image=args.caddy_image, volume=backend_config_volume, filename="Caddyfile", content=backend_config.read_text(encoding="utf-8"), ) _write_volume_file( image=args.caddy_image, volume=ingress_config_volume, filename="Caddyfile", content=ingress_config.read_text(encoding="utf-8"), ) _run( [ "docker", "run", "--rm", "--read-only", "--cap-drop", "ALL", "--mount", ( "type=volume," f"src={load_balancer_config_volume}," "dst=/usr/local/etc/haproxy,readonly" ), args.load_balancer_image, "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg", ] ) _run( [ "docker", "run", "--detach", "--name", backend, "--network", network, "--network-alias", "load-balancer", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", "--mount", ( "type=volume," f"src={backend_config_volume}," "dst=/govoplan-config,readonly" ), args.caddy_image, "caddy", "run", "--config", "/govoplan-config/Caddyfile", ] ) requested_http_port = _available_loopback_port() requested_https_port = _available_loopback_port( exclude=frozenset({requested_http_port}) ) ingress_command = [ "docker", "run", "--detach", "--name", ingress, "--network", network, "--network-alias", "ingress", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", "--publish", f"127.0.0.1:{requested_http_port}:8080/tcp", "--publish", f"127.0.0.1:{requested_https_port}:8443/tcp", "--mount", ( "type=volume," f"src={ingress_config_volume}," "dst=/govoplan-config,readonly" ), "--mount", f"type=volume,src={data_volume},dst=/data", "--mount", f"type=volume,src={config_volume},dst=/config", args.caddy_image, "caddy", "run", "--config", "/govoplan-config/Caddyfile", ] _run(ingress_command) http_port = _published_port(ingress, 8080) https_port = _published_port(ingress, 8443) if (http_port, https_port) != ( requested_http_port, requested_https_port, ): raise RuntimeError("Docker published unexpected ingress ports") _probe_ingress( image=args.probe_image, network=network, container=ingress, ) _run(["docker", "rm", "--force", ingress]) _run(ingress_command) https_port = _published_port(ingress, 8443) if https_port != requested_https_port: raise RuntimeError("Docker changed the ingress TLS binding on restart") _probe_ingress( image=args.probe_image, network=network, container=ingress, ) _run( [ "docker", "run", "--rm", "--mount", f"type=volume,src={data_volume},dst=/data,readonly", "--entrypoint", "sh", args.caddy_image, "-c", 'test -n "$(find /data -type f -print -quit)"', ] ) finally: for command in cleanup: _run(command, check=False) print("Managed ingress container drill passed.") return 0 if __name__ == "__main__": raise SystemExit(main())