From 2f28f22fd15c710838eeaca3266e9346537cb0fd Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Mon, 3 Aug 2026 19:41:04 +0200 Subject: [PATCH] Probe managed ingress across Docker namespaces --- .gitea/workflows/runtime-distribution.yml | 1 + ...NSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md | 5 +- tests/test_managed_ingress_drill.py | 46 ++++- tools/checks/managed-ingress-drill.py | 161 ++++++++++++++---- 4 files changed, 177 insertions(+), 36 deletions(-) diff --git a/.gitea/workflows/runtime-distribution.yml b/.gitea/workflows/runtime-distribution.yml index fcdbaae..b91e2dd 100644 --- a/.gitea/workflows/runtime-distribution.yml +++ b/.gitea/workflows/runtime-distribution.yml @@ -342,6 +342,7 @@ jobs: python tools/checks/managed-ingress-drill.py --caddy-image "$MANAGED_INGRESS_IMAGE" --load-balancer-image "$LOAD_BALANCER_IMAGE" + --probe-image "$(jq -r '.platforms[\"linux/amd64\"]' runtime-output/api-metadata.json)" - name: Publish immutable Gitea release assets working-directory: govoplan env: diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index 5504273..9e6092d 100644 --- a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -177,7 +177,10 @@ volumes before starting the read-only containers. It therefore also works when an Actions job reaches a host or remote Docker daemon through a mounted socket; the drill never assumes that a job-container path is visible to that daemon. The drill allocates explicit loopback-only host ports and verifies Docker's -published mappings, avoiding daemon-specific random-port shorthand behavior. +host binding configuration, avoiding daemon-specific random-port shorthand +behavior. Because an Actions job and deployment containers may be Docker +siblings, functional HTTP/TLS checks run from the digest-pinned API image on +the deployment network instead of assuming the Docker host is job-local. The bounded setup helper writes only generated public configuration as root so it can initialize a new volume; the actual HAProxy process retains the image's non-root identity and runs read-only with all capabilities dropped. diff --git a/tests/test_managed_ingress_drill.py b/tests/test_managed_ingress_drill.py index 1b48bd1..36d7d2f 100644 --- a/tests/test_managed_ingress_drill.py +++ b/tests/test_managed_ingress_drill.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib.util from pathlib import Path +import json import subprocess import sys import unittest @@ -68,7 +69,13 @@ class ManagedIngressDrillTests(unittest.TestCase): completed = subprocess.CompletedProcess( [], 0, - "127.0.0.1:49152\n", + json.dumps( + { + "8443/tcp": [ + {"HostIp": "127.0.0.1", "HostPort": "49152"} + ] + } + ), "", ) with patch.object(INGRESS, "_run", return_value=completed) as run: @@ -76,10 +83,45 @@ class ManagedIngressDrillTests(unittest.TestCase): self.assertEqual(49152, port) self.assertEqual( - ["docker", "port", "ingress", "8443/tcp"], + [ + "docker", + "inspect", + "--format", + "{{json .HostConfig.PortBindings}}", + "ingress", + ], run.call_args.args[0], ) + def test_published_port_rejects_non_loopback_binding(self) -> None: + completed = subprocess.CompletedProcess( + [], + 0, + '{"8443/tcp":[{"HostIp":"0.0.0.0","HostPort":"49152"}]}', + "", + ) + with patch.object(INGRESS, "_run", return_value=completed): + with self.assertRaisesRegex(RuntimeError, "loopback binding"): + INGRESS._published_port("ingress", 8443) + + def test_probe_runs_as_a_network_sibling_from_a_digest_image(self) -> None: + completed = subprocess.CompletedProcess([], 0, "", "") + image = "registry.example/runtime-api@sha256:" + "1" * 64 + with patch.object(INGRESS, "_run", return_value=completed) as run: + INGRESS._probe_ingress( + image=image, + network="deployment-network", + container="ingress", + ) + + argv = run.call_args.args[0] + self.assertEqual("docker", argv[0]) + self.assertIn("deployment-network", argv) + self.assertIn(image, argv) + self.assertIn('(\"ingress\", port)', argv[-1]) + self.assertIn("server_hostname=\"localhost\"", argv[-1]) + self.assertNotIn("localhost:49152", argv[-1]) + if __name__ == "__main__": unittest.main() diff --git a/tools/checks/managed-ingress-drill.py b/tools/checks/managed-ingress-drill.py index 759164a..29e7e1f 100644 --- a/tools/checks/managed-ingress-drill.py +++ b/tools/checks/managed-ingress-drill.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import json from pathlib import Path import re import shutil @@ -11,7 +12,6 @@ import socket import subprocess import sys import tempfile -import time from uuid import uuid4 @@ -80,11 +80,27 @@ def _write_volume_file( def _published_port(container: str, target: int) -> int: - output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip() + output = _run( + [ + "docker", + "inspect", + "--format", + "{{json .HostConfig.PortBindings}}", + container, + ] + ).stdout.strip() try: - return int(output.rsplit(":", 1)[1]) - except (IndexError, ValueError) as exc: - raise RuntimeError(f"cannot determine published port from {output!r}") from exc + bindings = json.loads(output)[f"{target}/tcp"] + if not isinstance(bindings, list) or len(bindings) != 1: + raise ValueError("expected exactly one published binding") + binding = bindings[0] + if binding.get("HostIp") != "127.0.0.1": + raise ValueError("published binding is not loopback-only") + return int(binding["HostPort"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"cannot determine loopback binding for {target}/tcp from {output!r}" + ) from exc def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int: @@ -97,39 +113,114 @@ def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int: raise RuntimeError("cannot allocate distinct loopback ports for ingress drill") -def _curl(url: str, *, headers: bool = False) -> str: - argv = ["curl", "--silent", "--show-error", "--insecure"] - if headers: - argv.extend(["--head"]) - argv.append(url) - return _run(argv).stdout +def _probe_ingress(*, image: str, network: str, container: str) -> None: + probe = r''' +import socket +import ssl +import time -def _wait_for_https(port: int) -> str: - deadline = time.monotonic() + 30 - last_error = "" - while time.monotonic() < deadline: - try: - return _curl(f"https://localhost:{port}/health") - except subprocess.CalledProcessError as exc: - last_error = exc.stderr.strip() - time.sleep(0.5) - raise RuntimeError(f"managed ingress did not become ready: {last_error}") +def request(port, payload, *, tls): + connection = socket.create_connection(("ingress", port), timeout=3) + if tls: + connection = ssl._create_unverified_context().wrap_socket( + connection, server_hostname="localhost" + ) + with connection: + connection.sendall(payload) + chunks = [] + while True: + chunk = connection.recv(65536) + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + +deadline = time.monotonic() + 30 +last_error = "" +while time.monotonic() < deadline: + try: + response = request( + 8443, + b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + tls=True, + ) + head, body_bytes = response.split(b"\r\n\r\n", 1) + status = int(head.split(b" ", 2)[1]) + if status != 200: + raise RuntimeError(f"HTTPS returned {status}, expected 200") + body = body_bytes.decode("utf-8").strip() + if body != "proto=https": + raise RuntimeError(f"forwarded protocol was not normalized: {body!r}") + break + except Exception as exc: + last_error = f"{type(exc).__name__}: {exc}" + time.sleep(0.5) +else: + raise SystemExit(f"managed ingress did not become ready: {last_error}") + +response = request( + 8080, + b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + tls=False, +) +head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1") +lines = head.split("\r\n") +status = int(lines[0].split(" ", 2)[1]) +if status != 308: + raise SystemExit(f"HTTP returned {status}, expected redirect 308") +headers = { + key.lower(): value.strip() + for key, separator, value in (line.partition(":") for line in lines[1:]) + if separator +} +location = headers.get("location", "") +if not location.startswith("https://localhost"): + raise SystemExit(f"HTTP redirect had unexpected location: {location!r}") +''' + try: + _run( + [ + "docker", + "run", + "--rm", + "--network", + network, + "--read-only", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--entrypoint", + "python", + image, + "-c", + probe, + ] + ) + except subprocess.CalledProcessError: + logs = _run(["docker", "logs", container], check=False).stdout.strip() + if logs: + print(f"managed ingress logs:\n{logs}", file=sys.stderr) + raise def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--caddy-image", required=True) parser.add_argument("--load-balancer-image", required=True) + parser.add_argument("--probe-image", required=True) args = parser.parse_args() for label, image in ( ("--caddy-image", args.caddy_image), ("--load-balancer-image", args.load_balancer_image), + ("--probe-image", args.probe_image), ): if DIGEST_IMAGE.fullmatch(image) is None: parser.error(f"{label} must be pinned by sha256 digest") - if shutil.which("docker") is None or shutil.which("curl") is None: - parser.error("docker and curl are required") + if shutil.which("docker") is None: + parser.error("docker is required") suffix = uuid4().hex[:10] network = f"govoplan-ingress-drill-{suffix}" @@ -276,6 +367,8 @@ def main() -> int: ingress, "--network", network, + "--network-alias", + "ingress", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", @@ -311,20 +404,22 @@ def main() -> int: requested_https_port, ): raise RuntimeError("Docker published unexpected ingress ports") - body = _wait_for_https(https_port) - if body.strip() != "proto=https": - raise RuntimeError(f"forwarded protocol was not normalized: {body!r}") - redirect = _curl(f"http://localhost:{http_port}/health", headers=True) - if not redirect.startswith("HTTP/1.1 308"): - raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}") + _probe_ingress( + image=args.probe_image, + network=network, + container=ingress, + ) _run(["docker", "rm", "--force", ingress]) _run(ingress_command) https_port = _published_port(ingress, 8443) - if _wait_for_https(https_port).strip() != "proto=https": - raise RuntimeError( - "managed ingress did not recover with persistent state" - ) + if https_port != requested_https_port: + raise RuntimeError("Docker changed the ingress TLS binding on restart") + _probe_ingress( + image=args.probe_image, + network=network, + container=ingress, + ) _run( [ "docker",