Probe managed ingress across Docker namespaces

.gitea/workflows/runtime-distribution.yml

@@ -342,6 +342,7 @@ jobs:
           python tools/checks/managed-ingress-drill.py
           --caddy-image "$MANAGED_INGRESS_IMAGE"
           --load-balancer-image "$LOAD_BALANCER_IMAGE"
+          --probe-image "$(jq -r '.platforms[\"linux/amd64\"]' runtime-output/api-metadata.json)"
       - name: Publish immutable Gitea release assets
         working-directory: govoplan
         env:

docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md

@@ -177,7 +177,10 @@ volumes before starting the read-only containers. It therefore also works when
 an Actions job reaches a host or remote Docker daemon through a mounted socket;
 the drill never assumes that a job-container path is visible to that daemon.
 The drill allocates explicit loopback-only host ports and verifies Docker's
-published mappings, avoiding daemon-specific random-port shorthand behavior.
+host binding configuration, avoiding daemon-specific random-port shorthand
+behavior. Because an Actions job and deployment containers may be Docker
+siblings, functional HTTP/TLS checks run from the digest-pinned API image on
+the deployment network instead of assuming the Docker host is job-local.
 The bounded setup helper writes only generated public configuration as root so
 it can initialize a new volume; the actual HAProxy process retains the image's
 non-root identity and runs read-only with all capabilities dropped.

tests/test_managed_ingress_drill.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import importlib.util
 from pathlib import Path
+import json
 import subprocess
 import sys
 import unittest
@@ -68,7 +69,13 @@ class ManagedIngressDrillTests(unittest.TestCase):
         completed = subprocess.CompletedProcess(
             [],
             0,
-            "127.0.0.1:49152\n",
+            json.dumps(
+                {
+                    "8443/tcp": [
+                        {"HostIp": "127.0.0.1", "HostPort": "49152"}
+                    ]
+                }
+            ),
             "",
         )
         with patch.object(INGRESS, "_run", return_value=completed) as run:
@@ -76,10 +83,45 @@ class ManagedIngressDrillTests(unittest.TestCase):
 
         self.assertEqual(49152, port)
         self.assertEqual(
-            ["docker", "port", "ingress", "8443/tcp"],
+            [
+                "docker",
+                "inspect",
+                "--format",
+                "{{json .HostConfig.PortBindings}}",
+                "ingress",
+            ],
             run.call_args.args[0],
         )
 
+    def test_published_port_rejects_non_loopback_binding(self) -> None:
+        completed = subprocess.CompletedProcess(
+            [],
+            0,
+            '{"8443/tcp":[{"HostIp":"0.0.0.0","HostPort":"49152"}]}',
+            "",
+        )
+        with patch.object(INGRESS, "_run", return_value=completed):
+            with self.assertRaisesRegex(RuntimeError, "loopback binding"):
+                INGRESS._published_port("ingress", 8443)
+
+    def test_probe_runs_as_a_network_sibling_from_a_digest_image(self) -> None:
+        completed = subprocess.CompletedProcess([], 0, "", "")
+        image = "registry.example/runtime-api@sha256:" + "1" * 64
+        with patch.object(INGRESS, "_run", return_value=completed) as run:
+            INGRESS._probe_ingress(
+                image=image,
+                network="deployment-network",
+                container="ingress",
+            )
+
+        argv = run.call_args.args[0]
+        self.assertEqual("docker", argv[0])
+        self.assertIn("deployment-network", argv)
+        self.assertIn(image, argv)
+        self.assertIn('(\"ingress\", port)', argv[-1])
+        self.assertIn("server_hostname=\"localhost\"", argv[-1])
+        self.assertNotIn("localhost:49152", argv[-1])
+
 
 if __name__ == "__main__":
     unittest.main()

tools/checks/managed-ingress-drill.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import argparse
+import json
 from pathlib import Path
 import re
 import shutil
@@ -11,7 +12,6 @@ import socket
 import subprocess
 import sys
 import tempfile
-import time
 from uuid import uuid4
 
 
@@ -80,11 +80,27 @@ def _write_volume_file(
 
 
 def _published_port(container: str, target: int) -> int:
-    output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip()
+    output = _run(
+        [
+            "docker",
+            "inspect",
+            "--format",
+            "{{json .HostConfig.PortBindings}}",
+            container,
+        ]
+    ).stdout.strip()
     try:
-        return int(output.rsplit(":", 1)[1])
-    except (IndexError, ValueError) as exc:
-        raise RuntimeError(f"cannot determine published port from {output!r}") from exc
+        bindings = json.loads(output)[f"{target}/tcp"]
+        if not isinstance(bindings, list) or len(bindings) != 1:
+            raise ValueError("expected exactly one published binding")
+        binding = bindings[0]
+        if binding.get("HostIp") != "127.0.0.1":
+            raise ValueError("published binding is not loopback-only")
+        return int(binding["HostPort"])
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+        raise RuntimeError(
+            f"cannot determine loopback binding for {target}/tcp from {output!r}"
+        ) from exc
 
 
 def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
@@ -97,39 +113,114 @@ def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
     raise RuntimeError("cannot allocate distinct loopback ports for ingress drill")
 
 
-def _curl(url: str, *, headers: bool = False) -> str:
-    argv = ["curl", "--silent", "--show-error", "--insecure"]
-    if headers:
-        argv.extend(["--head"])
-    argv.append(url)
-    return _run(argv).stdout
+def _probe_ingress(*, image: str, network: str, container: str) -> None:
+    probe = r'''
+import socket
+import ssl
+import time
 
 
-def _wait_for_https(port: int) -> str:
-    deadline = time.monotonic() + 30
-    last_error = ""
-    while time.monotonic() < deadline:
-        try:
-            return _curl(f"https://localhost:{port}/health")
-        except subprocess.CalledProcessError as exc:
-            last_error = exc.stderr.strip()
-            time.sleep(0.5)
-    raise RuntimeError(f"managed ingress did not become ready: {last_error}")
+def request(port, payload, *, tls):
+    connection = socket.create_connection(("ingress", port), timeout=3)
+    if tls:
+        connection = ssl._create_unverified_context().wrap_socket(
+            connection, server_hostname="localhost"
+        )
+    with connection:
+        connection.sendall(payload)
+        chunks = []
+        while True:
+            chunk = connection.recv(65536)
+            if not chunk:
+                break
+            chunks.append(chunk)
+    return b"".join(chunks)
+
+
+deadline = time.monotonic() + 30
+last_error = ""
+while time.monotonic() < deadline:
+    try:
+        response = request(
+            8443,
+            b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+            tls=True,
+        )
+        head, body_bytes = response.split(b"\r\n\r\n", 1)
+        status = int(head.split(b" ", 2)[1])
+        if status != 200:
+            raise RuntimeError(f"HTTPS returned {status}, expected 200")
+        body = body_bytes.decode("utf-8").strip()
+        if body != "proto=https":
+            raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
+        break
+    except Exception as exc:
+        last_error = f"{type(exc).__name__}: {exc}"
+        time.sleep(0.5)
+else:
+    raise SystemExit(f"managed ingress did not become ready: {last_error}")
+
+response = request(
+    8080,
+    b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+    tls=False,
+)
+head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1")
+lines = head.split("\r\n")
+status = int(lines[0].split(" ", 2)[1])
+if status != 308:
+    raise SystemExit(f"HTTP returned {status}, expected redirect 308")
+headers = {
+    key.lower(): value.strip()
+    for key, separator, value in (line.partition(":") for line in lines[1:])
+    if separator
+}
+location = headers.get("location", "")
+if not location.startswith("https://localhost"):
+    raise SystemExit(f"HTTP redirect had unexpected location: {location!r}")
+'''
+    try:
+        _run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--network",
+                network,
+                "--read-only",
+                "--security-opt",
+                "no-new-privileges",
+                "--cap-drop",
+                "ALL",
+                "--entrypoint",
+                "python",
+                image,
+                "-c",
+                probe,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        logs = _run(["docker", "logs", container], check=False).stdout.strip()
+        if logs:
+            print(f"managed ingress logs:\n{logs}", file=sys.stderr)
+        raise
 
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--caddy-image", required=True)
     parser.add_argument("--load-balancer-image", required=True)
+    parser.add_argument("--probe-image", required=True)
     args = parser.parse_args()
     for label, image in (
         ("--caddy-image", args.caddy_image),
         ("--load-balancer-image", args.load_balancer_image),
+        ("--probe-image", args.probe_image),
     ):
         if DIGEST_IMAGE.fullmatch(image) is None:
             parser.error(f"{label} must be pinned by sha256 digest")
-    if shutil.which("docker") is None or shutil.which("curl") is None:
-        parser.error("docker and curl are required")
+    if shutil.which("docker") is None:
+        parser.error("docker is required")
 
     suffix = uuid4().hex[:10]
     network = f"govoplan-ingress-drill-{suffix}"
@@ -276,6 +367,8 @@ def main() -> int:
                 ingress,
                 "--network",
                 network,
+                "--network-alias",
+                "ingress",
                 "--read-only",
                 "--tmpfs",
                 "/tmp:rw,noexec,nosuid,size=16m",
@@ -311,20 +404,22 @@ def main() -> int:
                 requested_https_port,
             ):
                 raise RuntimeError("Docker published unexpected ingress ports")
-            body = _wait_for_https(https_port)
-            if body.strip() != "proto=https":
-                raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
-            redirect = _curl(f"http://localhost:{http_port}/health", headers=True)
-            if not redirect.startswith("HTTP/1.1 308"):
-                raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
 
             _run(["docker", "rm", "--force", ingress])
             _run(ingress_command)
             https_port = _published_port(ingress, 8443)
-            if _wait_for_https(https_port).strip() != "proto=https":
-                raise RuntimeError(
-                    "managed ingress did not recover with persistent state"
-                )
+            if https_port != requested_https_port:
+                raise RuntimeError("Docker changed the ingress TLS binding on restart")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
             _run(
                 [
                     "docker",