Probe managed ingress across Docker namespaces

tools/checks/managed-ingress-drill.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import argparse
+import json
 from pathlib import Path
 import re
 import shutil
@@ -11,7 +12,6 @@ import socket
 import subprocess
 import sys
 import tempfile
-import time
 from uuid import uuid4
 
 
@@ -80,11 +80,27 @@ def _write_volume_file(
 
 
 def _published_port(container: str, target: int) -> int:
-    output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip()
+    output = _run(
+        [
+            "docker",
+            "inspect",
+            "--format",
+            "{{json .HostConfig.PortBindings}}",
+            container,
+        ]
+    ).stdout.strip()
     try:
-        return int(output.rsplit(":", 1)[1])
-    except (IndexError, ValueError) as exc:
-        raise RuntimeError(f"cannot determine published port from {output!r}") from exc
+        bindings = json.loads(output)[f"{target}/tcp"]
+        if not isinstance(bindings, list) or len(bindings) != 1:
+            raise ValueError("expected exactly one published binding")
+        binding = bindings[0]
+        if binding.get("HostIp") != "127.0.0.1":
+            raise ValueError("published binding is not loopback-only")
+        return int(binding["HostPort"])
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+        raise RuntimeError(
+            f"cannot determine loopback binding for {target}/tcp from {output!r}"
+        ) from exc
 
 
 def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
@@ -97,39 +113,114 @@ def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
     raise RuntimeError("cannot allocate distinct loopback ports for ingress drill")
 
 
-def _curl(url: str, *, headers: bool = False) -> str:
-    argv = ["curl", "--silent", "--show-error", "--insecure"]
-    if headers:
-        argv.extend(["--head"])
-    argv.append(url)
-    return _run(argv).stdout
+def _probe_ingress(*, image: str, network: str, container: str) -> None:
+    probe = r'''
+import socket
+import ssl
+import time
 
 
-def _wait_for_https(port: int) -> str:
-    deadline = time.monotonic() + 30
-    last_error = ""
-    while time.monotonic() < deadline:
-        try:
-            return _curl(f"https://localhost:{port}/health")
-        except subprocess.CalledProcessError as exc:
-            last_error = exc.stderr.strip()
-            time.sleep(0.5)
-    raise RuntimeError(f"managed ingress did not become ready: {last_error}")
+def request(port, payload, *, tls):
+    connection = socket.create_connection(("ingress", port), timeout=3)
+    if tls:
+        connection = ssl._create_unverified_context().wrap_socket(
+            connection, server_hostname="localhost"
+        )
+    with connection:
+        connection.sendall(payload)
+        chunks = []
+        while True:
+            chunk = connection.recv(65536)
+            if not chunk:
+                break
+            chunks.append(chunk)
+    return b"".join(chunks)
+
+
+deadline = time.monotonic() + 30
+last_error = ""
+while time.monotonic() < deadline:
+    try:
+        response = request(
+            8443,
+            b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+            tls=True,
+        )
+        head, body_bytes = response.split(b"\r\n\r\n", 1)
+        status = int(head.split(b" ", 2)[1])
+        if status != 200:
+            raise RuntimeError(f"HTTPS returned {status}, expected 200")
+        body = body_bytes.decode("utf-8").strip()
+        if body != "proto=https":
+            raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
+        break
+    except Exception as exc:
+        last_error = f"{type(exc).__name__}: {exc}"
+        time.sleep(0.5)
+else:
+    raise SystemExit(f"managed ingress did not become ready: {last_error}")
+
+response = request(
+    8080,
+    b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+    tls=False,
+)
+head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1")
+lines = head.split("\r\n")
+status = int(lines[0].split(" ", 2)[1])
+if status != 308:
+    raise SystemExit(f"HTTP returned {status}, expected redirect 308")
+headers = {
+    key.lower(): value.strip()
+    for key, separator, value in (line.partition(":") for line in lines[1:])
+    if separator
+}
+location = headers.get("location", "")
+if not location.startswith("https://localhost"):
+    raise SystemExit(f"HTTP redirect had unexpected location: {location!r}")
+'''
+    try:
+        _run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--network",
+                network,
+                "--read-only",
+                "--security-opt",
+                "no-new-privileges",
+                "--cap-drop",
+                "ALL",
+                "--entrypoint",
+                "python",
+                image,
+                "-c",
+                probe,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        logs = _run(["docker", "logs", container], check=False).stdout.strip()
+        if logs:
+            print(f"managed ingress logs:\n{logs}", file=sys.stderr)
+        raise
 
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--caddy-image", required=True)
     parser.add_argument("--load-balancer-image", required=True)
+    parser.add_argument("--probe-image", required=True)
     args = parser.parse_args()
     for label, image in (
         ("--caddy-image", args.caddy_image),
         ("--load-balancer-image", args.load_balancer_image),
+        ("--probe-image", args.probe_image),
     ):
         if DIGEST_IMAGE.fullmatch(image) is None:
             parser.error(f"{label} must be pinned by sha256 digest")
-    if shutil.which("docker") is None or shutil.which("curl") is None:
-        parser.error("docker and curl are required")
+    if shutil.which("docker") is None:
+        parser.error("docker is required")
 
     suffix = uuid4().hex[:10]
     network = f"govoplan-ingress-drill-{suffix}"
@@ -276,6 +367,8 @@ def main() -> int:
                 ingress,
                 "--network",
                 network,
+                "--network-alias",
+                "ingress",
                 "--read-only",
                 "--tmpfs",
                 "/tmp:rw,noexec,nosuid,size=16m",
@@ -311,20 +404,22 @@ def main() -> int:
                 requested_https_port,
             ):
                 raise RuntimeError("Docker published unexpected ingress ports")
-            body = _wait_for_https(https_port)
-            if body.strip() != "proto=https":
-                raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
-            redirect = _curl(f"http://localhost:{http_port}/health", headers=True)
-            if not redirect.startswith("HTTP/1.1 308"):
-                raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
 
             _run(["docker", "rm", "--force", ingress])
             _run(ingress_command)
             https_port = _published_port(ingress, 8443)
-            if _wait_for_https(https_port).strip() != "proto=https":
-                raise RuntimeError(
-                    "managed ingress did not recover with persistent state"
-                )
+            if https_port != requested_https_port:
+                raise RuntimeError("Docker changed the ingress TLS binding on restart")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
             _run(
                 [
                     "docker",