446 lines
14 KiB
Python
446 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise generated managed ingress with a real Caddy container."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from uuid import uuid4
|
|
|
|
|
|
META_ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(META_ROOT / "tools" / "deployment"))
|
|
|
|
from govoplan_deploy.bundle import ( # noqa: E402
|
|
render_caddy_config,
|
|
render_load_balancer_config,
|
|
)
|
|
from govoplan_deploy.model import default_spec # noqa: E402
|
|
|
|
|
|
DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
|
|
|
|
|
|
def _run(
|
|
argv: list[str],
|
|
*,
|
|
check: bool = True,
|
|
input_text: str | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
try:
|
|
return subprocess.run(
|
|
argv,
|
|
check=check,
|
|
capture_output=True,
|
|
input=input_text,
|
|
text=True,
|
|
timeout=60,
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
stderr = exc.stderr.strip()
|
|
if stderr:
|
|
print(stderr, file=sys.stderr)
|
|
raise
|
|
|
|
|
|
def _write_volume_file(
|
|
*,
|
|
image: str,
|
|
volume: str,
|
|
filename: str,
|
|
content: str,
|
|
) -> None:
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+", filename):
|
|
raise ValueError(f"invalid config filename: {filename!r}")
|
|
_run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--interactive",
|
|
"--user",
|
|
"0:0",
|
|
"--mount",
|
|
f"type=volume,src={volume},dst=/govoplan-config",
|
|
"--entrypoint",
|
|
"sh",
|
|
image,
|
|
"-c",
|
|
f"umask 022; cat > /govoplan-config/{filename}",
|
|
],
|
|
input_text=content,
|
|
)
|
|
|
|
|
|
def _published_port(container: str, target: int) -> int:
|
|
output = _run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{json .HostConfig.PortBindings}}",
|
|
container,
|
|
]
|
|
).stdout.strip()
|
|
try:
|
|
bindings = json.loads(output)[f"{target}/tcp"]
|
|
if not isinstance(bindings, list) or len(bindings) != 1:
|
|
raise ValueError("expected exactly one published binding")
|
|
binding = bindings[0]
|
|
if binding.get("HostIp") != "127.0.0.1":
|
|
raise ValueError("published binding is not loopback-only")
|
|
return int(binding["HostPort"])
|
|
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeError(
|
|
f"cannot determine loopback binding for {target}/tcp from {output!r}"
|
|
) from exc
|
|
|
|
|
|
def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
|
|
for _attempt in range(10):
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = int(listener.getsockname()[1])
|
|
if port not in exclude:
|
|
return port
|
|
raise RuntimeError("cannot allocate distinct loopback ports for ingress drill")
|
|
|
|
|
|
def _probe_ingress(*, image: str, network: str, container: str) -> None:
|
|
probe = r'''
|
|
import socket
|
|
import ssl
|
|
import time
|
|
|
|
|
|
def request(port, payload, *, tls):
|
|
connection = socket.create_connection(("ingress", port), timeout=3)
|
|
if tls:
|
|
connection = ssl._create_unverified_context().wrap_socket(
|
|
connection, server_hostname="localhost"
|
|
)
|
|
with connection:
|
|
connection.sendall(payload)
|
|
chunks = []
|
|
while True:
|
|
chunk = connection.recv(65536)
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
|
|
|
|
deadline = time.monotonic() + 30
|
|
last_error = ""
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = request(
|
|
8443,
|
|
b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
|
|
tls=True,
|
|
)
|
|
head, body_bytes = response.split(b"\r\n\r\n", 1)
|
|
status = int(head.split(b" ", 2)[1])
|
|
if status != 200:
|
|
raise RuntimeError(f"HTTPS returned {status}, expected 200")
|
|
body = body_bytes.decode("utf-8").strip()
|
|
if body != "proto=https":
|
|
raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
|
|
break
|
|
except Exception as exc:
|
|
last_error = f"{type(exc).__name__}: {exc}"
|
|
time.sleep(0.5)
|
|
else:
|
|
raise SystemExit(f"managed ingress did not become ready: {last_error}")
|
|
|
|
response = request(
|
|
8080,
|
|
b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
|
|
tls=False,
|
|
)
|
|
head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1")
|
|
lines = head.split("\r\n")
|
|
status = int(lines[0].split(" ", 2)[1])
|
|
if status != 308:
|
|
raise SystemExit(f"HTTP returned {status}, expected redirect 308")
|
|
headers = {
|
|
key.lower(): value.strip()
|
|
for key, separator, value in (line.partition(":") for line in lines[1:])
|
|
if separator
|
|
}
|
|
location = headers.get("location", "")
|
|
if not location.startswith("https://localhost"):
|
|
raise SystemExit(f"HTTP redirect had unexpected location: {location!r}")
|
|
'''
|
|
try:
|
|
_run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--network",
|
|
network,
|
|
"--read-only",
|
|
"--security-opt",
|
|
"no-new-privileges",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--entrypoint",
|
|
"python",
|
|
image,
|
|
"-c",
|
|
probe,
|
|
]
|
|
)
|
|
except subprocess.CalledProcessError:
|
|
logs = _run(["docker", "logs", container], check=False).stdout.strip()
|
|
if logs:
|
|
print(f"managed ingress logs:\n{logs}", file=sys.stderr)
|
|
raise
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--caddy-image", required=True)
|
|
parser.add_argument("--load-balancer-image", required=True)
|
|
parser.add_argument("--probe-image", required=True)
|
|
args = parser.parse_args()
|
|
for label, image in (
|
|
("--caddy-image", args.caddy_image),
|
|
("--load-balancer-image", args.load_balancer_image),
|
|
("--probe-image", args.probe_image),
|
|
):
|
|
if DIGEST_IMAGE.fullmatch(image) is None:
|
|
parser.error(f"{label} must be pinned by sha256 digest")
|
|
if shutil.which("docker") is None:
|
|
parser.error("docker is required")
|
|
|
|
suffix = uuid4().hex[:10]
|
|
network = f"govoplan-ingress-drill-{suffix}"
|
|
ingress = f"govoplan-ingress-{suffix}"
|
|
backend = f"govoplan-ingress-backend-{suffix}"
|
|
data_volume = f"govoplan-ingress-data-{suffix}"
|
|
config_volume = f"govoplan-ingress-config-{suffix}"
|
|
backend_config_volume = f"govoplan-ingress-backend-config-{suffix}"
|
|
ingress_config_volume = f"govoplan-ingress-caddy-config-{suffix}"
|
|
load_balancer_config_volume = f"govoplan-ingress-haproxy-config-{suffix}"
|
|
cleanup = [
|
|
["docker", "rm", "--force", ingress, backend],
|
|
["docker", "network", "rm", network],
|
|
[
|
|
"docker",
|
|
"volume",
|
|
"rm",
|
|
data_volume,
|
|
config_volume,
|
|
backend_config_volume,
|
|
ingress_config_volume,
|
|
load_balancer_config_volume,
|
|
],
|
|
]
|
|
try:
|
|
_run(["docker", "network", "create", network])
|
|
for volume in (
|
|
data_volume,
|
|
config_volume,
|
|
backend_config_volume,
|
|
ingress_config_volume,
|
|
load_balancer_config_volume,
|
|
):
|
|
_run(["docker", "volume", "create", volume])
|
|
with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory:
|
|
root = Path(directory)
|
|
backend_config = root / "backend.Caddyfile"
|
|
backend_config.write_text(
|
|
"""{
|
|
admin off
|
|
auto_https off
|
|
}
|
|
|
|
:8080 {
|
|
respond /health "proto={http.request.header.X-Forwarded-Proto}"
|
|
}
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
backend_config.chmod(0o644)
|
|
spec = default_spec(
|
|
profile="self-hosted",
|
|
public_url="https://localhost:8443",
|
|
ingress_mode="managed",
|
|
ingress_image=args.caddy_image,
|
|
ingress_https_port=8443,
|
|
acme_email="operator@example.test",
|
|
)
|
|
ingress_config = root / "Caddyfile"
|
|
ingress_config.write_text(render_caddy_config(spec), encoding="utf-8")
|
|
ingress_config.chmod(0o644)
|
|
load_balancer_config = root / "haproxy.cfg"
|
|
load_balancer_config.write_text(
|
|
render_load_balancer_config(spec),
|
|
encoding="utf-8",
|
|
)
|
|
load_balancer_config.chmod(0o644)
|
|
_write_volume_file(
|
|
image=args.load_balancer_image,
|
|
volume=load_balancer_config_volume,
|
|
filename="haproxy.cfg",
|
|
content=load_balancer_config.read_text(encoding="utf-8"),
|
|
)
|
|
_write_volume_file(
|
|
image=args.caddy_image,
|
|
volume=backend_config_volume,
|
|
filename="Caddyfile",
|
|
content=backend_config.read_text(encoding="utf-8"),
|
|
)
|
|
_write_volume_file(
|
|
image=args.caddy_image,
|
|
volume=ingress_config_volume,
|
|
filename="Caddyfile",
|
|
content=ingress_config.read_text(encoding="utf-8"),
|
|
)
|
|
_run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--read-only",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--mount",
|
|
(
|
|
"type=volume,"
|
|
f"src={load_balancer_config_volume},"
|
|
"dst=/usr/local/etc/haproxy,readonly"
|
|
),
|
|
args.load_balancer_image,
|
|
"haproxy",
|
|
"-c",
|
|
"-f",
|
|
"/usr/local/etc/haproxy/haproxy.cfg",
|
|
]
|
|
)
|
|
|
|
_run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--name",
|
|
backend,
|
|
"--network",
|
|
network,
|
|
"--network-alias",
|
|
"load-balancer",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:rw,noexec,nosuid,size=16m",
|
|
"--mount",
|
|
(
|
|
"type=volume,"
|
|
f"src={backend_config_volume},"
|
|
"dst=/govoplan-config,readonly"
|
|
),
|
|
args.caddy_image,
|
|
"caddy",
|
|
"run",
|
|
"--config",
|
|
"/govoplan-config/Caddyfile",
|
|
]
|
|
)
|
|
requested_http_port = _available_loopback_port()
|
|
requested_https_port = _available_loopback_port(
|
|
exclude=frozenset({requested_http_port})
|
|
)
|
|
ingress_command = [
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--name",
|
|
ingress,
|
|
"--network",
|
|
network,
|
|
"--network-alias",
|
|
"ingress",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:rw,noexec,nosuid,size=16m",
|
|
"--security-opt",
|
|
"no-new-privileges",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--publish",
|
|
f"127.0.0.1:{requested_http_port}:8080/tcp",
|
|
"--publish",
|
|
f"127.0.0.1:{requested_https_port}:8443/tcp",
|
|
"--mount",
|
|
(
|
|
"type=volume,"
|
|
f"src={ingress_config_volume},"
|
|
"dst=/govoplan-config,readonly"
|
|
),
|
|
"--mount",
|
|
f"type=volume,src={data_volume},dst=/data",
|
|
"--mount",
|
|
f"type=volume,src={config_volume},dst=/config",
|
|
args.caddy_image,
|
|
"caddy",
|
|
"run",
|
|
"--config",
|
|
"/govoplan-config/Caddyfile",
|
|
]
|
|
_run(ingress_command)
|
|
http_port = _published_port(ingress, 8080)
|
|
https_port = _published_port(ingress, 8443)
|
|
if (http_port, https_port) != (
|
|
requested_http_port,
|
|
requested_https_port,
|
|
):
|
|
raise RuntimeError("Docker published unexpected ingress ports")
|
|
_probe_ingress(
|
|
image=args.probe_image,
|
|
network=network,
|
|
container=ingress,
|
|
)
|
|
|
|
_run(["docker", "rm", "--force", ingress])
|
|
_run(ingress_command)
|
|
https_port = _published_port(ingress, 8443)
|
|
if https_port != requested_https_port:
|
|
raise RuntimeError("Docker changed the ingress TLS binding on restart")
|
|
_probe_ingress(
|
|
image=args.probe_image,
|
|
network=network,
|
|
container=ingress,
|
|
)
|
|
_run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--mount",
|
|
f"type=volume,src={data_volume},dst=/data,readonly",
|
|
"--entrypoint",
|
|
"sh",
|
|
args.caddy_image,
|
|
"-c",
|
|
'test -n "$(find /data -type f -print -quit)"',
|
|
]
|
|
)
|
|
finally:
|
|
for command in cleanup:
|
|
_run(command, check=False)
|
|
print("Managed ingress container drill passed.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|