diff --git a/.gitea/workflows/runtime-distribution.yml b/.gitea/workflows/runtime-distribution.yml index 0db84b3..546ec1a 100644 --- a/.gitea/workflows/runtime-distribution.yml +++ b/.gitea/workflows/runtime-distribution.yml @@ -162,6 +162,21 @@ jobs: WEB_DIGEST="sha256:$(sha256sum runtime-output/web-index.json | cut -d' ' -f1)" python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-api --index-digest "$API_DIGEST" --index runtime-output/api-index.json --output runtime-output/api-metadata.json python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-web --index-digest "$WEB_DIGEST" --index runtime-output/web-index.json --output runtime-output/web-metadata.json + - name: Exercise amd64 and arm64 runtime images + working-directory: govoplan + env: + POSTGRES_IMAGE: ${{ inputs.postgres_image }} + REDIS_IMAGE: ${{ inputs.redis_image }} + run: | + for ARCH in amd64 arm64; do + .runtime-build/bin/python tools/checks/runtime-image-smoke.py \ + --api-metadata runtime-output/api-metadata.json \ + --web-metadata runtime-output/web-metadata.json \ + --postgres-image "$POSTGRES_IMAGE" \ + --redis-image "$REDIS_IMAGE" \ + --platform "linux/$ARCH" \ + --output "runtime-output/evidence/runtime-smoke-$ARCH.json" + done - name: Generate and sign distribution evidence working-directory: govoplan env: @@ -209,14 +224,46 @@ jobs: openssl pkeyutl -sign -inkey runtime-output/signing-key.pem -rawin \ -in runtime-output/govoplan-deploy.pyz \ -out runtime-output/govoplan-deploy.pyz.sig - sha256sum runtime-output/govoplan-deploy.pyz > runtime-output/govoplan-deploy.pyz.sha256 - sha256sum runtime-output/distribution-manifest.json > runtime-output/distribution-manifest.json.sha256 + (cd runtime-output && sha256sum govoplan-deploy.pyz > govoplan-deploy.pyz.sha256) + (cd runtime-output && sha256sum distribution-manifest.json > distribution-manifest.json.sha256) rm runtime-output/signing-key.pem - name: Verify the published bundle contract with the zipapp working-directory: govoplan env: VERSION: ${{ inputs.version }} + SIGNING_KEY_ID: ${{ secrets.RUNTIME_DISTRIBUTION_SIGNING_KEY_ID }} run: | + (cd runtime-output && sha256sum --check govoplan-deploy.pyz.sha256) + (cd runtime-output && sha256sum --check distribution-manifest.json.sha256) + .runtime-build/bin/python - <<'PY' + import json + import os + from pathlib import Path + + keyring = json.loads( + Path("runtime-output/distribution-keyring.json").read_text(encoding="utf-8") + ) + key_id = os.environ["SIGNING_KEY_ID"] + matches = [item for item in keyring["keys"] if item.get("key_id") == key_id] + if len(matches) != 1 or matches[0].get("status") != "active": + raise SystemExit("runtime signing key is not uniquely active in the keyring") + Path("runtime-output/runtime-release-public.pem").write_text( + matches[0]["public_key_pem"], encoding="utf-8" + ) + PY + openssl pkeyutl -verify -pubin \ + -inkey runtime-output/runtime-release-public.pem -rawin \ + -in runtime-output/govoplan-deploy.pyz \ + -sigfile runtime-output/govoplan-deploy.pyz.sig + cp runtime-output/govoplan-deploy.pyz runtime-output/govoplan-deploy.tampered.pyz + printf '\0' >> runtime-output/govoplan-deploy.tampered.pyz + if openssl pkeyutl -verify -pubin \ + -inkey runtime-output/runtime-release-public.pem -rawin \ + -in runtime-output/govoplan-deploy.tampered.pyz \ + -sigfile runtime-output/govoplan-deploy.pyz.sig >/dev/null 2>&1; then + echo "Tampered deployment bootstrap unexpectedly verified" >&2 + exit 1 + fi MANIFEST_SHA256="$(cut -d' ' -f1 runtime-output/distribution-manifest.json.sha256)" python runtime-output/govoplan-deploy.pyz init \ --directory runtime-output/acceptance-install \ @@ -255,4 +302,6 @@ jobs: --asset runtime-output/evidence/api-sbom.cdx.json \ --asset runtime-output/evidence/web-sbom.cdx.json \ --asset runtime-output/evidence/api-provenance.json \ - --asset runtime-output/evidence/web-provenance.json + --asset runtime-output/evidence/web-provenance.json \ + --asset runtime-output/evidence/runtime-smoke-amd64.json \ + --asset runtime-output/evidence/runtime-smoke-arm64.json diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index f4e8262..73380cd 100644 --- a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -148,6 +148,16 @@ single-file deployer, its detached Ed25519 signature, and a signed, expiring distribution manifest. Evidence generation and signing run through the workflow's isolated release Python environment so their cryptographic tooling is explicit and independent of packages preinstalled in the Actions runner. +Before publication, the exact amd64 and arm64 image manifests each run release +migrations against the pinned PostgreSQL image, reach API and WebUI readiness +as non-root/read-only processes, and complete a task through the pinned Redis +image and packaged worker. Sanitized per-platform smoke receipts are retained +as immutable release assets. + +Ingress acceptance streams generated configuration into Docker-managed +volumes before starting the read-only containers. It therefore also works when +an Actions job reaches a host or remote Docker daemon through a mounted socket; +the drill never assumes that a job-container path is visible to that daemon. The manifest contract is [`runtime-distribution-manifest.schema.json`](runtime-distribution-manifest.schema.json), @@ -446,15 +456,27 @@ of the reviewed update recipe instead of a non-functional update button. ## Distribution Workflow -The downloadable entry point is a release asset. Obtain the zipapp, detached -signature, checksum, and trusted public keyring through independently -authenticated paths before execution: +The downloadable entry point is a reproducible release asset: sorted source +paths, fixed ZIP metadata, fixed compression settings, and identical source +bytes produce an identical zipapp regardless of checkout timestamps. Obtain the +zipapp, detached signature, checksum, and trusted public keyring through +independently authenticated paths before execution: ```sh curl --proto '=https' --tlsv1.2 --fail --location \ https://git.add-ideas.de/GovOPlaN/govoplan/releases/download/vX.Y.Z/govoplan-deploy.pyz \ --output govoplan-deploy.pyz sha256sum --check govoplan-deploy.pyz.sha256 +python3 - <<'PY' +import json +from pathlib import Path + +keyring = json.loads(Path("distribution-keyring.json").read_text()) +active = [key for key in keyring["keys"] if key["status"] == "active"] +if len(active) != 1: + raise SystemExit("expected exactly one active runtime release key") +Path("runtime-release-public.pem").write_text(active[0]["public_key_pem"]) +PY openssl pkeyutl -verify -pubin -inkey runtime-release-public.pem -rawin \ -in govoplan-deploy.pyz -sigfile govoplan-deploy.pyz.sig python3 govoplan-deploy.pyz init diff --git a/tests/test_managed_ingress_drill.py b/tests/test_managed_ingress_drill.py new file mode 100644 index 0000000..1423891 --- /dev/null +++ b/tests/test_managed_ingress_drill.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(): + path = ROOT / "tools/checks/managed-ingress-drill.py" + spec = importlib.util.spec_from_file_location("managed_ingress_drill", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +INGRESS = _load_module() + + +class ManagedIngressDrillTests(unittest.TestCase): + def test_config_is_streamed_into_a_daemon_visible_volume(self) -> None: + completed = subprocess.CompletedProcess([], 0, "", "") + with patch.object(INGRESS, "_run", return_value=completed) as run: + INGRESS._write_volume_file( + image="registry.example/caddy@sha256:" + "1" * 64, + volume="config-volume", + filename="Caddyfile", + content=":8080 { respond /health 200 }\n", + ) + + argv = run.call_args.args[0] + self.assertIn("type=volume,src=config-volume,dst=/govoplan-config", argv) + self.assertNotIn("type=bind", " ".join(argv)) + self.assertEqual( + ":8080 { respond /health 200 }\n", + run.call_args.kwargs["input_text"], + ) + + def test_config_filename_cannot_escape_the_volume(self) -> None: + with self.assertRaisesRegex(ValueError, "invalid config filename"): + INGRESS._write_volume_file( + image="registry.example/caddy@sha256:" + "1" * 64, + volume="config-volume", + filename="../Caddyfile", + content="", + ) + + def test_drill_has_no_runner_local_bind_mounts(self) -> None: + source = (ROOT / "tools/checks/managed-ingress-drill.py").read_text( + encoding="utf-8" + ) + + self.assertNotIn("type=bind", source) + self.assertIn('"--network-alias",\n "load-balancer"', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_distribution_build.py b/tests/test_runtime_distribution_build.py index a00b2ec..441880c 100644 --- a/tests/test_runtime_distribution_build.py +++ b/tests/test_runtime_distribution_build.py @@ -3,7 +3,9 @@ from __future__ import annotations import argparse import importlib.util import json +import os from pathlib import Path +import shutil import sys import tempfile import unittest @@ -26,9 +28,32 @@ FINALIZE = _load( "finalize_runtime_distribution", ROOT / "tools/release/finalize-runtime-distribution.py", ) +DEPLOYER_BUILD = _load( + "build_deployer_zipapp", + ROOT / "tools/deployment/build-deployer-zipapp.py", +) class RuntimeDistributionBuildTests(unittest.TestCase): + def test_deployment_zipapp_is_reproducible_across_source_mtimes(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-reproducible-zipapp-") as value: + root = Path(value) + source = root / "source" + shutil.copytree(ROOT / "tools/deployment", source) + first = root / "first.pyz" + second = root / "second.pyz" + original_root = DEPLOYER_BUILD.ROOT + try: + DEPLOYER_BUILD.ROOT = source + self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(first)])) + for path in source.rglob("*.py"): + os.utime(path, (2_000_000_000, 2_000_000_000)) + self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(second)])) + finally: + DEPLOYER_BUILD.ROOT = original_root + + self.assertEqual(first.read_bytes(), second.read_bytes()) + def test_workflow_signs_with_the_release_environment(self) -> None: workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text( encoding="utf-8" @@ -43,6 +68,35 @@ class RuntimeDistributionBuildTests(unittest.TestCase): workflow, ) + def test_workflow_verifies_portable_bootstrap_artifacts_before_execution( + self, + ) -> None: + workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text( + encoding="utf-8" + ) + + self.assertIn( + "(cd runtime-output && sha256sum govoplan-deploy.pyz > " + "govoplan-deploy.pyz.sha256)", + workflow, + ) + self.assertIn("openssl pkeyutl -verify -pubin", workflow) + self.assertIn("govoplan-deploy.tampered.pyz", workflow) + self.assertLess( + workflow.index("openssl pkeyutl -verify -pubin"), + workflow.index("python runtime-output/govoplan-deploy.pyz init"), + ) + + def test_workflow_retains_both_platform_runtime_smoke_receipts(self) -> None: + workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text( + encoding="utf-8" + ) + + self.assertIn('for ARCH in amd64 arm64; do', workflow) + self.assertIn("tools/checks/runtime-image-smoke.py", workflow) + self.assertIn("runtime-smoke-amd64.json", workflow) + self.assertIn("runtime-smoke-arm64.json", workflow) + def test_resolves_platforms_and_builds_evidence_descriptor(self) -> None: index = { "schemaVersion": 2, diff --git a/tests/test_runtime_image_smoke.py b/tests/test_runtime_image_smoke.py new file mode 100644 index 0000000..c74b9c9 --- /dev/null +++ b/tests/test_runtime_image_smoke.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "tools/checks/runtime-image-smoke.py" +SPEC = importlib.util.spec_from_file_location("runtime_image_smoke", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class RuntimeImageSmokeTests(unittest.TestCase): + def test_selects_the_exact_platform_digest(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value: + path = Path(value) / "metadata.json" + path.write_text( + json.dumps( + { + "index": "registry.example/api@sha256:" + "a" * 64, + "platforms": { + "linux/amd64": "registry.example/api@sha256:" + "1" * 64, + "linux/arm64": "registry.example/api@sha256:" + "2" * 64, + }, + } + ), + encoding="utf-8", + ) + + self.assertEqual( + "registry.example/api@sha256:" + "2" * 64, + MODULE.platform_image(path, "linux/arm64", "API"), + ) + + def test_rejects_mutable_or_missing_platform_images(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value: + path = Path(value) / "metadata.json" + path.write_text( + json.dumps({"platforms": {"linux/amd64": "registry.example/api:latest"}}), + encoding="utf-8", + ) + + with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"): + MODULE.platform_image(path, "linux/amd64", "API") + with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"): + MODULE.platform_image(path, "linux/arm64", "API") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/checks/managed-ingress-drill.py b/tools/checks/managed-ingress-drill.py index 1d39aa9..0af19b7 100644 --- a/tools/checks/managed-ingress-drill.py +++ b/tools/checks/managed-ingress-drill.py @@ -27,13 +27,52 @@ from govoplan_deploy.model import default_spec # noqa: E402 DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") -def _run(argv: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]: - return subprocess.run( - argv, - check=check, - capture_output=True, - text=True, - timeout=60, +def _run( + argv: list[str], + *, + check: bool = True, + input_text: str | None = None, +) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + argv, + check=check, + capture_output=True, + input=input_text, + text=True, + timeout=60, + ) + except subprocess.CalledProcessError as exc: + stderr = exc.stderr.strip() + if stderr: + print(stderr, file=sys.stderr) + raise + + +def _write_volume_file( + *, + image: str, + volume: str, + filename: str, + content: str, +) -> None: + if not re.fullmatch(r"[A-Za-z0-9_.-]+", filename): + raise ValueError(f"invalid config filename: {filename!r}") + _run( + [ + "docker", + "run", + "--rm", + "--interactive", + "--mount", + f"type=volume,src={volume},dst=/govoplan-config", + "--entrypoint", + "sh", + image, + "-c", + f"umask 022; cat > /govoplan-config/{filename}", + ], + input_text=content, ) @@ -85,15 +124,33 @@ def main() -> int: backend = f"govoplan-ingress-backend-{suffix}" data_volume = f"govoplan-ingress-data-{suffix}" config_volume = f"govoplan-ingress-config-{suffix}" + backend_config_volume = f"govoplan-ingress-backend-config-{suffix}" + ingress_config_volume = f"govoplan-ingress-caddy-config-{suffix}" + load_balancer_config_volume = f"govoplan-ingress-haproxy-config-{suffix}" cleanup = [ ["docker", "rm", "--force", ingress, backend], ["docker", "network", "rm", network], - ["docker", "volume", "rm", data_volume, config_volume], + [ + "docker", + "volume", + "rm", + data_volume, + config_volume, + backend_config_volume, + ingress_config_volume, + load_balancer_config_volume, + ], ] try: _run(["docker", "network", "create", network]) - _run(["docker", "volume", "create", data_volume]) - _run(["docker", "volume", "create", config_volume]) + for volume in ( + data_volume, + config_volume, + backend_config_volume, + ingress_config_volume, + load_balancer_config_volume, + ): + _run(["docker", "volume", "create", volume]) with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory: root = Path(directory) backend_config = root / "backend.Caddyfile" @@ -127,6 +184,24 @@ def main() -> int: encoding="utf-8", ) load_balancer_config.chmod(0o644) + _write_volume_file( + image=args.load_balancer_image, + volume=load_balancer_config_volume, + filename="haproxy.cfg", + content=load_balancer_config.read_text(encoding="utf-8"), + ) + _write_volume_file( + image=args.caddy_image, + volume=backend_config_volume, + filename="Caddyfile", + content=backend_config.read_text(encoding="utf-8"), + ) + _write_volume_file( + image=args.caddy_image, + volume=ingress_config_volume, + filename="Caddyfile", + content=ingress_config.read_text(encoding="utf-8"), + ) _run( [ "docker", @@ -136,7 +211,11 @@ def main() -> int: "--cap-drop", "ALL", "--mount", - f"type=bind,src={load_balancer_config},dst=/usr/local/etc/haproxy/haproxy.cfg,readonly", + ( + "type=volume," + f"src={load_balancer_config_volume}," + "dst=/usr/local/etc/haproxy,readonly" + ), args.load_balancer_image, "haproxy", "-c", @@ -154,16 +233,22 @@ def main() -> int: backend, "--network", network, + "--network-alias", + "load-balancer", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", "--mount", - f"type=bind,src={backend_config},dst=/etc/caddy/Caddyfile,readonly", + ( + "type=volume," + f"src={backend_config_volume}," + "dst=/govoplan-config,readonly" + ), args.caddy_image, "caddy", "run", "--config", - "/etc/caddy/Caddyfile", + "/govoplan-config/Caddyfile", ] ) ingress_command = [ @@ -186,7 +271,11 @@ def main() -> int: "--publish", "127.0.0.1::8443", "--mount", - f"type=bind,src={ingress_config},dst=/etc/caddy/Caddyfile,readonly", + ( + "type=volume," + f"src={ingress_config_volume}," + "dst=/govoplan-config,readonly" + ), "--mount", f"type=volume,src={data_volume},dst=/data", "--mount", @@ -195,7 +284,7 @@ def main() -> int: "caddy", "run", "--config", - "/etc/caddy/Caddyfile", + "/govoplan-config/Caddyfile", ] _run(ingress_command) http_port = _published_port(ingress, 8080) diff --git a/tools/checks/runtime-image-smoke.py b/tools/checks/runtime-image-smoke.py new file mode 100644 index 0000000..e05cf62 --- /dev/null +++ b/tools/checks/runtime-image-smoke.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +"""Exercise a pinned GovOPlaN runtime image pair on one OCI platform.""" + +from __future__ import annotations + +import argparse +import base64 +from datetime import UTC, datetime +import json +import os +from pathlib import Path +import re +import secrets +import subprocess +import time +from typing import Callable, Sequence + + +PLATFORMS = frozenset({"linux/amd64", "linux/arm64"}) +DIGEST_IMAGE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$") +BASE_MODULES = ( + "tenancy", + "organizations", + "identity", + "idm", + "access", + "admin", + "dashboard", + "policy", + "audit", + "docs", + "ops", +) + + +class SmokeError(RuntimeError): + """A runtime image failed its bounded acceptance drill.""" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--api-metadata", type=Path, required=True) + parser.add_argument("--web-metadata", type=Path, required=True) + parser.add_argument("--postgres-image", required=True) + parser.add_argument("--redis-image", required=True) + parser.add_argument("--platform", choices=sorted(PLATFORMS), required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout-seconds", type=float, default=600.0) + return parser + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _digest_image(value: object, label: str) -> str: + if not isinstance(value, str) or DIGEST_IMAGE.fullmatch(value) is None: + raise SmokeError(f"{label} must be an exact sha256 image reference") + return value + + +def platform_image(path: Path, platform: str, label: str) -> str: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SmokeError(f"cannot read {label} OCI metadata") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("platforms"), dict): + raise SmokeError(f"{label} OCI metadata has no platform map") + return _digest_image(payload["platforms"].get(platform), f"{label} {platform}") + + +def _tail(value: str, *, limit: int = 4000) -> str: + return value[-limit:].strip() + + +def _run( + arguments: Sequence[str], + *, + check: bool = True, + timeout: float = 600.0, + redactions: Sequence[str] = (), +) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + list(arguments), + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise SmokeError(f"container command could not complete: {type(exc).__name__}") from exc + if check and result.returncode != 0: + detail = _tail(result.stderr or result.stdout or "no diagnostic output") + for secret in redactions: + if secret: + detail = detail.replace(secret, "[redacted]") + raise SmokeError(f"container command failed: {detail}") + return result + + +def _wait_for( + label: str, + probe: Callable[[], subprocess.CompletedProcess[str]], + *, + timeout: float, +) -> None: + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + result = probe() + if result.returncode == 0: + return + last = _tail(result.stderr or result.stdout) + time.sleep(2.0) + raise SmokeError(f"{label} did not become ready: {last or 'probe failed'}") + + +def _environment( + *, + database_password: str, + master_key: str, + platform_slug: str, +) -> dict[str, str]: + database = ( + f"postgresql+psycopg://govoplan:{database_password}@postgres:5432/govoplan" + ) + return { + "APP_ENV": "dev", + "GOVOPLAN_INSTALL_PROFILE": "evaluation", + "GOVOPLAN_INSTALLATION_ID": f"runtime-smoke-{platform_slug}", + "GOVOPLAN_STATE_PROFILE": "host-shared", + "GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "5", + "GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "30", + "GOVOPLAN_EXPECTED_API_REPLICAS": "1", + "GOVOPLAN_EXPECTED_WORKER_REPLICAS": "1", + "DATABASE_URL": database, + "GOVOPLAN_DATABASE_URL_PGTOOLS": ( + f"postgresql://govoplan:{database_password}@postgres:5432/govoplan" + ), + "GOVOPLAN_DB_CONNECTION_LIMIT": "100", + "GOVOPLAN_DB_CONNECTION_RESERVE": "10", + "REDIS_URL": "redis://redis:6379/0", + "CELERY_ENABLED": "true", + "CELERY_QUEUES": "default", + "CELERY_WORKER_CONCURRENCY": "1", + "ENABLED_MODULES": ",".join(BASE_MODULES), + "GOVOPLAN_MIGRATION_TRACK": "release", + "DEV_AUTO_MIGRATE_ENABLED": "false", + "DEV_BOOTSTRAP_ENABLED": "false", + "AUTH_LOGIN_THROTTLE_ENABLED": "true", + "AUTH_COOKIE_SECURE": "false", + "CORS_ORIGINS": "http://localhost", + "GOVOPLAN_TRUSTED_HOSTS": "127.0.0.1,localhost,api", + "FORWARDED_ALLOW_IPS": "127.0.0.1", + "MASTER_KEY_B64": master_key, + "FILE_STORAGE_BACKEND": "local", + "FILE_STORAGE_LOCAL_ROOT": "/var/lib/govoplan/files", + "GOVOPLAN_MODULE_LIVE_APPLY_ENABLED": "false", + } + + +def _env_arguments(values: dict[str, str], *, role: str, node_id: str) -> list[str]: + arguments: list[str] = [] + for key, value in sorted( + {**values, "GOVOPLAN_RUNTIME_ROLE": role, "GOVOPLAN_NODE_ID": node_id}.items() + ): + arguments.extend(("--env", f"{key}={value}")) + return arguments + + +def run_smoke( + *, + api_image: str, + web_image: str, + postgres_image: str, + redis_image: str, + platform: str, + timeout: float, +) -> dict[str, object]: + for label, value in ( + ("API image", api_image), + ("Web image", web_image), + ("PostgreSQL image", postgres_image), + ("Redis image", redis_image), + ): + _digest_image(value, label) + if platform not in PLATFORMS: + raise SmokeError(f"unsupported runtime smoke platform: {platform}") + + slug = platform.replace("linux/", "").replace("/", "-") + suffix = secrets.token_hex(4) + prefix = f"govoplan-runtime-{slug}-{suffix}" + names = { + "network": f"{prefix}-network", + "volume": f"{prefix}-data", + "postgres": f"{prefix}-postgres", + "redis": f"{prefix}-redis", + "api": f"{prefix}-api", + "web": f"{prefix}-web", + "worker": f"{prefix}-worker", + } + database_password = secrets.token_hex(20) + master_key = base64.urlsafe_b64encode(os.urandom(32)).decode("ascii") + redactions = (database_password, master_key) + environment = _environment( + database_password=database_password, + master_key=master_key, + platform_slug=slug, + ) + checks: list[dict[str, object]] = [] + started = time.monotonic() + + def record(check_id: str, began: float) -> None: + checks.append( + { + "id": check_id, + "state": "passed", + "duration_seconds": round(time.monotonic() - began, 3), + } + ) + + common_runtime = [ + "--platform", + platform, + "--network", + names["network"], + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=64m", + "--security-opt", + "no-new-privileges:true", + "--cap-drop", + "ALL", + "--mount", + f"type=volume,source={names['volume']},target=/var/lib/govoplan", + ] + + try: + _run(("docker", "network", "create", names["network"]), timeout=timeout) + _run(("docker", "volume", "create", names["volume"]), timeout=timeout) + + began = time.monotonic() + _run( + ( + "docker", + "run", + "--detach", + "--platform", + platform, + "--name", + names["postgres"], + "--network", + names["network"], + "--network-alias", + "postgres", + "--env", + "POSTGRES_DB=govoplan", + "--env", + "POSTGRES_USER=govoplan", + "--env", + f"POSTGRES_PASSWORD={database_password}", + "--tmpfs", + "/var/lib/postgresql/data:rw,noexec,nosuid,size=384m", + postgres_image, + ), + timeout=timeout, + redactions=redactions, + ) + _run( + ( + "docker", + "run", + "--detach", + "--platform", + platform, + "--name", + names["redis"], + "--network", + names["network"], + "--network-alias", + "redis", + "--read-only", + "--tmpfs", + "/data:rw,noexec,nosuid,size=64m", + redis_image, + "redis-server", + "--save", + "", + "--appendonly", + "no", + ), + timeout=timeout, + ) + _wait_for( + "PostgreSQL", + lambda: _run( + ( + "docker", + "exec", + names["postgres"], + "pg_isready", + "--username", + "govoplan", + "--dbname", + "govoplan", + ), + check=False, + timeout=30, + ), + timeout=timeout, + ) + _wait_for( + "Redis", + lambda: _run( + ("docker", "exec", names["redis"], "redis-cli", "ping"), + check=False, + timeout=30, + ), + timeout=timeout, + ) + record("managed_dependencies_ready", began) + + began = time.monotonic() + _run( + ( + "docker", + "run", + "--rm", + "--name", + f"{prefix}-migrate", + *common_runtime, + *_env_arguments( + environment, + role="migration", + node_id=f"runtime-smoke-{slug}-migration", + ), + api_image, + "python", + "-m", + "govoplan_core.commands.init_db", + "--migration-track", + "release", + ), + timeout=timeout, + redactions=redactions, + ) + record("release_migrations", began) + + began = time.monotonic() + _run( + ( + "docker", + "run", + "--detach", + "--name", + names["api"], + "--network-alias", + "api", + *common_runtime, + *_env_arguments( + environment, + role="api", + node_id=f"runtime-smoke-{slug}-api", + ), + api_image, + ), + timeout=timeout, + redactions=redactions, + ) + _wait_for( + "GovOPlaN API", + lambda: _run( + ( + "docker", + "exec", + names["api"], + "python", + "-c", + ( + "import urllib.request;" + "r=urllib.request.Request('http://127.0.0.1:8000/health/ready'," + "headers={'Host':'127.0.0.1'});" + "assert urllib.request.urlopen(r,timeout=3).status==200" + ), + ), + check=False, + timeout=30, + ), + timeout=timeout, + ) + _run( + ( + "docker", + "exec", + names["api"], + "python", + "-c", + "import os; assert os.getuid() == 10001", + ), + timeout=30, + ) + record("api_non_root_readiness", began) + + began = time.monotonic() + _run( + ( + "docker", + "run", + "--detach", + "--platform", + platform, + "--name", + names["web"], + "--network", + names["network"], + "--network-alias", + "web", + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=64m", + "--security-opt", + "no-new-privileges:true", + "--cap-drop", + "ALL", + web_image, + ), + timeout=timeout, + ) + _wait_for( + "GovOPlaN WebUI", + lambda: _run( + ( + "docker", + "exec", + names["api"], + "python", + "-c", + ( + "import urllib.request;" + "assert urllib.request.urlopen('http://web:8080/health',timeout=3).status==200;" + "assert urllib.request.urlopen('http://web:8080/',timeout=3).status==200" + ), + ), + check=False, + timeout=30, + ), + timeout=timeout, + ) + _run( + ("docker", "exec", names["web"], "sh", "-c", "test \"$(id -u)\" = 101"), + timeout=30, + ) + record("web_non_root_readiness", began) + + began = time.monotonic() + _run( + ( + "docker", + "run", + "--detach", + "--name", + names["worker"], + *common_runtime, + *_env_arguments( + environment, + role="worker", + node_id=f"runtime-smoke-{slug}-worker", + ), + api_image, + "python", + "-m", + "celery", + "-A", + "govoplan_core.celery_app:celery", + "worker", + "--queues", + "default", + "--pool", + "solo", + "--concurrency", + "1", + "--hostname", + f"runtime-smoke-{slug}@%h", + "--loglevel", + "WARNING", + ), + timeout=timeout, + redactions=redactions, + ) + _wait_for( + "GovOPlaN worker", + lambda: _run( + ( + "docker", + "exec", + names["api"], + "python", + "-c", + ( + "from govoplan_core.celery_app import celery;" + "result=celery.send_task('govoplan.ping',queue='default');" + "assert result.get(timeout=10)=='pong'" + ), + ), + check=False, + timeout=30, + ), + timeout=timeout, + ) + _run(("docker", "stop", "--time", "20", names["worker"]), timeout=30) + record("worker_delivery_and_shutdown", began) + except SmokeError as exc: + for role in ("api", "web", "worker", "postgres", "redis"): + result = _run( + ("docker", "logs", "--tail", "100", names[role]), + check=False, + timeout=30, + ) + if result.stdout or result.stderr: + detail = _tail(result.stdout + result.stderr, limit=8000) + for secret in redactions: + detail = detail.replace(secret, "[redacted]") + print(f"--- {role} logs ---\n{detail}") + raise exc + finally: + for role in ("worker", "web", "api", "redis", "postgres"): + _run( + ("docker", "rm", "--force", names[role]), + check=False, + timeout=30, + ) + _run(("docker", "volume", "rm", "--force", names["volume"]), check=False) + _run(("docker", "network", "rm", names["network"]), check=False) + + return { + "schema_version": "1", + "evidence_kind": "govoplan.runtime-image-smoke", + "captured_at": _utc_now(), + "platform": platform, + "images": { + "api": api_image, + "web": web_image, + "postgres": postgres_image, + "redis": redis_image, + }, + "result": {"state": "passed"}, + "checks": checks, + "duration_seconds": round(time.monotonic() - started, 3), + } + + +def main() -> int: + args = build_parser().parse_args() + try: + api_image = platform_image(args.api_metadata, args.platform, "API") + web_image = platform_image(args.web_metadata, args.platform, "Web") + evidence = run_smoke( + api_image=api_image, + web_image=web_image, + postgres_image=args.postgres_image, + redis_image=args.redis_image, + platform=args.platform, + timeout=args.timeout_seconds, + ) + except (OSError, SmokeError, ValueError) as exc: + print(f"runtime image smoke failed: {exc}") + return 1 + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.chmod(0o644) + temporary.replace(args.output) + print(f"Runtime image smoke evidence written to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deployment/build-deployer-zipapp.py b/tools/deployment/build-deployer-zipapp.py index 01370f2..b8808d4 100644 --- a/tools/deployment/build-deployer-zipapp.py +++ b/tools/deployment/build-deployer-zipapp.py @@ -6,11 +6,13 @@ from __future__ import annotations import argparse from hashlib import sha256 from pathlib import Path -import zipapp +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo ROOT = Path(__file__).resolve().parent DEFAULT_OUTPUT = ROOT.parent.parent / "runtime" / "deployment" / "govoplan-deploy.pyz" +ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +PYTHON_FILE_MODE = 0o100644 def main(argv: list[str] | None = None) -> int: @@ -25,13 +27,7 @@ def main(argv: list[str] | None = None) -> int: temporary = output.with_name(f".{output.name}.tmp") if temporary.exists(): temporary.unlink() - zipapp.create_archive( - ROOT, - target=temporary, - interpreter="/usr/bin/env python3", - compressed=True, - filter=_include_source, - ) + _write_reproducible_zipapp(temporary) temporary.chmod(0o755) temporary.replace(output) digest = sha256(output.read_bytes()).hexdigest() @@ -39,6 +35,42 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _write_reproducible_zipapp(target: Path) -> None: + sources = tuple( + path + for path in sorted(ROOT.rglob("*"), key=lambda item: item.as_posix()) + if path.is_file() and _include_source(path.relative_to(ROOT)) + ) + if not any(path.relative_to(ROOT).as_posix() == "__main__.py" for path in sources): + raise ValueError("deployment source has no __main__.py") + for path in sources: + if path.is_symlink(): + raise ValueError(f"deployment source must not contain symlinks: {path}") + + with target.open("wb") as handle: + handle.write(b"#!/usr/bin/env python3\n") + with ZipFile( + handle, + mode="w", + compression=ZIP_DEFLATED, + compresslevel=9, + strict_timestamps=True, + ) as archive: + for source in sources: + relative = source.relative_to(ROOT).as_posix() + info = ZipInfo(relative, date_time=ZIP_TIMESTAMP) + info.compress_type = ZIP_DEFLATED + info.create_system = 3 + info.external_attr = PYTHON_FILE_MODE << 16 + info.flag_bits |= 0x800 + archive.writestr( + info, + source.read_bytes(), + compress_type=ZIP_DEFLATED, + compresslevel=9, + ) + + def _include_source(path: Path) -> bool: return ( "__pycache__" not in path.parts