580 lines
18 KiB
Python
580 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise a pinned GovOPlaN runtime image pair on one OCI platform."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
from datetime import UTC, datetime
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import subprocess
|
|
import time
|
|
from typing import Callable, Sequence
|
|
|
|
|
|
PLATFORMS = frozenset({"linux/amd64", "linux/arm64"})
|
|
DIGEST_IMAGE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$")
|
|
BASE_MODULES = (
|
|
"tenancy",
|
|
"organizations",
|
|
"identity",
|
|
"idm",
|
|
"access",
|
|
"admin",
|
|
"dashboard",
|
|
"policy",
|
|
"audit",
|
|
"docs",
|
|
"ops",
|
|
)
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
"""A runtime image failed its bounded acceptance drill."""
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--api-metadata", type=Path, required=True)
|
|
parser.add_argument("--web-metadata", type=Path, required=True)
|
|
parser.add_argument("--postgres-image", required=True)
|
|
parser.add_argument("--redis-image", required=True)
|
|
parser.add_argument("--platform", choices=sorted(PLATFORMS), required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--timeout-seconds", type=float, default=600.0)
|
|
return parser
|
|
|
|
|
|
def _utc_now() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _digest_image(value: object, label: str) -> str:
|
|
if not isinstance(value, str) or DIGEST_IMAGE.fullmatch(value) is None:
|
|
raise SmokeError(f"{label} must be an exact sha256 image reference")
|
|
return value
|
|
|
|
|
|
def platform_image(path: Path, platform: str, label: str) -> str:
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise SmokeError(f"cannot read {label} OCI metadata") from exc
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("platforms"), dict):
|
|
raise SmokeError(f"{label} OCI metadata has no platform map")
|
|
return _digest_image(payload["platforms"].get(platform), f"{label} {platform}")
|
|
|
|
|
|
def _tail(value: str, *, limit: int = 4000) -> str:
|
|
return value[-limit:].strip()
|
|
|
|
|
|
def _run(
|
|
arguments: Sequence[str],
|
|
*,
|
|
check: bool = True,
|
|
timeout: float = 600.0,
|
|
redactions: Sequence[str] = (),
|
|
) -> subprocess.CompletedProcess[str]:
|
|
try:
|
|
result = subprocess.run(
|
|
list(arguments),
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise SmokeError(f"container command could not complete: {type(exc).__name__}") from exc
|
|
if check and result.returncode != 0:
|
|
detail = _tail(result.stderr or result.stdout or "no diagnostic output")
|
|
for secret in redactions:
|
|
if secret:
|
|
detail = detail.replace(secret, "[redacted]")
|
|
raise SmokeError(f"container command failed: {detail}")
|
|
return result
|
|
|
|
|
|
def _wait_for(
|
|
label: str,
|
|
probe: Callable[[], subprocess.CompletedProcess[str]],
|
|
*,
|
|
timeout: float,
|
|
) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
result = probe()
|
|
if result.returncode == 0:
|
|
return
|
|
last = _tail(result.stderr or result.stdout)
|
|
time.sleep(2.0)
|
|
raise SmokeError(f"{label} did not become ready: {last or 'probe failed'}")
|
|
|
|
|
|
def _environment(
|
|
*,
|
|
database_password: str,
|
|
master_key: str,
|
|
platform_slug: str,
|
|
) -> dict[str, str]:
|
|
database = (
|
|
f"postgresql+psycopg://govoplan:{database_password}@postgres:5432/govoplan"
|
|
)
|
|
return {
|
|
"APP_ENV": "dev",
|
|
"GOVOPLAN_INSTALL_PROFILE": "evaluation",
|
|
"GOVOPLAN_INSTALLATION_ID": f"runtime-smoke-{platform_slug}",
|
|
"GOVOPLAN_STATE_PROFILE": "host-shared",
|
|
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "5",
|
|
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "30",
|
|
"GOVOPLAN_EXPECTED_API_REPLICAS": "1",
|
|
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": "1",
|
|
"DATABASE_URL": database,
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": (
|
|
f"postgresql://govoplan:{database_password}@postgres:5432/govoplan"
|
|
),
|
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
|
"GOVOPLAN_DB_CONNECTION_RESERVE": "10",
|
|
"REDIS_URL": "redis://redis:6379/0",
|
|
"CELERY_ENABLED": "true",
|
|
"CELERY_QUEUES": "default",
|
|
"CELERY_WORKER_CONCURRENCY": "1",
|
|
"ENABLED_MODULES": ",".join(BASE_MODULES),
|
|
"GOVOPLAN_MIGRATION_TRACK": "release",
|
|
"DEV_AUTO_MIGRATE_ENABLED": "false",
|
|
"DEV_BOOTSTRAP_ENABLED": "false",
|
|
"AUTH_LOGIN_THROTTLE_ENABLED": "true",
|
|
"AUTH_COOKIE_SECURE": "false",
|
|
"CORS_ORIGINS": "http://localhost",
|
|
"GOVOPLAN_TRUSTED_HOSTS": "127.0.0.1,localhost,api",
|
|
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
|
"MASTER_KEY_B64": master_key,
|
|
"FILE_STORAGE_BACKEND": "local",
|
|
"FILE_STORAGE_LOCAL_ROOT": "/var/lib/govoplan/files",
|
|
"GOVOPLAN_MODULE_LIVE_APPLY_ENABLED": "false",
|
|
}
|
|
|
|
|
|
def _env_arguments(values: dict[str, str], *, role: str, node_id: str) -> list[str]:
|
|
arguments: list[str] = []
|
|
for key, value in sorted(
|
|
{**values, "GOVOPLAN_RUNTIME_ROLE": role, "GOVOPLAN_NODE_ID": node_id}.items()
|
|
):
|
|
arguments.extend(("--env", f"{key}={value}"))
|
|
return arguments
|
|
|
|
|
|
def run_smoke(
|
|
*,
|
|
api_image: str,
|
|
web_image: str,
|
|
postgres_image: str,
|
|
redis_image: str,
|
|
platform: str,
|
|
timeout: float,
|
|
) -> dict[str, object]:
|
|
for label, value in (
|
|
("API image", api_image),
|
|
("Web image", web_image),
|
|
("PostgreSQL image", postgres_image),
|
|
("Redis image", redis_image),
|
|
):
|
|
_digest_image(value, label)
|
|
if platform not in PLATFORMS:
|
|
raise SmokeError(f"unsupported runtime smoke platform: {platform}")
|
|
|
|
slug = platform.replace("linux/", "").replace("/", "-")
|
|
suffix = secrets.token_hex(4)
|
|
prefix = f"govoplan-runtime-{slug}-{suffix}"
|
|
names = {
|
|
"network": f"{prefix}-network",
|
|
"volume": f"{prefix}-data",
|
|
"postgres": f"{prefix}-postgres",
|
|
"redis": f"{prefix}-redis",
|
|
"api": f"{prefix}-api",
|
|
"web": f"{prefix}-web",
|
|
"worker": f"{prefix}-worker",
|
|
}
|
|
database_password = secrets.token_hex(20)
|
|
master_key = base64.urlsafe_b64encode(os.urandom(32)).decode("ascii")
|
|
redactions = (database_password, master_key)
|
|
environment = _environment(
|
|
database_password=database_password,
|
|
master_key=master_key,
|
|
platform_slug=slug,
|
|
)
|
|
checks: list[dict[str, object]] = []
|
|
started = time.monotonic()
|
|
|
|
def record(check_id: str, began: float) -> None:
|
|
checks.append(
|
|
{
|
|
"id": check_id,
|
|
"state": "passed",
|
|
"duration_seconds": round(time.monotonic() - began, 3),
|
|
}
|
|
)
|
|
|
|
common_runtime = [
|
|
"--platform",
|
|
platform,
|
|
"--network",
|
|
names["network"],
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:rw,noexec,nosuid,size=64m",
|
|
"--security-opt",
|
|
"no-new-privileges:true",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--mount",
|
|
f"type=volume,source={names['volume']},target=/var/lib/govoplan",
|
|
]
|
|
|
|
try:
|
|
_run(("docker", "network", "create", names["network"]), timeout=timeout)
|
|
_run(("docker", "volume", "create", names["volume"]), timeout=timeout)
|
|
|
|
began = time.monotonic()
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--platform",
|
|
platform,
|
|
"--name",
|
|
names["postgres"],
|
|
"--network",
|
|
names["network"],
|
|
"--network-alias",
|
|
"postgres",
|
|
"--env",
|
|
"POSTGRES_DB=govoplan",
|
|
"--env",
|
|
"POSTGRES_USER=govoplan",
|
|
"--env",
|
|
f"POSTGRES_PASSWORD={database_password}",
|
|
"--tmpfs",
|
|
"/var/lib/postgresql/data:rw,noexec,nosuid,size=384m",
|
|
postgres_image,
|
|
),
|
|
timeout=timeout,
|
|
redactions=redactions,
|
|
)
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--platform",
|
|
platform,
|
|
"--name",
|
|
names["redis"],
|
|
"--network",
|
|
names["network"],
|
|
"--network-alias",
|
|
"redis",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/data:rw,noexec,nosuid,size=64m",
|
|
redis_image,
|
|
"redis-server",
|
|
"--save",
|
|
"",
|
|
"--appendonly",
|
|
"no",
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_wait_for(
|
|
"PostgreSQL",
|
|
lambda: _run(
|
|
(
|
|
"docker",
|
|
"exec",
|
|
names["postgres"],
|
|
"pg_isready",
|
|
"--username",
|
|
"govoplan",
|
|
"--dbname",
|
|
"govoplan",
|
|
),
|
|
check=False,
|
|
timeout=30,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_wait_for(
|
|
"Redis",
|
|
lambda: _run(
|
|
("docker", "exec", names["redis"], "redis-cli", "ping"),
|
|
check=False,
|
|
timeout=30,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
record("managed_dependencies_ready", began)
|
|
|
|
began = time.monotonic()
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--name",
|
|
f"{prefix}-migrate",
|
|
*common_runtime,
|
|
*_env_arguments(
|
|
environment,
|
|
role="migration",
|
|
node_id=f"runtime-smoke-{slug}-migration",
|
|
),
|
|
api_image,
|
|
"python",
|
|
"-m",
|
|
"govoplan_core.commands.init_db",
|
|
"--migration-track",
|
|
"release",
|
|
),
|
|
timeout=timeout,
|
|
redactions=redactions,
|
|
)
|
|
record("release_migrations", began)
|
|
|
|
began = time.monotonic()
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--name",
|
|
names["api"],
|
|
"--network-alias",
|
|
"api",
|
|
*common_runtime,
|
|
*_env_arguments(
|
|
environment,
|
|
role="api",
|
|
node_id=f"runtime-smoke-{slug}-api",
|
|
),
|
|
api_image,
|
|
),
|
|
timeout=timeout,
|
|
redactions=redactions,
|
|
)
|
|
_wait_for(
|
|
"GovOPlaN API",
|
|
lambda: _run(
|
|
(
|
|
"docker",
|
|
"exec",
|
|
names["api"],
|
|
"python",
|
|
"-c",
|
|
(
|
|
"import urllib.request;"
|
|
"r=urllib.request.Request('http://127.0.0.1:8000/health/ready',"
|
|
"headers={'Host':'127.0.0.1'});"
|
|
"assert urllib.request.urlopen(r,timeout=3).status==200"
|
|
),
|
|
),
|
|
check=False,
|
|
timeout=30,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_run(
|
|
(
|
|
"docker",
|
|
"exec",
|
|
names["api"],
|
|
"python",
|
|
"-c",
|
|
"import os; assert os.getuid() == 10001",
|
|
),
|
|
timeout=30,
|
|
)
|
|
record("api_non_root_readiness", began)
|
|
|
|
began = time.monotonic()
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--platform",
|
|
platform,
|
|
"--name",
|
|
names["web"],
|
|
"--network",
|
|
names["network"],
|
|
"--network-alias",
|
|
"web",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:rw,noexec,nosuid,size=64m",
|
|
"--security-opt",
|
|
"no-new-privileges:true",
|
|
"--cap-drop",
|
|
"ALL",
|
|
web_image,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_wait_for(
|
|
"GovOPlaN WebUI",
|
|
lambda: _run(
|
|
(
|
|
"docker",
|
|
"exec",
|
|
names["api"],
|
|
"python",
|
|
"-c",
|
|
(
|
|
"import urllib.request;"
|
|
"assert urllib.request.urlopen('http://web:8080/health',timeout=3).status==200;"
|
|
"assert urllib.request.urlopen('http://web:8080/',timeout=3).status==200"
|
|
),
|
|
),
|
|
check=False,
|
|
timeout=30,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_run(
|
|
("docker", "exec", names["web"], "sh", "-c", "test \"$(id -u)\" = 101"),
|
|
timeout=30,
|
|
)
|
|
record("web_non_root_readiness", began)
|
|
|
|
began = time.monotonic()
|
|
_run(
|
|
(
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--name",
|
|
names["worker"],
|
|
*common_runtime,
|
|
*_env_arguments(
|
|
environment,
|
|
role="worker",
|
|
node_id=f"runtime-smoke-{slug}-worker",
|
|
),
|
|
api_image,
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"worker",
|
|
"--queues",
|
|
"default",
|
|
"--pool",
|
|
"solo",
|
|
"--concurrency",
|
|
"1",
|
|
"--hostname",
|
|
f"runtime-smoke-{slug}@%h",
|
|
"--loglevel",
|
|
"WARNING",
|
|
),
|
|
timeout=timeout,
|
|
redactions=redactions,
|
|
)
|
|
_wait_for(
|
|
"GovOPlaN worker",
|
|
lambda: _run(
|
|
(
|
|
"docker",
|
|
"exec",
|
|
names["api"],
|
|
"python",
|
|
"-c",
|
|
(
|
|
"from govoplan_core.celery_app import celery;"
|
|
"result=celery.send_task('govoplan.ping',queue='default');"
|
|
"assert result.get(timeout=10)=='pong'"
|
|
),
|
|
),
|
|
check=False,
|
|
timeout=30,
|
|
),
|
|
timeout=timeout,
|
|
)
|
|
_run(("docker", "stop", "--time", "20", names["worker"]), timeout=30)
|
|
record("worker_delivery_and_shutdown", began)
|
|
except SmokeError as exc:
|
|
for role in ("api", "web", "worker", "postgres", "redis"):
|
|
result = _run(
|
|
("docker", "logs", "--tail", "100", names[role]),
|
|
check=False,
|
|
timeout=30,
|
|
)
|
|
if result.stdout or result.stderr:
|
|
detail = _tail(result.stdout + result.stderr, limit=8000)
|
|
for secret in redactions:
|
|
detail = detail.replace(secret, "[redacted]")
|
|
print(f"--- {role} logs ---\n{detail}")
|
|
raise exc
|
|
finally:
|
|
for role in ("worker", "web", "api", "redis", "postgres"):
|
|
_run(
|
|
("docker", "rm", "--force", names[role]),
|
|
check=False,
|
|
timeout=30,
|
|
)
|
|
_run(("docker", "volume", "rm", "--force", names["volume"]), check=False)
|
|
_run(("docker", "network", "rm", names["network"]), check=False)
|
|
|
|
return {
|
|
"schema_version": "1",
|
|
"evidence_kind": "govoplan.runtime-image-smoke",
|
|
"captured_at": _utc_now(),
|
|
"platform": platform,
|
|
"images": {
|
|
"api": api_image,
|
|
"web": web_image,
|
|
"postgres": postgres_image,
|
|
"redis": redis_image,
|
|
},
|
|
"result": {"state": "passed"},
|
|
"checks": checks,
|
|
"duration_seconds": round(time.monotonic() - started, 3),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
args = build_parser().parse_args()
|
|
try:
|
|
api_image = platform_image(args.api_metadata, args.platform, "API")
|
|
web_image = platform_image(args.web_metadata, args.platform, "Web")
|
|
evidence = run_smoke(
|
|
api_image=api_image,
|
|
web_image=web_image,
|
|
postgres_image=args.postgres_image,
|
|
redis_image=args.redis_image,
|
|
platform=args.platform,
|
|
timeout=args.timeout_seconds,
|
|
)
|
|
except (OSError, SmokeError, ValueError) as exc:
|
|
print(f"runtime image smoke failed: {exc}")
|
|
return 1
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = args.output.with_suffix(args.output.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
temporary.chmod(0o644)
|
|
temporary.replace(args.output)
|
|
print(f"Runtime image smoke evidence written to {args.output}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|