580 lines
19 KiB
Python
580 lines
19 KiB
Python
"""Deployment plan and host preflight checks."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import asdict, dataclass
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
from typing import Callable, Mapping, Sequence
|
|
from urllib.parse import urlsplit
|
|
|
|
from .bundle import (
|
|
BundlePaths,
|
|
digest_json,
|
|
environment_fingerprint,
|
|
read_env,
|
|
render_compose,
|
|
service_names,
|
|
)
|
|
from .model import (
|
|
InstallationSpec,
|
|
image_is_digest_pinned,
|
|
image_is_unpublished,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Check:
|
|
id: str
|
|
level: str
|
|
message: str
|
|
action: str = ""
|
|
|
|
def to_dict(self) -> dict[str, str]:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PlanAction:
|
|
action: str
|
|
target: str
|
|
detail: str
|
|
|
|
def to_dict(self) -> dict[str, str]:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class DeploymentPlan:
|
|
installation_id: str
|
|
desired_spec_sha256: str
|
|
desired_compose_sha256: str
|
|
desired_environment_fingerprint: str
|
|
actions: tuple[PlanAction, ...]
|
|
checks: tuple[Check, ...]
|
|
|
|
@property
|
|
def blocked(self) -> bool:
|
|
return any(check.level == "error" for check in self.checks)
|
|
|
|
def to_dict(self) -> dict[str, object]:
|
|
return {
|
|
"installation_id": self.installation_id,
|
|
"desired_spec_sha256": self.desired_spec_sha256,
|
|
"desired_compose_sha256": self.desired_compose_sha256,
|
|
"desired_environment_fingerprint": (
|
|
self.desired_environment_fingerprint
|
|
),
|
|
"blocked": self.blocked,
|
|
"actions": [action.to_dict() for action in self.actions],
|
|
"checks": [check.to_dict() for check in self.checks],
|
|
}
|
|
|
|
|
|
CommandRunner = Callable[[Sequence[str], Path], subprocess.CompletedProcess[str]]
|
|
|
|
|
|
def build_plan(
|
|
spec: InstallationSpec,
|
|
paths: BundlePaths,
|
|
*,
|
|
include_host_checks: bool = True,
|
|
command_runner: CommandRunner | None = None,
|
|
) -> DeploymentPlan:
|
|
compose = render_compose(spec)
|
|
desired_services = set(service_names(spec))
|
|
previous = _read_receipt(paths.receipt)
|
|
previous_services = {
|
|
str(item) for item in previous.get("services", []) if isinstance(item, str)
|
|
}
|
|
previous_spec_digest = previous.get("spec_sha256")
|
|
previous_compose_digest = previous.get("compose_sha256")
|
|
previous_environment_fingerprint = previous.get("environment_fingerprint")
|
|
spec_digest = digest_json(spec.to_dict())
|
|
compose_digest = digest_json(compose)
|
|
environment_digest = environment_fingerprint(read_env(paths.env))
|
|
|
|
actions: list[PlanAction] = []
|
|
if not previous:
|
|
actions.append(
|
|
PlanAction(
|
|
"create",
|
|
"installation",
|
|
"Create the first deployment revision.",
|
|
)
|
|
)
|
|
if previous_spec_digest != spec_digest:
|
|
actions.append(
|
|
PlanAction(
|
|
"reconfigure",
|
|
"installation",
|
|
"Reconcile runtime configuration with installation.json.",
|
|
)
|
|
)
|
|
if previous_compose_digest != compose_digest:
|
|
actions.append(
|
|
PlanAction(
|
|
"render",
|
|
"compose",
|
|
"Render a new deterministic Compose definition.",
|
|
)
|
|
)
|
|
if previous_environment_fingerprint != environment_digest:
|
|
actions.append(
|
|
PlanAction(
|
|
"reconfigure",
|
|
"environment",
|
|
"Reconcile changed secret bindings or runtime environment values.",
|
|
)
|
|
)
|
|
for name in sorted(desired_services - previous_services):
|
|
actions.append(PlanAction("start", name, "Start the selected service."))
|
|
for name in sorted(previous_services - desired_services):
|
|
actions.append(
|
|
PlanAction(
|
|
"remove",
|
|
name,
|
|
"Remove the service container; retained volumes are not deleted.",
|
|
)
|
|
)
|
|
if (
|
|
previous
|
|
and previous_spec_digest == spec_digest
|
|
and previous_compose_digest == compose_digest
|
|
and previous_environment_fingerprint == environment_digest
|
|
and previous_services == desired_services
|
|
):
|
|
actions.append(
|
|
PlanAction("noop", "installation", "Desired state matches the last receipt.")
|
|
)
|
|
|
|
checks = list(static_checks(spec, paths))
|
|
if include_host_checks:
|
|
checks.extend(host_checks(spec, paths, command_runner=command_runner))
|
|
return DeploymentPlan(
|
|
installation_id=spec.installation_id,
|
|
desired_spec_sha256=spec_digest,
|
|
desired_compose_sha256=compose_digest,
|
|
desired_environment_fingerprint=environment_digest,
|
|
actions=tuple(actions),
|
|
checks=tuple(checks),
|
|
)
|
|
|
|
|
|
def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ...]:
|
|
checks: list[Check] = []
|
|
images = {
|
|
"release.api_image": spec.release.api_image,
|
|
"release.web_image": spec.release.web_image,
|
|
}
|
|
if spec.components.postgres.mode == "managed":
|
|
images["components.postgres.image"] = spec.components.postgres.image
|
|
if spec.components.redis.mode == "managed":
|
|
images["components.redis.image"] = spec.components.redis.image
|
|
if spec.components.mail.mode == "test-mail":
|
|
images["components.mail.image"] = spec.components.mail.image
|
|
|
|
for label, image in images.items():
|
|
if image_is_unpublished(image):
|
|
checks.append(
|
|
Check(
|
|
f"{label}.published",
|
|
"error",
|
|
f"{label} still uses the unpublished placeholder.",
|
|
"Set an available image reference before apply.",
|
|
)
|
|
)
|
|
elif not image_is_digest_pinned(image):
|
|
level = "error" if spec.profile == "self-hosted" else "warning"
|
|
checks.append(
|
|
Check(
|
|
f"{label}.digest",
|
|
level,
|
|
f"{label} is not pinned by OCI digest.",
|
|
"Use an image@sha256:... reference from a verified distribution.",
|
|
)
|
|
)
|
|
else:
|
|
checks.append(
|
|
Check(
|
|
f"{label}.digest",
|
|
"ok",
|
|
f"{label} is pinned by OCI digest.",
|
|
)
|
|
)
|
|
|
|
if spec.release.manifest_url and spec.release.manifest_sha256:
|
|
checks.append(
|
|
Check(
|
|
"release.manifest",
|
|
"ok",
|
|
"A distribution manifest URL and expected digest are recorded.",
|
|
)
|
|
)
|
|
else:
|
|
checks.append(
|
|
Check(
|
|
"release.manifest",
|
|
"error" if spec.profile == "self-hosted" else "warning",
|
|
"No verified distribution manifest is recorded.",
|
|
"Use a published signed distribution manifest for self-hosted apply.",
|
|
)
|
|
)
|
|
if spec.profile == "self-hosted":
|
|
checks.append(
|
|
Check(
|
|
"release.signature_verification",
|
|
"error",
|
|
"Signed distribution-manifest verification is not implemented in the deployer yet.",
|
|
"Use the published verifier/bootstrap slice before a production apply.",
|
|
)
|
|
)
|
|
checks.append(
|
|
Check(
|
|
"modules.image_composition",
|
|
"error" if spec.enabled_modules else "warning",
|
|
"The selected module set is not yet verified against image package contents.",
|
|
"Use the signed distribution composition evidence before production apply.",
|
|
)
|
|
)
|
|
|
|
values = read_env(paths.env)
|
|
required = {"MASTER_KEY_B64", "DATABASE_URL"}
|
|
if spec.components.redis.mode != "disabled":
|
|
required.add("REDIS_URL")
|
|
if spec.components.storage.mode == "s3":
|
|
required.update(
|
|
{
|
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
|
"FILE_STORAGE_S3_REGION",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
|
"FILE_STORAGE_S3_BUCKET",
|
|
}
|
|
)
|
|
missing = sorted(name for name in required if not values.get(name))
|
|
checks.append(
|
|
Check(
|
|
"secrets.required",
|
|
"error" if missing else "ok",
|
|
(
|
|
"Missing required secret values: " + ", ".join(missing)
|
|
if missing
|
|
else "Required runtime secret references are populated."
|
|
),
|
|
"Re-run configure with the required external service values." if missing else "",
|
|
)
|
|
)
|
|
if paths.env.exists():
|
|
mode = stat.S_IMODE(paths.env.stat().st_mode)
|
|
checks.append(
|
|
Check(
|
|
"secrets.permissions",
|
|
"error" if mode & 0o077 else "ok",
|
|
(
|
|
f"{paths.env.name} has unsafe mode {mode:04o}."
|
|
if mode & 0o077
|
|
else f"{paths.env.name} is private ({mode:04o})."
|
|
),
|
|
f"Run chmod 600 {paths.env}" if mode & 0o077 else "",
|
|
)
|
|
)
|
|
if spec.profile == "self-hosted":
|
|
checks.append(
|
|
Check(
|
|
"bootstrap.administrator",
|
|
"warning",
|
|
"Production first-administrator enrollment is not automated yet.",
|
|
"Use the controlled one-time administrator procedure until the enrollment slice lands.",
|
|
)
|
|
)
|
|
if spec.components.mail.mode == "external-relay":
|
|
checks.append(
|
|
Check(
|
|
"mail.provisioning",
|
|
"warning",
|
|
"The external relay is selected but Mail profile seeding remains an explicit post-install configuration task.",
|
|
"Create the reusable server and credential profile in Mail after first login.",
|
|
)
|
|
)
|
|
if spec.components.storage.mode == "local" and spec.profile == "self-hosted":
|
|
checks.append(
|
|
Check(
|
|
"storage.local",
|
|
"warning",
|
|
"Local file storage is suitable for one host but prevents stateless API scale-out.",
|
|
"Include files-data in backup/restore drills or configure S3 storage.",
|
|
)
|
|
)
|
|
return tuple(checks)
|
|
|
|
|
|
def host_checks(
|
|
spec: InstallationSpec,
|
|
paths: BundlePaths,
|
|
*,
|
|
command_runner: CommandRunner | None = None,
|
|
) -> tuple[Check, ...]:
|
|
checks: list[Check] = []
|
|
machine = platform.machine().lower()
|
|
supported = machine in {"x86_64", "amd64", "aarch64", "arm64"}
|
|
checks.append(
|
|
Check(
|
|
"host.architecture",
|
|
"ok" if supported else "error",
|
|
f"Host architecture is {machine or 'unknown'}.",
|
|
"Use a supported amd64 or arm64 host." if not supported else "",
|
|
)
|
|
)
|
|
|
|
memory_bytes = _memory_bytes()
|
|
if memory_bytes is not None:
|
|
minimum = 4 * 1024**3
|
|
checks.append(
|
|
Check(
|
|
"host.memory",
|
|
"ok" if memory_bytes >= minimum else "warning",
|
|
f"Host memory is approximately {memory_bytes / 1024**3:.1f} GiB.",
|
|
"Use at least 4 GiB for the base container profile."
|
|
if memory_bytes < minimum
|
|
else "",
|
|
)
|
|
)
|
|
cpu_count = os.cpu_count()
|
|
if cpu_count is not None:
|
|
checks.append(
|
|
Check(
|
|
"host.cpu",
|
|
"ok" if cpu_count >= 2 else "warning",
|
|
f"Host reports {cpu_count} logical CPU(s).",
|
|
"Use at least two logical CPUs for the base container profile."
|
|
if cpu_count < 2
|
|
else "",
|
|
)
|
|
)
|
|
entropy = _entropy_available()
|
|
if entropy is not None:
|
|
checks.append(
|
|
Check(
|
|
"host.entropy",
|
|
"ok" if entropy >= 256 else "warning",
|
|
f"Kernel entropy availability is {entropy}.",
|
|
"Wait for or provide sufficient host entropy before generating production keys."
|
|
if entropy < 256
|
|
else "",
|
|
)
|
|
)
|
|
free_bytes = shutil.disk_usage(paths.root).free
|
|
minimum_free = 10 * 1024**3
|
|
checks.append(
|
|
Check(
|
|
"host.disk",
|
|
"ok" if free_bytes >= minimum_free else "warning",
|
|
f"Free installation filesystem space is approximately {free_bytes / 1024**3:.1f} GiB.",
|
|
"Keep at least 10 GiB free before pulling images and creating data volumes."
|
|
if free_bytes < minimum_free
|
|
else "",
|
|
)
|
|
)
|
|
|
|
docker = shutil.which("docker")
|
|
if docker is None:
|
|
checks.append(
|
|
Check(
|
|
"host.compose",
|
|
"error",
|
|
"Docker CLI is not installed or not on PATH.",
|
|
"Install Docker Engine with the Compose v2 plugin.",
|
|
)
|
|
)
|
|
else:
|
|
runner = command_runner or _run_command
|
|
result = runner((docker, "compose", "version", "--short"), paths.root)
|
|
checks.append(
|
|
Check(
|
|
"host.compose",
|
|
"ok" if result.returncode == 0 else "error",
|
|
(
|
|
f"Docker Compose is available ({result.stdout.strip()})."
|
|
if result.returncode == 0
|
|
else "Docker Compose v2 is not available."
|
|
),
|
|
"Install or enable the Docker Compose v2 plugin."
|
|
if result.returncode != 0
|
|
else "",
|
|
)
|
|
)
|
|
daemon = runner(
|
|
(docker, "info", "--format", "{{.ServerVersion}}"),
|
|
paths.root,
|
|
)
|
|
checks.append(
|
|
Check(
|
|
"host.container_runtime",
|
|
"ok" if daemon.returncode == 0 else "error",
|
|
(
|
|
f"Docker daemon is reachable ({daemon.stdout.strip()})."
|
|
if daemon.returncode == 0
|
|
else "Docker CLI cannot reach the Docker daemon."
|
|
),
|
|
"Start Docker and grant this operator access to the daemon."
|
|
if daemon.returncode != 0
|
|
else "",
|
|
)
|
|
)
|
|
|
|
values = read_env(paths.env)
|
|
if spec.components.postgres.mode == "external":
|
|
checks.append(
|
|
_endpoint_check(
|
|
"external.postgres",
|
|
"External PostgreSQL",
|
|
values.get("DATABASE_URL", ""),
|
|
default_ports={"postgresql": 5432, "postgresql+psycopg": 5432},
|
|
)
|
|
)
|
|
if spec.components.redis.mode == "external":
|
|
checks.append(
|
|
_endpoint_check(
|
|
"external.redis",
|
|
"External Redis",
|
|
values.get("REDIS_URL", ""),
|
|
default_ports={"redis": 6379, "rediss": 6379},
|
|
)
|
|
)
|
|
if spec.components.storage.mode == "s3":
|
|
checks.append(
|
|
_endpoint_check(
|
|
"external.s3",
|
|
"S3-compatible storage",
|
|
values.get("FILE_STORAGE_S3_ENDPOINT_URL", ""),
|
|
default_ports={"http": 80, "https": 443},
|
|
)
|
|
)
|
|
|
|
receipt = _read_receipt(paths.receipt)
|
|
previous_listen = receipt.get("listen")
|
|
desired_listen = {
|
|
"address": spec.listen.address,
|
|
"port": spec.listen.port,
|
|
}
|
|
if not receipt or previous_listen != desired_listen:
|
|
available = _port_available(spec.listen.address, spec.listen.port)
|
|
checks.append(
|
|
Check(
|
|
"host.listen_port",
|
|
"ok" if available else "error",
|
|
(
|
|
f"Listen endpoint {spec.listen.address}:{spec.listen.port} is available."
|
|
if available
|
|
else f"Listen endpoint {spec.listen.address}:{spec.listen.port} is already in use."
|
|
),
|
|
"Choose another listen port or stop the conflicting service."
|
|
if not available
|
|
else "",
|
|
)
|
|
)
|
|
return tuple(checks)
|
|
|
|
|
|
def _read_receipt(path: Path) -> Mapping[str, object]:
|
|
if not path.exists():
|
|
return {}
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return {}
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _memory_bytes() -> int | None:
|
|
try:
|
|
for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("MemTotal:"):
|
|
return int(line.split()[1]) * 1024
|
|
except (OSError, ValueError, IndexError):
|
|
return None
|
|
return None
|
|
|
|
|
|
def _entropy_available() -> int | None:
|
|
try:
|
|
return int(
|
|
Path("/proc/sys/kernel/random/entropy_avail")
|
|
.read_text(encoding="utf-8")
|
|
.strip()
|
|
)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _port_available(address: str, port: int) -> bool:
|
|
family = socket.AF_INET6 if ":" in address else socket.AF_INET
|
|
with socket.socket(family, socket.SOCK_STREAM) as handle:
|
|
try:
|
|
handle.bind((address, port))
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _endpoint_check(
|
|
check_id: str,
|
|
label: str,
|
|
url: str,
|
|
*,
|
|
default_ports: Mapping[str, int],
|
|
) -> Check:
|
|
try:
|
|
parsed = urlsplit(url)
|
|
host = parsed.hostname
|
|
port = parsed.port or default_ports.get(parsed.scheme)
|
|
except ValueError as exc:
|
|
return Check(
|
|
check_id,
|
|
"error",
|
|
f"{label} endpoint is invalid: {exc}",
|
|
"Correct the private endpoint binding and rerun doctor.",
|
|
)
|
|
if not host or port is None:
|
|
return Check(
|
|
check_id,
|
|
"error",
|
|
f"{label} endpoint has no usable host and port.",
|
|
"Correct the private endpoint binding and rerun doctor.",
|
|
)
|
|
try:
|
|
with socket.create_connection((host, port), timeout=1.5):
|
|
pass
|
|
except OSError as exc:
|
|
return Check(
|
|
check_id,
|
|
"error",
|
|
f"{label} is not reachable at {host}:{port}: {exc}",
|
|
"Check DNS, routing, firewall, service health, and the selected endpoint.",
|
|
)
|
|
return Check(
|
|
check_id,
|
|
"ok",
|
|
f"{label} is reachable at {host}:{port}.",
|
|
)
|
|
|
|
|
|
def _run_command(
|
|
argv: Sequence[str], cwd: Path
|
|
) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
list(argv),
|
|
cwd=cwd,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
)
|