"""Durable deployment operation journal and bounded bundle recovery.""" from __future__ import annotations from dataclasses import dataclass from datetime import UTC, datetime from hashlib import sha256 import json from pathlib import Path import re import shutil from typing import Any from uuid import uuid4 from .bundle import BundlePaths, atomic_write, canonical_json OPERATIONS_DIRECTORY = "operations" APPLIED_STATE_DIRECTORY = "applied-state" OPERATION_FILENAME = "operation.json" _OPERATION_ID = re.compile(r"^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$") _BUNDLE_FILES = ( "installation.json", "secrets.env", "compose.json", "garage.toml", "load-balancer.cfg", "receipt.json", ) @dataclass(frozen=True, slots=True) class RecoveryResult: operation_id: str action: str detail: str apply_allowed: bool class DeploymentOperationJournal: def __init__(self, operation_path: Path, payload: dict[str, Any]) -> None: self.path = operation_path self.payload = payload @classmethod def begin( cls, paths: BundlePaths, *, plan: dict[str, Any], ) -> DeploymentOperationJournal: operations_root = paths.root / OPERATIONS_DIRECTORY _private_directory(operations_root) operation_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid4().hex[:8] operation_path = operations_root / operation_id _private_directory(operation_path) snapshot_source = paths.root / APPLIED_STATE_DIRECTORY snapshot_available = snapshot_source.is_dir() if snapshot_available: shutil.copytree( snapshot_source, operation_path / "before", copy_function=_private_copy, ) payload: dict[str, Any] = { "schema_version": 2, "operation_id": operation_id, "installation_id": plan.get("installation_id"), "status": "running", "recovery_mode": ( "configuration_rollback" if snapshot_available else "forward_recovery" ), "previous_applied_snapshot": snapshot_available, "migration_started": False, "migration_completed": False, "started_at": _now(), "updated_at": _now(), "completed_at": None, "failure_summary": None, "desired": plan, "stages": [], "evidence_head_sha256": None, } journal = cls(operation_path, payload) journal.record( "plan-recorded", "succeeded", { "previous_applied_snapshot": snapshot_available, "desired_sha256": sha256(canonical_json(plan)).hexdigest(), }, ) return journal @property def operation_id(self) -> str: return str(self.payload["operation_id"]) def record( self, stage: str, status: str, evidence: dict[str, Any] | None = None, ) -> None: stages = list(self.payload.get("stages") or []) item = { "sequence": len(stages) + 1, "stage": stage, "status": status, "at": _now(), "evidence": dict(evidence or {}), "previous_sha256": self.payload.get("evidence_head_sha256"), } item["checkpoint_sha256"] = sha256(canonical_json(item)).hexdigest() stages.append(item) self.payload["stages"] = stages self.payload["evidence_head_sha256"] = item["checkpoint_sha256"] self.payload["updated_at"] = item["at"] self._write() def migration_started(self) -> None: self.payload["migration_started"] = True self.payload["recovery_mode"] = "forward_recovery" self.record( "migration-started", "running", { "rollback_boundary": ( "Database migration has started; automatic release/configuration rollback is disabled." ) }, ) def migration_completed(self) -> None: self.payload["migration_completed"] = True self.record("migration-completed", "succeeded") def succeeded(self, paths: BundlePaths, *, receipt: dict[str, Any]) -> None: write_applied_state(paths) self.payload["status"] = "succeeded" self.payload["completed_at"] = _now() self.record( "deployment-verified", "succeeded", { "receipt_spec_sha256": receipt.get("spec_sha256"), "receipt_compose_sha256": receipt.get("compose_sha256"), }, ) def failed(self, exc: BaseException) -> None: self.payload["status"] = "failed" self.payload["completed_at"] = _now() self.payload["failure_summary"] = f"{type(exc).__name__}: {str(exc)[:1000]}" self.record( "deployment-failed", "failed", { "exception_type": type(exc).__name__, "recovery_mode": self.payload.get("recovery_mode"), "failure_summary_sha256": sha256( str(self.payload["failure_summary"]).encode("utf-8") ).hexdigest(), }, ) def _write(self) -> None: atomic_write( self.path / OPERATION_FILENAME, canonical_json(self.payload), mode=0o600, ) def write_applied_state(paths: BundlePaths) -> Path: target = paths.root / APPLIED_STATE_DIRECTORY temporary = paths.root / f".{APPLIED_STATE_DIRECTORY}.{uuid4().hex}.tmp" previous = paths.root / f".{APPLIED_STATE_DIRECTORY}.previous" _private_directory(temporary) manifest: dict[str, Any] = {"schema_version": 1, "captured_at": _now(), "files": {}} try: for filename in _BUNDLE_FILES: source = paths.root / filename if not source.is_file(): manifest["files"][filename] = {"present": False} continue destination = temporary / filename _private_copy(source, destination) manifest["files"][filename] = { "present": True, "sha256": sha256(destination.read_bytes()).hexdigest(), } atomic_write( temporary / "manifest.json", canonical_json(manifest), mode=0o600, ) if previous.exists() and not target.exists(): previous.replace(target) elif previous.exists(): shutil.rmtree(previous) if target.exists(): target.replace(previous) try: temporary.replace(target) except BaseException: if previous.exists() and not target.exists(): previous.replace(target) raise if previous.exists(): shutil.rmtree(previous) finally: if temporary.exists(): shutil.rmtree(temporary) return target def list_operations(paths: BundlePaths) -> list[dict[str, Any]]: root = paths.root / OPERATIONS_DIRECTORY if not root.is_dir(): return [] operations: list[dict[str, Any]] = [] for path in sorted(root.iterdir(), reverse=True): if not path.is_dir() or not _OPERATION_ID.fullmatch(path.name): continue payload = _read_operation(path) operations.append( { "operation_id": payload.get("operation_id"), "status": payload.get("status"), "recovery_mode": payload.get("recovery_mode"), "migration_started": bool(payload.get("migration_started")), "started_at": payload.get("started_at"), "completed_at": payload.get("completed_at"), "failure_summary": payload.get("failure_summary"), "stage_count": len(payload.get("stages") or []), } ) return operations def recover_operation( paths: BundlePaths, *, operation_id: str | None = None, ) -> RecoveryResult: operation_path, payload = load_operation(paths, operation_id=operation_id) effective_id = str(payload["operation_id"]) if bool(payload.get("migration_started")): journal = DeploymentOperationJournal(operation_path, payload) journal.payload["status"] = "forward_recovery_required" journal.payload["recovery_mode"] = "forward_recovery" journal.record( "forward-recovery-required", "blocked", { "reason": "migration_started", "configuration_restored": False, }, ) return RecoveryResult( operation_id=effective_id, action="forward_recovery", detail=( "Database migration started. The previous release/configuration was not restored; " "repair the current release or restore a separately verified database backup." ), apply_allowed=True, ) before = operation_path / "before" if not before.is_dir(): return RecoveryResult( operation_id=effective_id, action="manual_intervention", detail="No independently captured previous applied state is available.", apply_allowed=False, ) manifest = json.loads((before / "manifest.json").read_text(encoding="utf-8")) entries = _validate_snapshot(before, manifest) for filename, evidence in entries: target = paths.root / filename if evidence.get("present"): source = before / filename _private_copy(source, target) else: target.unlink(missing_ok=True) journal = DeploymentOperationJournal(operation_path, payload) journal.payload["status"] = "configuration_restored" journal.payload["recovery_mode"] = "configuration_rollback" journal.record( "configuration-restored", "succeeded", {"snapshot_manifest_verified": True}, ) return RecoveryResult( operation_id=effective_id, action="configuration_restored", detail=( "The previously applied bundle was restored and checksum-verified. " "Run apply to reconcile containers to that state." ), apply_allowed=True, ) def load_operation( paths: BundlePaths, *, operation_id: str | None = None, ) -> tuple[Path, dict[str, Any]]: root = paths.root / OPERATIONS_DIRECTORY if operation_id is None: candidates = ( [ item for item in sorted(root.iterdir(), reverse=True) if item.is_dir() and _OPERATION_ID.fullmatch(item.name) ] if root.is_dir() else [] ) if not candidates: raise ValueError("No deployment operations are recorded") path = candidates[0] else: if not _OPERATION_ID.fullmatch(operation_id): raise ValueError("Invalid deployment operation id") path = root / operation_id return path, _read_operation(path) def _read_operation(path: Path) -> dict[str, Any]: try: payload = json.loads((path / OPERATION_FILENAME).read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError) as exc: raise ValueError(f"Deployment operation is unreadable: {path.name}") from exc if not isinstance(payload, dict) or payload.get("operation_id") != path.name: raise ValueError(f"Deployment operation identity is invalid: {path.name}") verify_operation_chain(payload) return payload def verify_operation_chain(payload: dict[str, Any]) -> None: stages = payload.get("stages") if not isinstance(stages, list) or not stages: raise ValueError("Deployment operation has no recovery evidence chain") previous: str | None = None observed_migration_started = False observed_migration_completed = False for sequence, raw_stage in enumerate(stages, start=1): if not isinstance(raw_stage, dict): raise ValueError("Deployment recovery evidence contains an invalid stage") stage = dict(raw_stage) checkpoint = str(stage.pop("checkpoint_sha256", "")) if ( stage.get("sequence") != sequence or stage.get("previous_sha256") != previous or sha256(canonical_json(stage)).hexdigest() != checkpoint ): raise ValueError( f"Deployment recovery evidence chain failed at stage {sequence}" ) previous = checkpoint observed_migration_started |= stage.get("stage") == "migration-started" observed_migration_completed |= stage.get("stage") == "migration-completed" if previous != payload.get("evidence_head_sha256"): raise ValueError("Deployment recovery evidence head does not match its chain") if bool(payload.get("migration_started")) != observed_migration_started: raise ValueError( "Deployment migration boundary does not match recovery evidence" ) if bool(payload.get("migration_completed")) != observed_migration_completed: raise ValueError( "Deployment migration completion does not match recovery evidence" ) if int(payload.get("schema_version") or 0) >= 2: _verify_operation_state(payload, stages) def _verify_operation_state( payload: dict[str, Any], stages: list[dict[str, Any]], ) -> None: plan_evidence = stages[0].get("evidence") if ( stages[0].get("stage") != "plan-recorded" or not isinstance(plan_evidence, dict) or plan_evidence.get("desired_sha256") != sha256(canonical_json(payload.get("desired"))).hexdigest() or bool(plan_evidence.get("previous_applied_snapshot")) != bool(payload.get("previous_applied_snapshot")) ): raise ValueError("Deployment plan does not match recovery evidence") desired = payload.get("desired") if not isinstance(desired, dict) or desired.get("installation_id") != payload.get( "installation_id" ): raise ValueError("Deployment installation identity does not match its plan") status = str(payload.get("status") or "") expected_last_stage = { "failed": "deployment-failed", "succeeded": "deployment-verified", "configuration_restored": "configuration-restored", "forward_recovery_required": "forward-recovery-required", }.get(status) if expected_last_stage and stages[-1].get("stage") != expected_last_stage: raise ValueError("Deployment status does not match recovery evidence") if status == "failed": failure_summary = str(payload.get("failure_summary") or "") evidence = stages[-1].get("evidence") if ( not failure_summary or not isinstance(evidence, dict) or evidence.get("failure_summary_sha256") != sha256(failure_summary.encode("utf-8")).hexdigest() ): raise ValueError( "Deployment failure summary does not match recovery evidence" ) if status in {"failed", "succeeded"} and not payload.get("completed_at"): raise ValueError("Completed deployment operation has no completion timestamp") migration_started = bool(payload.get("migration_started")) if bool(payload.get("migration_completed")) and not migration_started: raise ValueError("Completed migration has no recorded migration boundary") expected_mode = ( "forward_recovery" if migration_started or not bool(payload.get("previous_applied_snapshot")) else "configuration_rollback" ) if status == "configuration_restored": expected_mode = "configuration_rollback" if payload.get("recovery_mode") != expected_mode: raise ValueError("Deployment recovery mode does not match recovery evidence") def _validate_snapshot( before: Path, manifest: object, ) -> list[tuple[str, dict[str, Any]]]: if not isinstance(manifest, dict) or manifest.get("schema_version") != 1: raise ValueError("Applied-state snapshot manifest is invalid") files = manifest.get("files") if not isinstance(files, dict) or set(files) != set(_BUNDLE_FILES): raise ValueError("Applied-state snapshot manifest is incomplete") entries: list[tuple[str, dict[str, Any]]] = [] for filename in _BUNDLE_FILES: evidence = files.get(filename) if not isinstance(evidence, dict) or not isinstance( evidence.get("present"), bool, ): raise ValueError( f"Applied-state snapshot evidence is invalid for {filename}" ) if evidence["present"]: source = before / filename expected = str(evidence.get("sha256") or "") if ( source.is_symlink() or not source.is_file() or not re.fullmatch(r"[0-9a-f]{64}", expected) or sha256(source.read_bytes()).hexdigest() != expected ): raise ValueError( f"Applied-state snapshot checksum failed for {filename}" ) entries.append((filename, evidence)) return entries def _private_directory(path: Path) -> None: path.mkdir(parents=True, exist_ok=True, mode=0o700) path.chmod(0o700) def _private_copy(source: Path, destination: Path) -> Path: source = Path(source) destination = Path(destination) if source.is_symlink() or not source.is_file(): raise ValueError(f"Refusing to copy non-regular bundle file: {source.name}") atomic_write(destination, source.read_bytes(), mode=0o600) return destination def _now() -> str: return datetime.now(UTC).isoformat() __all__ = [ "DeploymentOperationJournal", "RecoveryResult", "list_operations", "load_operation", "recover_operation", "verify_operation_chain", "write_applied_state", ]