498 lines
18 KiB
Python
498 lines
18 KiB
Python
"""Durable deployment operation journal and bounded bundle recovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from .bundle import BundlePaths, atomic_write, canonical_json
|
|
|
|
|
|
OPERATIONS_DIRECTORY = "operations"
|
|
APPLIED_STATE_DIRECTORY = "applied-state"
|
|
OPERATION_FILENAME = "operation.json"
|
|
_OPERATION_ID = re.compile(r"^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$")
|
|
_BUNDLE_FILES = (
|
|
"installation.json",
|
|
"secrets.env",
|
|
"compose.json",
|
|
"garage.toml",
|
|
"load-balancer.cfg",
|
|
"receipt.json",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecoveryResult:
|
|
operation_id: str
|
|
action: str
|
|
detail: str
|
|
apply_allowed: bool
|
|
|
|
|
|
class DeploymentOperationJournal:
|
|
def __init__(self, operation_path: Path, payload: dict[str, Any]) -> None:
|
|
self.path = operation_path
|
|
self.payload = payload
|
|
|
|
@classmethod
|
|
def begin(
|
|
cls,
|
|
paths: BundlePaths,
|
|
*,
|
|
plan: dict[str, Any],
|
|
) -> DeploymentOperationJournal:
|
|
operations_root = paths.root / OPERATIONS_DIRECTORY
|
|
_private_directory(operations_root)
|
|
operation_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid4().hex[:8]
|
|
operation_path = operations_root / operation_id
|
|
_private_directory(operation_path)
|
|
snapshot_source = paths.root / APPLIED_STATE_DIRECTORY
|
|
snapshot_available = snapshot_source.is_dir()
|
|
if snapshot_available:
|
|
shutil.copytree(
|
|
snapshot_source,
|
|
operation_path / "before",
|
|
copy_function=_private_copy,
|
|
)
|
|
payload: dict[str, Any] = {
|
|
"schema_version": 2,
|
|
"operation_id": operation_id,
|
|
"installation_id": plan.get("installation_id"),
|
|
"status": "running",
|
|
"recovery_mode": (
|
|
"configuration_rollback" if snapshot_available else "forward_recovery"
|
|
),
|
|
"previous_applied_snapshot": snapshot_available,
|
|
"migration_started": False,
|
|
"migration_completed": False,
|
|
"started_at": _now(),
|
|
"updated_at": _now(),
|
|
"completed_at": None,
|
|
"failure_summary": None,
|
|
"desired": plan,
|
|
"stages": [],
|
|
"evidence_head_sha256": None,
|
|
}
|
|
journal = cls(operation_path, payload)
|
|
journal.record(
|
|
"plan-recorded",
|
|
"succeeded",
|
|
{
|
|
"previous_applied_snapshot": snapshot_available,
|
|
"desired_sha256": sha256(canonical_json(plan)).hexdigest(),
|
|
},
|
|
)
|
|
return journal
|
|
|
|
@property
|
|
def operation_id(self) -> str:
|
|
return str(self.payload["operation_id"])
|
|
|
|
def record(
|
|
self,
|
|
stage: str,
|
|
status: str,
|
|
evidence: dict[str, Any] | None = None,
|
|
) -> None:
|
|
stages = list(self.payload.get("stages") or [])
|
|
item = {
|
|
"sequence": len(stages) + 1,
|
|
"stage": stage,
|
|
"status": status,
|
|
"at": _now(),
|
|
"evidence": dict(evidence or {}),
|
|
"previous_sha256": self.payload.get("evidence_head_sha256"),
|
|
}
|
|
item["checkpoint_sha256"] = sha256(canonical_json(item)).hexdigest()
|
|
stages.append(item)
|
|
self.payload["stages"] = stages
|
|
self.payload["evidence_head_sha256"] = item["checkpoint_sha256"]
|
|
self.payload["updated_at"] = item["at"]
|
|
self._write()
|
|
|
|
def migration_started(self) -> None:
|
|
self.payload["migration_started"] = True
|
|
self.payload["recovery_mode"] = "forward_recovery"
|
|
self.record(
|
|
"migration-started",
|
|
"running",
|
|
{
|
|
"rollback_boundary": (
|
|
"Database migration has started; automatic release/configuration rollback is disabled."
|
|
)
|
|
},
|
|
)
|
|
|
|
def migration_completed(self) -> None:
|
|
self.payload["migration_completed"] = True
|
|
self.record("migration-completed", "succeeded")
|
|
|
|
def succeeded(self, paths: BundlePaths, *, receipt: dict[str, Any]) -> None:
|
|
write_applied_state(paths)
|
|
self.payload["status"] = "succeeded"
|
|
self.payload["completed_at"] = _now()
|
|
self.record(
|
|
"deployment-verified",
|
|
"succeeded",
|
|
{
|
|
"receipt_spec_sha256": receipt.get("spec_sha256"),
|
|
"receipt_compose_sha256": receipt.get("compose_sha256"),
|
|
},
|
|
)
|
|
|
|
def failed(self, exc: BaseException) -> None:
|
|
self.payload["status"] = "failed"
|
|
self.payload["completed_at"] = _now()
|
|
self.payload["failure_summary"] = f"{type(exc).__name__}: {str(exc)[:1000]}"
|
|
self.record(
|
|
"deployment-failed",
|
|
"failed",
|
|
{
|
|
"exception_type": type(exc).__name__,
|
|
"recovery_mode": self.payload.get("recovery_mode"),
|
|
"failure_summary_sha256": sha256(
|
|
str(self.payload["failure_summary"]).encode("utf-8")
|
|
).hexdigest(),
|
|
},
|
|
)
|
|
|
|
def _write(self) -> None:
|
|
atomic_write(
|
|
self.path / OPERATION_FILENAME,
|
|
canonical_json(self.payload),
|
|
mode=0o600,
|
|
)
|
|
|
|
|
|
def write_applied_state(paths: BundlePaths) -> Path:
|
|
target = paths.root / APPLIED_STATE_DIRECTORY
|
|
temporary = paths.root / f".{APPLIED_STATE_DIRECTORY}.{uuid4().hex}.tmp"
|
|
previous = paths.root / f".{APPLIED_STATE_DIRECTORY}.previous"
|
|
_private_directory(temporary)
|
|
manifest: dict[str, Any] = {"schema_version": 1, "captured_at": _now(), "files": {}}
|
|
try:
|
|
for filename in _BUNDLE_FILES:
|
|
source = paths.root / filename
|
|
if not source.is_file():
|
|
manifest["files"][filename] = {"present": False}
|
|
continue
|
|
destination = temporary / filename
|
|
_private_copy(source, destination)
|
|
manifest["files"][filename] = {
|
|
"present": True,
|
|
"sha256": sha256(destination.read_bytes()).hexdigest(),
|
|
}
|
|
atomic_write(
|
|
temporary / "manifest.json",
|
|
canonical_json(manifest),
|
|
mode=0o600,
|
|
)
|
|
if previous.exists() and not target.exists():
|
|
previous.replace(target)
|
|
elif previous.exists():
|
|
shutil.rmtree(previous)
|
|
if target.exists():
|
|
target.replace(previous)
|
|
try:
|
|
temporary.replace(target)
|
|
except BaseException:
|
|
if previous.exists() and not target.exists():
|
|
previous.replace(target)
|
|
raise
|
|
if previous.exists():
|
|
shutil.rmtree(previous)
|
|
finally:
|
|
if temporary.exists():
|
|
shutil.rmtree(temporary)
|
|
return target
|
|
|
|
|
|
def list_operations(paths: BundlePaths) -> list[dict[str, Any]]:
|
|
root = paths.root / OPERATIONS_DIRECTORY
|
|
if not root.is_dir():
|
|
return []
|
|
operations: list[dict[str, Any]] = []
|
|
for path in sorted(root.iterdir(), reverse=True):
|
|
if not path.is_dir() or not _OPERATION_ID.fullmatch(path.name):
|
|
continue
|
|
payload = _read_operation(path)
|
|
operations.append(
|
|
{
|
|
"operation_id": payload.get("operation_id"),
|
|
"status": payload.get("status"),
|
|
"recovery_mode": payload.get("recovery_mode"),
|
|
"migration_started": bool(payload.get("migration_started")),
|
|
"started_at": payload.get("started_at"),
|
|
"completed_at": payload.get("completed_at"),
|
|
"failure_summary": payload.get("failure_summary"),
|
|
"stage_count": len(payload.get("stages") or []),
|
|
}
|
|
)
|
|
return operations
|
|
|
|
|
|
def recover_operation(
|
|
paths: BundlePaths,
|
|
*,
|
|
operation_id: str | None = None,
|
|
) -> RecoveryResult:
|
|
operation_path, payload = load_operation(paths, operation_id=operation_id)
|
|
effective_id = str(payload["operation_id"])
|
|
if bool(payload.get("migration_started")):
|
|
journal = DeploymentOperationJournal(operation_path, payload)
|
|
journal.payload["status"] = "forward_recovery_required"
|
|
journal.payload["recovery_mode"] = "forward_recovery"
|
|
journal.record(
|
|
"forward-recovery-required",
|
|
"blocked",
|
|
{
|
|
"reason": "migration_started",
|
|
"configuration_restored": False,
|
|
},
|
|
)
|
|
return RecoveryResult(
|
|
operation_id=effective_id,
|
|
action="forward_recovery",
|
|
detail=(
|
|
"Database migration started. The previous release/configuration was not restored; "
|
|
"repair the current release or restore a separately verified database backup."
|
|
),
|
|
apply_allowed=True,
|
|
)
|
|
before = operation_path / "before"
|
|
if not before.is_dir():
|
|
return RecoveryResult(
|
|
operation_id=effective_id,
|
|
action="manual_intervention",
|
|
detail="No independently captured previous applied state is available.",
|
|
apply_allowed=False,
|
|
)
|
|
manifest = json.loads((before / "manifest.json").read_text(encoding="utf-8"))
|
|
entries = _validate_snapshot(before, manifest)
|
|
for filename, evidence in entries:
|
|
target = paths.root / filename
|
|
if evidence.get("present"):
|
|
source = before / filename
|
|
_private_copy(source, target)
|
|
else:
|
|
target.unlink(missing_ok=True)
|
|
journal = DeploymentOperationJournal(operation_path, payload)
|
|
journal.payload["status"] = "configuration_restored"
|
|
journal.payload["recovery_mode"] = "configuration_rollback"
|
|
journal.record(
|
|
"configuration-restored",
|
|
"succeeded",
|
|
{"snapshot_manifest_verified": True},
|
|
)
|
|
return RecoveryResult(
|
|
operation_id=effective_id,
|
|
action="configuration_restored",
|
|
detail=(
|
|
"The previously applied bundle was restored and checksum-verified. "
|
|
"Run apply to reconcile containers to that state."
|
|
),
|
|
apply_allowed=True,
|
|
)
|
|
|
|
|
|
def load_operation(
|
|
paths: BundlePaths,
|
|
*,
|
|
operation_id: str | None = None,
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
root = paths.root / OPERATIONS_DIRECTORY
|
|
if operation_id is None:
|
|
candidates = (
|
|
[
|
|
item
|
|
for item in sorted(root.iterdir(), reverse=True)
|
|
if item.is_dir() and _OPERATION_ID.fullmatch(item.name)
|
|
]
|
|
if root.is_dir()
|
|
else []
|
|
)
|
|
if not candidates:
|
|
raise ValueError("No deployment operations are recorded")
|
|
path = candidates[0]
|
|
else:
|
|
if not _OPERATION_ID.fullmatch(operation_id):
|
|
raise ValueError("Invalid deployment operation id")
|
|
path = root / operation_id
|
|
return path, _read_operation(path)
|
|
|
|
|
|
def _read_operation(path: Path) -> dict[str, Any]:
|
|
try:
|
|
payload = json.loads((path / OPERATION_FILENAME).read_text(encoding="utf-8"))
|
|
except (FileNotFoundError, json.JSONDecodeError) as exc:
|
|
raise ValueError(f"Deployment operation is unreadable: {path.name}") from exc
|
|
if not isinstance(payload, dict) or payload.get("operation_id") != path.name:
|
|
raise ValueError(f"Deployment operation identity is invalid: {path.name}")
|
|
verify_operation_chain(payload)
|
|
return payload
|
|
|
|
|
|
def verify_operation_chain(payload: dict[str, Any]) -> None:
|
|
stages = payload.get("stages")
|
|
if not isinstance(stages, list) or not stages:
|
|
raise ValueError("Deployment operation has no recovery evidence chain")
|
|
previous: str | None = None
|
|
observed_migration_started = False
|
|
observed_migration_completed = False
|
|
for sequence, raw_stage in enumerate(stages, start=1):
|
|
if not isinstance(raw_stage, dict):
|
|
raise ValueError("Deployment recovery evidence contains an invalid stage")
|
|
stage = dict(raw_stage)
|
|
checkpoint = str(stage.pop("checkpoint_sha256", ""))
|
|
if (
|
|
stage.get("sequence") != sequence
|
|
or stage.get("previous_sha256") != previous
|
|
or sha256(canonical_json(stage)).hexdigest() != checkpoint
|
|
):
|
|
raise ValueError(
|
|
f"Deployment recovery evidence chain failed at stage {sequence}"
|
|
)
|
|
previous = checkpoint
|
|
observed_migration_started |= stage.get("stage") == "migration-started"
|
|
observed_migration_completed |= stage.get("stage") == "migration-completed"
|
|
if previous != payload.get("evidence_head_sha256"):
|
|
raise ValueError("Deployment recovery evidence head does not match its chain")
|
|
if bool(payload.get("migration_started")) != observed_migration_started:
|
|
raise ValueError(
|
|
"Deployment migration boundary does not match recovery evidence"
|
|
)
|
|
if bool(payload.get("migration_completed")) != observed_migration_completed:
|
|
raise ValueError(
|
|
"Deployment migration completion does not match recovery evidence"
|
|
)
|
|
if int(payload.get("schema_version") or 0) >= 2:
|
|
_verify_operation_state(payload, stages)
|
|
|
|
|
|
def _verify_operation_state(
|
|
payload: dict[str, Any],
|
|
stages: list[dict[str, Any]],
|
|
) -> None:
|
|
plan_evidence = stages[0].get("evidence")
|
|
if (
|
|
stages[0].get("stage") != "plan-recorded"
|
|
or not isinstance(plan_evidence, dict)
|
|
or plan_evidence.get("desired_sha256")
|
|
!= sha256(canonical_json(payload.get("desired"))).hexdigest()
|
|
or bool(plan_evidence.get("previous_applied_snapshot"))
|
|
!= bool(payload.get("previous_applied_snapshot"))
|
|
):
|
|
raise ValueError("Deployment plan does not match recovery evidence")
|
|
desired = payload.get("desired")
|
|
if not isinstance(desired, dict) or desired.get("installation_id") != payload.get(
|
|
"installation_id"
|
|
):
|
|
raise ValueError("Deployment installation identity does not match its plan")
|
|
status = str(payload.get("status") or "")
|
|
expected_last_stage = {
|
|
"failed": "deployment-failed",
|
|
"succeeded": "deployment-verified",
|
|
"configuration_restored": "configuration-restored",
|
|
"forward_recovery_required": "forward-recovery-required",
|
|
}.get(status)
|
|
if expected_last_stage and stages[-1].get("stage") != expected_last_stage:
|
|
raise ValueError("Deployment status does not match recovery evidence")
|
|
if status == "failed":
|
|
failure_summary = str(payload.get("failure_summary") or "")
|
|
evidence = stages[-1].get("evidence")
|
|
if (
|
|
not failure_summary
|
|
or not isinstance(evidence, dict)
|
|
or evidence.get("failure_summary_sha256")
|
|
!= sha256(failure_summary.encode("utf-8")).hexdigest()
|
|
):
|
|
raise ValueError(
|
|
"Deployment failure summary does not match recovery evidence"
|
|
)
|
|
if status in {"failed", "succeeded"} and not payload.get("completed_at"):
|
|
raise ValueError("Completed deployment operation has no completion timestamp")
|
|
migration_started = bool(payload.get("migration_started"))
|
|
if bool(payload.get("migration_completed")) and not migration_started:
|
|
raise ValueError("Completed migration has no recorded migration boundary")
|
|
expected_mode = (
|
|
"forward_recovery"
|
|
if migration_started or not bool(payload.get("previous_applied_snapshot"))
|
|
else "configuration_rollback"
|
|
)
|
|
if status == "configuration_restored":
|
|
expected_mode = "configuration_rollback"
|
|
if payload.get("recovery_mode") != expected_mode:
|
|
raise ValueError("Deployment recovery mode does not match recovery evidence")
|
|
|
|
|
|
def _validate_snapshot(
|
|
before: Path,
|
|
manifest: object,
|
|
) -> list[tuple[str, dict[str, Any]]]:
|
|
if not isinstance(manifest, dict) or manifest.get("schema_version") != 1:
|
|
raise ValueError("Applied-state snapshot manifest is invalid")
|
|
files = manifest.get("files")
|
|
if not isinstance(files, dict) or set(files) != set(_BUNDLE_FILES):
|
|
raise ValueError("Applied-state snapshot manifest is incomplete")
|
|
entries: list[tuple[str, dict[str, Any]]] = []
|
|
for filename in _BUNDLE_FILES:
|
|
evidence = files.get(filename)
|
|
if not isinstance(evidence, dict) or not isinstance(
|
|
evidence.get("present"),
|
|
bool,
|
|
):
|
|
raise ValueError(
|
|
f"Applied-state snapshot evidence is invalid for {filename}"
|
|
)
|
|
if evidence["present"]:
|
|
source = before / filename
|
|
expected = str(evidence.get("sha256") or "")
|
|
if (
|
|
source.is_symlink()
|
|
or not source.is_file()
|
|
or not re.fullmatch(r"[0-9a-f]{64}", expected)
|
|
or sha256(source.read_bytes()).hexdigest() != expected
|
|
):
|
|
raise ValueError(
|
|
f"Applied-state snapshot checksum failed for {filename}"
|
|
)
|
|
entries.append((filename, evidence))
|
|
return entries
|
|
|
|
|
|
def _private_directory(path: Path) -> None:
|
|
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
path.chmod(0o700)
|
|
|
|
|
|
def _private_copy(source: Path, destination: Path) -> Path:
|
|
source = Path(source)
|
|
destination = Path(destination)
|
|
if source.is_symlink() or not source.is_file():
|
|
raise ValueError(f"Refusing to copy non-regular bundle file: {source.name}")
|
|
atomic_write(destination, source.read_bytes(), mode=0o600)
|
|
return destination
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
__all__ = [
|
|
"DeploymentOperationJournal",
|
|
"RecoveryResult",
|
|
"list_operations",
|
|
"load_operation",
|
|
"recover_operation",
|
|
"verify_operation_chain",
|
|
"write_applied_state",
|
|
]
|