feat: implement institutional governance and recovery architecture
This commit is contained in:
@@ -0,0 +1,497 @@
|
||||
"""Durable deployment operation journal and bounded bundle recovery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from .bundle import BundlePaths, atomic_write, canonical_json
|
||||
|
||||
|
||||
OPERATIONS_DIRECTORY = "operations"
|
||||
APPLIED_STATE_DIRECTORY = "applied-state"
|
||||
OPERATION_FILENAME = "operation.json"
|
||||
_OPERATION_ID = re.compile(r"^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$")
|
||||
_BUNDLE_FILES = (
|
||||
"installation.json",
|
||||
"secrets.env",
|
||||
"compose.json",
|
||||
"garage.toml",
|
||||
"load-balancer.cfg",
|
||||
"receipt.json",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecoveryResult:
|
||||
operation_id: str
|
||||
action: str
|
||||
detail: str
|
||||
apply_allowed: bool
|
||||
|
||||
|
||||
class DeploymentOperationJournal:
|
||||
def __init__(self, operation_path: Path, payload: dict[str, Any]) -> None:
|
||||
self.path = operation_path
|
||||
self.payload = payload
|
||||
|
||||
@classmethod
|
||||
def begin(
|
||||
cls,
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
plan: dict[str, Any],
|
||||
) -> DeploymentOperationJournal:
|
||||
operations_root = paths.root / OPERATIONS_DIRECTORY
|
||||
_private_directory(operations_root)
|
||||
operation_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid4().hex[:8]
|
||||
operation_path = operations_root / operation_id
|
||||
_private_directory(operation_path)
|
||||
snapshot_source = paths.root / APPLIED_STATE_DIRECTORY
|
||||
snapshot_available = snapshot_source.is_dir()
|
||||
if snapshot_available:
|
||||
shutil.copytree(
|
||||
snapshot_source,
|
||||
operation_path / "before",
|
||||
copy_function=_private_copy,
|
||||
)
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 2,
|
||||
"operation_id": operation_id,
|
||||
"installation_id": plan.get("installation_id"),
|
||||
"status": "running",
|
||||
"recovery_mode": (
|
||||
"configuration_rollback" if snapshot_available else "forward_recovery"
|
||||
),
|
||||
"previous_applied_snapshot": snapshot_available,
|
||||
"migration_started": False,
|
||||
"migration_completed": False,
|
||||
"started_at": _now(),
|
||||
"updated_at": _now(),
|
||||
"completed_at": None,
|
||||
"failure_summary": None,
|
||||
"desired": plan,
|
||||
"stages": [],
|
||||
"evidence_head_sha256": None,
|
||||
}
|
||||
journal = cls(operation_path, payload)
|
||||
journal.record(
|
||||
"plan-recorded",
|
||||
"succeeded",
|
||||
{
|
||||
"previous_applied_snapshot": snapshot_available,
|
||||
"desired_sha256": sha256(canonical_json(plan)).hexdigest(),
|
||||
},
|
||||
)
|
||||
return journal
|
||||
|
||||
@property
|
||||
def operation_id(self) -> str:
|
||||
return str(self.payload["operation_id"])
|
||||
|
||||
def record(
|
||||
self,
|
||||
stage: str,
|
||||
status: str,
|
||||
evidence: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
stages = list(self.payload.get("stages") or [])
|
||||
item = {
|
||||
"sequence": len(stages) + 1,
|
||||
"stage": stage,
|
||||
"status": status,
|
||||
"at": _now(),
|
||||
"evidence": dict(evidence or {}),
|
||||
"previous_sha256": self.payload.get("evidence_head_sha256"),
|
||||
}
|
||||
item["checkpoint_sha256"] = sha256(canonical_json(item)).hexdigest()
|
||||
stages.append(item)
|
||||
self.payload["stages"] = stages
|
||||
self.payload["evidence_head_sha256"] = item["checkpoint_sha256"]
|
||||
self.payload["updated_at"] = item["at"]
|
||||
self._write()
|
||||
|
||||
def migration_started(self) -> None:
|
||||
self.payload["migration_started"] = True
|
||||
self.payload["recovery_mode"] = "forward_recovery"
|
||||
self.record(
|
||||
"migration-started",
|
||||
"running",
|
||||
{
|
||||
"rollback_boundary": (
|
||||
"Database migration has started; automatic release/configuration rollback is disabled."
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
def migration_completed(self) -> None:
|
||||
self.payload["migration_completed"] = True
|
||||
self.record("migration-completed", "succeeded")
|
||||
|
||||
def succeeded(self, paths: BundlePaths, *, receipt: dict[str, Any]) -> None:
|
||||
write_applied_state(paths)
|
||||
self.payload["status"] = "succeeded"
|
||||
self.payload["completed_at"] = _now()
|
||||
self.record(
|
||||
"deployment-verified",
|
||||
"succeeded",
|
||||
{
|
||||
"receipt_spec_sha256": receipt.get("spec_sha256"),
|
||||
"receipt_compose_sha256": receipt.get("compose_sha256"),
|
||||
},
|
||||
)
|
||||
|
||||
def failed(self, exc: BaseException) -> None:
|
||||
self.payload["status"] = "failed"
|
||||
self.payload["completed_at"] = _now()
|
||||
self.payload["failure_summary"] = f"{type(exc).__name__}: {str(exc)[:1000]}"
|
||||
self.record(
|
||||
"deployment-failed",
|
||||
"failed",
|
||||
{
|
||||
"exception_type": type(exc).__name__,
|
||||
"recovery_mode": self.payload.get("recovery_mode"),
|
||||
"failure_summary_sha256": sha256(
|
||||
str(self.payload["failure_summary"]).encode("utf-8")
|
||||
).hexdigest(),
|
||||
},
|
||||
)
|
||||
|
||||
def _write(self) -> None:
|
||||
atomic_write(
|
||||
self.path / OPERATION_FILENAME,
|
||||
canonical_json(self.payload),
|
||||
mode=0o600,
|
||||
)
|
||||
|
||||
|
||||
def write_applied_state(paths: BundlePaths) -> Path:
|
||||
target = paths.root / APPLIED_STATE_DIRECTORY
|
||||
temporary = paths.root / f".{APPLIED_STATE_DIRECTORY}.{uuid4().hex}.tmp"
|
||||
previous = paths.root / f".{APPLIED_STATE_DIRECTORY}.previous"
|
||||
_private_directory(temporary)
|
||||
manifest: dict[str, Any] = {"schema_version": 1, "captured_at": _now(), "files": {}}
|
||||
try:
|
||||
for filename in _BUNDLE_FILES:
|
||||
source = paths.root / filename
|
||||
if not source.is_file():
|
||||
manifest["files"][filename] = {"present": False}
|
||||
continue
|
||||
destination = temporary / filename
|
||||
_private_copy(source, destination)
|
||||
manifest["files"][filename] = {
|
||||
"present": True,
|
||||
"sha256": sha256(destination.read_bytes()).hexdigest(),
|
||||
}
|
||||
atomic_write(
|
||||
temporary / "manifest.json",
|
||||
canonical_json(manifest),
|
||||
mode=0o600,
|
||||
)
|
||||
if previous.exists() and not target.exists():
|
||||
previous.replace(target)
|
||||
elif previous.exists():
|
||||
shutil.rmtree(previous)
|
||||
if target.exists():
|
||||
target.replace(previous)
|
||||
try:
|
||||
temporary.replace(target)
|
||||
except BaseException:
|
||||
if previous.exists() and not target.exists():
|
||||
previous.replace(target)
|
||||
raise
|
||||
if previous.exists():
|
||||
shutil.rmtree(previous)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
shutil.rmtree(temporary)
|
||||
return target
|
||||
|
||||
|
||||
def list_operations(paths: BundlePaths) -> list[dict[str, Any]]:
|
||||
root = paths.root / OPERATIONS_DIRECTORY
|
||||
if not root.is_dir():
|
||||
return []
|
||||
operations: list[dict[str, Any]] = []
|
||||
for path in sorted(root.iterdir(), reverse=True):
|
||||
if not path.is_dir() or not _OPERATION_ID.fullmatch(path.name):
|
||||
continue
|
||||
payload = _read_operation(path)
|
||||
operations.append(
|
||||
{
|
||||
"operation_id": payload.get("operation_id"),
|
||||
"status": payload.get("status"),
|
||||
"recovery_mode": payload.get("recovery_mode"),
|
||||
"migration_started": bool(payload.get("migration_started")),
|
||||
"started_at": payload.get("started_at"),
|
||||
"completed_at": payload.get("completed_at"),
|
||||
"failure_summary": payload.get("failure_summary"),
|
||||
"stage_count": len(payload.get("stages") or []),
|
||||
}
|
||||
)
|
||||
return operations
|
||||
|
||||
|
||||
def recover_operation(
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
operation_id: str | None = None,
|
||||
) -> RecoveryResult:
|
||||
operation_path, payload = load_operation(paths, operation_id=operation_id)
|
||||
effective_id = str(payload["operation_id"])
|
||||
if bool(payload.get("migration_started")):
|
||||
journal = DeploymentOperationJournal(operation_path, payload)
|
||||
journal.payload["status"] = "forward_recovery_required"
|
||||
journal.payload["recovery_mode"] = "forward_recovery"
|
||||
journal.record(
|
||||
"forward-recovery-required",
|
||||
"blocked",
|
||||
{
|
||||
"reason": "migration_started",
|
||||
"configuration_restored": False,
|
||||
},
|
||||
)
|
||||
return RecoveryResult(
|
||||
operation_id=effective_id,
|
||||
action="forward_recovery",
|
||||
detail=(
|
||||
"Database migration started. The previous release/configuration was not restored; "
|
||||
"repair the current release or restore a separately verified database backup."
|
||||
),
|
||||
apply_allowed=True,
|
||||
)
|
||||
before = operation_path / "before"
|
||||
if not before.is_dir():
|
||||
return RecoveryResult(
|
||||
operation_id=effective_id,
|
||||
action="manual_intervention",
|
||||
detail="No independently captured previous applied state is available.",
|
||||
apply_allowed=False,
|
||||
)
|
||||
manifest = json.loads((before / "manifest.json").read_text(encoding="utf-8"))
|
||||
entries = _validate_snapshot(before, manifest)
|
||||
for filename, evidence in entries:
|
||||
target = paths.root / filename
|
||||
if evidence.get("present"):
|
||||
source = before / filename
|
||||
_private_copy(source, target)
|
||||
else:
|
||||
target.unlink(missing_ok=True)
|
||||
journal = DeploymentOperationJournal(operation_path, payload)
|
||||
journal.payload["status"] = "configuration_restored"
|
||||
journal.payload["recovery_mode"] = "configuration_rollback"
|
||||
journal.record(
|
||||
"configuration-restored",
|
||||
"succeeded",
|
||||
{"snapshot_manifest_verified": True},
|
||||
)
|
||||
return RecoveryResult(
|
||||
operation_id=effective_id,
|
||||
action="configuration_restored",
|
||||
detail=(
|
||||
"The previously applied bundle was restored and checksum-verified. "
|
||||
"Run apply to reconcile containers to that state."
|
||||
),
|
||||
apply_allowed=True,
|
||||
)
|
||||
|
||||
|
||||
def load_operation(
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
operation_id: str | None = None,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
root = paths.root / OPERATIONS_DIRECTORY
|
||||
if operation_id is None:
|
||||
candidates = (
|
||||
[
|
||||
item
|
||||
for item in sorted(root.iterdir(), reverse=True)
|
||||
if item.is_dir() and _OPERATION_ID.fullmatch(item.name)
|
||||
]
|
||||
if root.is_dir()
|
||||
else []
|
||||
)
|
||||
if not candidates:
|
||||
raise ValueError("No deployment operations are recorded")
|
||||
path = candidates[0]
|
||||
else:
|
||||
if not _OPERATION_ID.fullmatch(operation_id):
|
||||
raise ValueError("Invalid deployment operation id")
|
||||
path = root / operation_id
|
||||
return path, _read_operation(path)
|
||||
|
||||
|
||||
def _read_operation(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads((path / OPERATION_FILENAME).read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Deployment operation is unreadable: {path.name}") from exc
|
||||
if not isinstance(payload, dict) or payload.get("operation_id") != path.name:
|
||||
raise ValueError(f"Deployment operation identity is invalid: {path.name}")
|
||||
verify_operation_chain(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def verify_operation_chain(payload: dict[str, Any]) -> None:
|
||||
stages = payload.get("stages")
|
||||
if not isinstance(stages, list) or not stages:
|
||||
raise ValueError("Deployment operation has no recovery evidence chain")
|
||||
previous: str | None = None
|
||||
observed_migration_started = False
|
||||
observed_migration_completed = False
|
||||
for sequence, raw_stage in enumerate(stages, start=1):
|
||||
if not isinstance(raw_stage, dict):
|
||||
raise ValueError("Deployment recovery evidence contains an invalid stage")
|
||||
stage = dict(raw_stage)
|
||||
checkpoint = str(stage.pop("checkpoint_sha256", ""))
|
||||
if (
|
||||
stage.get("sequence") != sequence
|
||||
or stage.get("previous_sha256") != previous
|
||||
or sha256(canonical_json(stage)).hexdigest() != checkpoint
|
||||
):
|
||||
raise ValueError(
|
||||
f"Deployment recovery evidence chain failed at stage {sequence}"
|
||||
)
|
||||
previous = checkpoint
|
||||
observed_migration_started |= stage.get("stage") == "migration-started"
|
||||
observed_migration_completed |= stage.get("stage") == "migration-completed"
|
||||
if previous != payload.get("evidence_head_sha256"):
|
||||
raise ValueError("Deployment recovery evidence head does not match its chain")
|
||||
if bool(payload.get("migration_started")) != observed_migration_started:
|
||||
raise ValueError(
|
||||
"Deployment migration boundary does not match recovery evidence"
|
||||
)
|
||||
if bool(payload.get("migration_completed")) != observed_migration_completed:
|
||||
raise ValueError(
|
||||
"Deployment migration completion does not match recovery evidence"
|
||||
)
|
||||
if int(payload.get("schema_version") or 0) >= 2:
|
||||
_verify_operation_state(payload, stages)
|
||||
|
||||
|
||||
def _verify_operation_state(
|
||||
payload: dict[str, Any],
|
||||
stages: list[dict[str, Any]],
|
||||
) -> None:
|
||||
plan_evidence = stages[0].get("evidence")
|
||||
if (
|
||||
stages[0].get("stage") != "plan-recorded"
|
||||
or not isinstance(plan_evidence, dict)
|
||||
or plan_evidence.get("desired_sha256")
|
||||
!= sha256(canonical_json(payload.get("desired"))).hexdigest()
|
||||
or bool(plan_evidence.get("previous_applied_snapshot"))
|
||||
!= bool(payload.get("previous_applied_snapshot"))
|
||||
):
|
||||
raise ValueError("Deployment plan does not match recovery evidence")
|
||||
desired = payload.get("desired")
|
||||
if not isinstance(desired, dict) or desired.get("installation_id") != payload.get(
|
||||
"installation_id"
|
||||
):
|
||||
raise ValueError("Deployment installation identity does not match its plan")
|
||||
status = str(payload.get("status") or "")
|
||||
expected_last_stage = {
|
||||
"failed": "deployment-failed",
|
||||
"succeeded": "deployment-verified",
|
||||
"configuration_restored": "configuration-restored",
|
||||
"forward_recovery_required": "forward-recovery-required",
|
||||
}.get(status)
|
||||
if expected_last_stage and stages[-1].get("stage") != expected_last_stage:
|
||||
raise ValueError("Deployment status does not match recovery evidence")
|
||||
if status == "failed":
|
||||
failure_summary = str(payload.get("failure_summary") or "")
|
||||
evidence = stages[-1].get("evidence")
|
||||
if (
|
||||
not failure_summary
|
||||
or not isinstance(evidence, dict)
|
||||
or evidence.get("failure_summary_sha256")
|
||||
!= sha256(failure_summary.encode("utf-8")).hexdigest()
|
||||
):
|
||||
raise ValueError(
|
||||
"Deployment failure summary does not match recovery evidence"
|
||||
)
|
||||
if status in {"failed", "succeeded"} and not payload.get("completed_at"):
|
||||
raise ValueError("Completed deployment operation has no completion timestamp")
|
||||
migration_started = bool(payload.get("migration_started"))
|
||||
if bool(payload.get("migration_completed")) and not migration_started:
|
||||
raise ValueError("Completed migration has no recorded migration boundary")
|
||||
expected_mode = (
|
||||
"forward_recovery"
|
||||
if migration_started or not bool(payload.get("previous_applied_snapshot"))
|
||||
else "configuration_rollback"
|
||||
)
|
||||
if status == "configuration_restored":
|
||||
expected_mode = "configuration_rollback"
|
||||
if payload.get("recovery_mode") != expected_mode:
|
||||
raise ValueError("Deployment recovery mode does not match recovery evidence")
|
||||
|
||||
|
||||
def _validate_snapshot(
|
||||
before: Path,
|
||||
manifest: object,
|
||||
) -> list[tuple[str, dict[str, Any]]]:
|
||||
if not isinstance(manifest, dict) or manifest.get("schema_version") != 1:
|
||||
raise ValueError("Applied-state snapshot manifest is invalid")
|
||||
files = manifest.get("files")
|
||||
if not isinstance(files, dict) or set(files) != set(_BUNDLE_FILES):
|
||||
raise ValueError("Applied-state snapshot manifest is incomplete")
|
||||
entries: list[tuple[str, dict[str, Any]]] = []
|
||||
for filename in _BUNDLE_FILES:
|
||||
evidence = files.get(filename)
|
||||
if not isinstance(evidence, dict) or not isinstance(
|
||||
evidence.get("present"),
|
||||
bool,
|
||||
):
|
||||
raise ValueError(
|
||||
f"Applied-state snapshot evidence is invalid for {filename}"
|
||||
)
|
||||
if evidence["present"]:
|
||||
source = before / filename
|
||||
expected = str(evidence.get("sha256") or "")
|
||||
if (
|
||||
source.is_symlink()
|
||||
or not source.is_file()
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", expected)
|
||||
or sha256(source.read_bytes()).hexdigest() != expected
|
||||
):
|
||||
raise ValueError(
|
||||
f"Applied-state snapshot checksum failed for {filename}"
|
||||
)
|
||||
entries.append((filename, evidence))
|
||||
return entries
|
||||
|
||||
|
||||
def _private_directory(path: Path) -> None:
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
path.chmod(0o700)
|
||||
|
||||
|
||||
def _private_copy(source: Path, destination: Path) -> Path:
|
||||
source = Path(source)
|
||||
destination = Path(destination)
|
||||
if source.is_symlink() or not source.is_file():
|
||||
raise ValueError(f"Refusing to copy non-regular bundle file: {source.name}")
|
||||
atomic_write(destination, source.read_bytes(), mode=0o600)
|
||||
return destination
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeploymentOperationJournal",
|
||||
"RecoveryResult",
|
||||
"list_operations",
|
||||
"load_operation",
|
||||
"recover_operation",
|
||||
"verify_operation_chain",
|
||||
"write_applied_state",
|
||||
]
|
||||
Reference in New Issue
Block a user