from __future__ import annotations from datetime import datetime, timezone from sqlalchemy import create_engine from sqlalchemy.orm import Session import pytest from govoplan_core.core.recovery import ( RecoveryCheckpoint, RecoveryGuaranteeError, RecoveryMode, RecoveryOperation, RecoveryPlan, RecoveryStatus, plan_recovery_operation, prepare_recovery_operation, record_recovery_checkpoint, start_recovery_operation, transition_recovery_operation, verify_recovery_evidence_chain, ) from govoplan_core.core.runtime_coordination import DistributedLease, acquire_lease from govoplan_core.db.base import Base def _session() -> tuple[Session, object]: engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all( engine, tables=[ DistributedLease.__table__, RecoveryOperation.__table__, RecoveryCheckpoint.__table__, ], ) return Session(engine, expire_on_commit=False), engine def test_recovery_plan_requires_mode_specific_evidence() -> None: with pytest.raises(RecoveryGuaranteeError, match="compensation steps"): RecoveryPlan( mode=RecoveryMode.COMPENSATION, verification_steps=("verify",), ).validate() with pytest.raises(RecoveryGuaranteeError, match="backup reference"): RecoveryPlan( mode=RecoveryMode.SNAPSHOT_RESTORE, verification_steps=("verify",), ).validate() with pytest.raises(RecoveryGuaranteeError, match="approval reference"): RecoveryPlan( mode=RecoveryMode.IRREVERSIBLE, verification_steps=("verify",), ).validate() def test_recovery_plan_hashes_backup_and_approval_references() -> None: plan = RecoveryPlan( mode=RecoveryMode.SNAPSHOT_RESTORE, verification_steps=("restore drill passed",), backup_reference="backup://postgres/2026-08-01", approval_reference="approval://change/42", ) assert plan.as_dict()["backup_reference"] == "backup://postgres/2026-08-01" assert plan.as_dict()["approval_reference"] == "approval://change/42" def test_recovery_operation_records_verifiable_checkpoint_chain() -> None: session, engine = _session() try: operation = plan_recovery_operation( session, installation_id="installation-1", module_id="files", operation_type="move-tree", idempotency_key="move-42", request={"source": "a", "target": "b"}, recovery_plan=RecoveryPlan( mode=RecoveryMode.COMPENSATION, compensation_steps=("restore source links",), verification_steps=("compare object inventory",), ), ) prepare_recovery_operation( session, operation, evidence={"inventory_sha256": "a" * 64}, ) start_recovery_operation(session, operation) transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERY_REQUIRED, kind="side-effect-failed", summary="Target index update failed", evidence={"completed_steps": 1}, failure_summary="target index unavailable", ) transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERING, kind="compensation-started", summary="Compensation started", ) transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERED, kind="compensation-verified", summary="Source inventory restored", evidence={ "verified": True, "checks": {"object_inventory": "matched"}, }, ) session.commit() assert operation.status == RecoveryStatus.RECOVERED.value assert operation.checkpoint_count == 6 assert verify_recovery_evidence_chain(session, operation.id) is True finally: session.close() engine.dispose() def test_recovery_transition_cannot_skip_preparation() -> None: session, engine = _session() try: operation = plan_recovery_operation( session, installation_id="installation-1", module_id="core", operation_type="configuration-change", idempotency_key="config-1", request={"revision": 2}, recovery_plan=RecoveryPlan( mode=RecoveryMode.ATOMIC, verification_steps=("read current revision",), ), ) with pytest.raises(RecoveryGuaranteeError, match="not allowed"): start_recovery_operation(session, operation) finally: session.close() engine.dispose() def test_non_atomic_failure_requires_explicit_recovery_and_verified_completion() -> ( None ): session, engine = _session() try: operation = plan_recovery_operation( session, installation_id="installation-1", module_id="campaign", operation_type="artifact-publish", idempotency_key="publish-1", request={"version": "version-1"}, recovery_plan=RecoveryPlan( mode=RecoveryMode.COMPENSATION, compensation_steps=("delete published objects",), verification_steps=("verify object inventory",), ), ) prepare_recovery_operation( session, operation, evidence={"preconditions": "verified"}, ) start_recovery_operation(session, operation) with pytest.raises(RecoveryGuaranteeError, match="cannot be marked failed"): transition_recovery_operation( session, operation, status=RecoveryStatus.FAILED, kind="failed", summary="Object publication failed", ) transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERY_REQUIRED, kind="recovery-required", summary="Published objects require compensation", ) transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERING, kind="compensation-started", summary="Removing published objects", ) with pytest.raises(RecoveryGuaranteeError, match="verified evidence"): transition_recovery_operation( session, operation, status=RecoveryStatus.RECOVERED, kind="compensation-finished", summary="Objects removed", ) finally: session.close() engine.dispose() def test_recovery_evidence_rejects_plaintext_secrets() -> None: session, engine = _session() try: with pytest.raises(RecoveryGuaranteeError, match="plaintext secrets"): plan_recovery_operation( session, installation_id="installation-1", module_id="mail", operation_type="transport-change", idempotency_key="transport-1", request={"transport_id": "smtp-1"}, metadata={"password": "do-not-store"}, recovery_plan=RecoveryPlan( mode=RecoveryMode.ATOMIC, verification_steps=("read persisted transport",), ), ) finally: session.close() engine.dispose() def test_fenced_recovery_rejects_low_level_mutation_without_claim() -> None: session, engine = _session() try: claim = acquire_lease( session, installation_id="installation-1", resource_key="files:move-tree", holder_node_id="worker-1", holder_incarnation="worker-1-incarnation", ttl_seconds=300, now=datetime.now(timezone.utc), ) assert claim is not None operation = plan_recovery_operation( session, installation_id="installation-1", module_id="files", operation_type="move-tree", idempotency_key="move-fenced-1", request={"source": "a", "target": "b"}, recovery_plan=RecoveryPlan( mode=RecoveryMode.COMPENSATION, compensation_steps=("restore source links",), verification_steps=("compare object inventory",), ), lease_claim=claim, ) with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"): transition_recovery_operation( session, operation, status=RecoveryStatus.PREPARED, kind="prepared", summary="Attempted without fence", evidence={"inventory_sha256": "a" * 64}, ) with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"): record_recovery_checkpoint( session, operation, kind="manual-note", summary="Attempted without fence", evidence={}, ) prepared = prepare_recovery_operation( session, operation, evidence={"inventory_sha256": "a" * 64}, lease_claim=claim, ) assert prepared.status == RecoveryStatus.PREPARED.value finally: session.close() engine.dispose()