289 lines
9.6 KiB
Python
289 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session
|
|
import pytest
|
|
|
|
from govoplan_core.core.recovery import (
|
|
RecoveryCheckpoint,
|
|
RecoveryGuaranteeError,
|
|
RecoveryMode,
|
|
RecoveryOperation,
|
|
RecoveryPlan,
|
|
RecoveryStatus,
|
|
plan_recovery_operation,
|
|
prepare_recovery_operation,
|
|
record_recovery_checkpoint,
|
|
start_recovery_operation,
|
|
transition_recovery_operation,
|
|
verify_recovery_evidence_chain,
|
|
)
|
|
from govoplan_core.core.runtime_coordination import DistributedLease, acquire_lease
|
|
from govoplan_core.db.base import Base
|
|
|
|
|
|
def _session() -> tuple[Session, object]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(
|
|
engine,
|
|
tables=[
|
|
DistributedLease.__table__,
|
|
RecoveryOperation.__table__,
|
|
RecoveryCheckpoint.__table__,
|
|
],
|
|
)
|
|
return Session(engine, expire_on_commit=False), engine
|
|
|
|
|
|
def test_recovery_plan_requires_mode_specific_evidence() -> None:
|
|
with pytest.raises(RecoveryGuaranteeError, match="compensation steps"):
|
|
RecoveryPlan(
|
|
mode=RecoveryMode.COMPENSATION,
|
|
verification_steps=("verify",),
|
|
).validate()
|
|
with pytest.raises(RecoveryGuaranteeError, match="backup reference"):
|
|
RecoveryPlan(
|
|
mode=RecoveryMode.SNAPSHOT_RESTORE,
|
|
verification_steps=("verify",),
|
|
).validate()
|
|
with pytest.raises(RecoveryGuaranteeError, match="approval reference"):
|
|
RecoveryPlan(
|
|
mode=RecoveryMode.IRREVERSIBLE,
|
|
verification_steps=("verify",),
|
|
).validate()
|
|
|
|
|
|
def test_recovery_plan_hashes_backup_and_approval_references() -> None:
|
|
plan = RecoveryPlan(
|
|
mode=RecoveryMode.SNAPSHOT_RESTORE,
|
|
verification_steps=("restore drill passed",),
|
|
backup_reference="backup://postgres/2026-08-01",
|
|
approval_reference="approval://change/42",
|
|
)
|
|
|
|
assert plan.as_dict()["backup_reference"] == "backup://postgres/2026-08-01"
|
|
assert plan.as_dict()["approval_reference"] == "approval://change/42"
|
|
|
|
|
|
def test_recovery_operation_records_verifiable_checkpoint_chain() -> None:
|
|
session, engine = _session()
|
|
try:
|
|
operation = plan_recovery_operation(
|
|
session,
|
|
installation_id="installation-1",
|
|
module_id="files",
|
|
operation_type="move-tree",
|
|
idempotency_key="move-42",
|
|
request={"source": "a", "target": "b"},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.COMPENSATION,
|
|
compensation_steps=("restore source links",),
|
|
verification_steps=("compare object inventory",),
|
|
),
|
|
)
|
|
prepare_recovery_operation(
|
|
session,
|
|
operation,
|
|
evidence={"inventory_sha256": "a" * 64},
|
|
)
|
|
start_recovery_operation(session, operation)
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
kind="side-effect-failed",
|
|
summary="Target index update failed",
|
|
evidence={"completed_steps": 1},
|
|
failure_summary="target index unavailable",
|
|
)
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERING,
|
|
kind="compensation-started",
|
|
summary="Compensation started",
|
|
)
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERED,
|
|
kind="compensation-verified",
|
|
summary="Source inventory restored",
|
|
evidence={
|
|
"verified": True,
|
|
"checks": {"object_inventory": "matched"},
|
|
},
|
|
)
|
|
session.commit()
|
|
|
|
assert operation.status == RecoveryStatus.RECOVERED.value
|
|
assert operation.checkpoint_count == 6
|
|
assert verify_recovery_evidence_chain(session, operation.id) is True
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def test_recovery_transition_cannot_skip_preparation() -> None:
|
|
session, engine = _session()
|
|
try:
|
|
operation = plan_recovery_operation(
|
|
session,
|
|
installation_id="installation-1",
|
|
module_id="core",
|
|
operation_type="configuration-change",
|
|
idempotency_key="config-1",
|
|
request={"revision": 2},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.ATOMIC,
|
|
verification_steps=("read current revision",),
|
|
),
|
|
)
|
|
with pytest.raises(RecoveryGuaranteeError, match="not allowed"):
|
|
start_recovery_operation(session, operation)
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def test_non_atomic_failure_requires_explicit_recovery_and_verified_completion() -> (
|
|
None
|
|
):
|
|
session, engine = _session()
|
|
try:
|
|
operation = plan_recovery_operation(
|
|
session,
|
|
installation_id="installation-1",
|
|
module_id="campaign",
|
|
operation_type="artifact-publish",
|
|
idempotency_key="publish-1",
|
|
request={"version": "version-1"},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.COMPENSATION,
|
|
compensation_steps=("delete published objects",),
|
|
verification_steps=("verify object inventory",),
|
|
),
|
|
)
|
|
prepare_recovery_operation(
|
|
session,
|
|
operation,
|
|
evidence={"preconditions": "verified"},
|
|
)
|
|
start_recovery_operation(session, operation)
|
|
|
|
with pytest.raises(RecoveryGuaranteeError, match="cannot be marked failed"):
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.FAILED,
|
|
kind="failed",
|
|
summary="Object publication failed",
|
|
)
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
kind="recovery-required",
|
|
summary="Published objects require compensation",
|
|
)
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERING,
|
|
kind="compensation-started",
|
|
summary="Removing published objects",
|
|
)
|
|
with pytest.raises(RecoveryGuaranteeError, match="verified evidence"):
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.RECOVERED,
|
|
kind="compensation-finished",
|
|
summary="Objects removed",
|
|
)
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def test_recovery_evidence_rejects_plaintext_secrets() -> None:
|
|
session, engine = _session()
|
|
try:
|
|
with pytest.raises(RecoveryGuaranteeError, match="plaintext secrets"):
|
|
plan_recovery_operation(
|
|
session,
|
|
installation_id="installation-1",
|
|
module_id="mail",
|
|
operation_type="transport-change",
|
|
idempotency_key="transport-1",
|
|
request={"transport_id": "smtp-1"},
|
|
metadata={"password": "do-not-store"},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.ATOMIC,
|
|
verification_steps=("read persisted transport",),
|
|
),
|
|
)
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def test_fenced_recovery_rejects_low_level_mutation_without_claim() -> None:
|
|
session, engine = _session()
|
|
try:
|
|
claim = acquire_lease(
|
|
session,
|
|
installation_id="installation-1",
|
|
resource_key="files:move-tree",
|
|
holder_node_id="worker-1",
|
|
holder_incarnation="worker-1-incarnation",
|
|
ttl_seconds=300,
|
|
now=datetime.now(timezone.utc),
|
|
)
|
|
assert claim is not None
|
|
operation = plan_recovery_operation(
|
|
session,
|
|
installation_id="installation-1",
|
|
module_id="files",
|
|
operation_type="move-tree",
|
|
idempotency_key="move-fenced-1",
|
|
request={"source": "a", "target": "b"},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.COMPENSATION,
|
|
compensation_steps=("restore source links",),
|
|
verification_steps=("compare object inventory",),
|
|
),
|
|
lease_claim=claim,
|
|
)
|
|
|
|
with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"):
|
|
transition_recovery_operation(
|
|
session,
|
|
operation,
|
|
status=RecoveryStatus.PREPARED,
|
|
kind="prepared",
|
|
summary="Attempted without fence",
|
|
evidence={"inventory_sha256": "a" * 64},
|
|
)
|
|
with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"):
|
|
record_recovery_checkpoint(
|
|
session,
|
|
operation,
|
|
kind="manual-note",
|
|
summary="Attempted without fence",
|
|
evidence={},
|
|
)
|
|
|
|
prepared = prepare_recovery_operation(
|
|
session,
|
|
operation,
|
|
evidence={"inventory_sha256": "a" * 64},
|
|
lease_claim=claim,
|
|
)
|
|
assert prepared.status == RecoveryStatus.PREPARED.value
|
|
finally:
|
|
session.close()
|
|
engine.dispose()
|