feat: add institutional governance and recovery contracts
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
import pytest
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryCheckpoint,
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryOperation,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
plan_recovery_operation,
|
||||
prepare_recovery_operation,
|
||||
record_recovery_checkpoint,
|
||||
start_recovery_operation,
|
||||
transition_recovery_operation,
|
||||
verify_recovery_evidence_chain,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import DistributedLease, acquire_lease
|
||||
from govoplan_core.db.base import Base
|
||||
|
||||
|
||||
def _session() -> tuple[Session, object]:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(
|
||||
engine,
|
||||
tables=[
|
||||
DistributedLease.__table__,
|
||||
RecoveryOperation.__table__,
|
||||
RecoveryCheckpoint.__table__,
|
||||
],
|
||||
)
|
||||
return Session(engine, expire_on_commit=False), engine
|
||||
|
||||
|
||||
def test_recovery_plan_requires_mode_specific_evidence() -> None:
|
||||
with pytest.raises(RecoveryGuaranteeError, match="compensation steps"):
|
||||
RecoveryPlan(
|
||||
mode=RecoveryMode.COMPENSATION,
|
||||
verification_steps=("verify",),
|
||||
).validate()
|
||||
with pytest.raises(RecoveryGuaranteeError, match="backup reference"):
|
||||
RecoveryPlan(
|
||||
mode=RecoveryMode.SNAPSHOT_RESTORE,
|
||||
verification_steps=("verify",),
|
||||
).validate()
|
||||
with pytest.raises(RecoveryGuaranteeError, match="approval reference"):
|
||||
RecoveryPlan(
|
||||
mode=RecoveryMode.IRREVERSIBLE,
|
||||
verification_steps=("verify",),
|
||||
).validate()
|
||||
|
||||
|
||||
def test_recovery_plan_hashes_backup_and_approval_references() -> None:
|
||||
plan = RecoveryPlan(
|
||||
mode=RecoveryMode.SNAPSHOT_RESTORE,
|
||||
verification_steps=("restore drill passed",),
|
||||
backup_reference="backup://postgres/2026-08-01",
|
||||
approval_reference="approval://change/42",
|
||||
)
|
||||
|
||||
assert plan.as_dict()["backup_reference"] == "backup://postgres/2026-08-01"
|
||||
assert plan.as_dict()["approval_reference"] == "approval://change/42"
|
||||
|
||||
|
||||
def test_recovery_operation_records_verifiable_checkpoint_chain() -> None:
|
||||
session, engine = _session()
|
||||
try:
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
module_id="files",
|
||||
operation_type="move-tree",
|
||||
idempotency_key="move-42",
|
||||
request={"source": "a", "target": "b"},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.COMPENSATION,
|
||||
compensation_steps=("restore source links",),
|
||||
verification_steps=("compare object inventory",),
|
||||
),
|
||||
)
|
||||
prepare_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence={"inventory_sha256": "a" * 64},
|
||||
)
|
||||
start_recovery_operation(session, operation)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="side-effect-failed",
|
||||
summary="Target index update failed",
|
||||
evidence={"completed_steps": 1},
|
||||
failure_summary="target index unavailable",
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERING,
|
||||
kind="compensation-started",
|
||||
summary="Compensation started",
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERED,
|
||||
kind="compensation-verified",
|
||||
summary="Source inventory restored",
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {"object_inventory": "matched"},
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
assert operation.status == RecoveryStatus.RECOVERED.value
|
||||
assert operation.checkpoint_count == 6
|
||||
assert verify_recovery_evidence_chain(session, operation.id) is True
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_recovery_transition_cannot_skip_preparation() -> None:
|
||||
session, engine = _session()
|
||||
try:
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
module_id="core",
|
||||
operation_type="configuration-change",
|
||||
idempotency_key="config-1",
|
||||
request={"revision": 2},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.ATOMIC,
|
||||
verification_steps=("read current revision",),
|
||||
),
|
||||
)
|
||||
with pytest.raises(RecoveryGuaranteeError, match="not allowed"):
|
||||
start_recovery_operation(session, operation)
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_non_atomic_failure_requires_explicit_recovery_and_verified_completion() -> (
|
||||
None
|
||||
):
|
||||
session, engine = _session()
|
||||
try:
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
module_id="campaign",
|
||||
operation_type="artifact-publish",
|
||||
idempotency_key="publish-1",
|
||||
request={"version": "version-1"},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.COMPENSATION,
|
||||
compensation_steps=("delete published objects",),
|
||||
verification_steps=("verify object inventory",),
|
||||
),
|
||||
)
|
||||
prepare_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence={"preconditions": "verified"},
|
||||
)
|
||||
start_recovery_operation(session, operation)
|
||||
|
||||
with pytest.raises(RecoveryGuaranteeError, match="cannot be marked failed"):
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.FAILED,
|
||||
kind="failed",
|
||||
summary="Object publication failed",
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="recovery-required",
|
||||
summary="Published objects require compensation",
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERING,
|
||||
kind="compensation-started",
|
||||
summary="Removing published objects",
|
||||
)
|
||||
with pytest.raises(RecoveryGuaranteeError, match="verified evidence"):
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERED,
|
||||
kind="compensation-finished",
|
||||
summary="Objects removed",
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_recovery_evidence_rejects_plaintext_secrets() -> None:
|
||||
session, engine = _session()
|
||||
try:
|
||||
with pytest.raises(RecoveryGuaranteeError, match="plaintext secrets"):
|
||||
plan_recovery_operation(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
module_id="mail",
|
||||
operation_type="transport-change",
|
||||
idempotency_key="transport-1",
|
||||
request={"transport_id": "smtp-1"},
|
||||
metadata={"password": "do-not-store"},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.ATOMIC,
|
||||
verification_steps=("read persisted transport",),
|
||||
),
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_fenced_recovery_rejects_low_level_mutation_without_claim() -> None:
|
||||
session, engine = _session()
|
||||
try:
|
||||
claim = acquire_lease(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
resource_key="files:move-tree",
|
||||
holder_node_id="worker-1",
|
||||
holder_incarnation="worker-1-incarnation",
|
||||
ttl_seconds=300,
|
||||
now=datetime.now(timezone.utc),
|
||||
)
|
||||
assert claim is not None
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id="installation-1",
|
||||
module_id="files",
|
||||
operation_type="move-tree",
|
||||
idempotency_key="move-fenced-1",
|
||||
request={"source": "a", "target": "b"},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.COMPENSATION,
|
||||
compensation_steps=("restore source links",),
|
||||
verification_steps=("compare object inventory",),
|
||||
),
|
||||
lease_claim=claim,
|
||||
)
|
||||
|
||||
with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"):
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.PREPARED,
|
||||
kind="prepared",
|
||||
summary="Attempted without fence",
|
||||
evidence={"inventory_sha256": "a" * 64},
|
||||
)
|
||||
with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"):
|
||||
record_recovery_checkpoint(
|
||||
session,
|
||||
operation,
|
||||
kind="manual-note",
|
||||
summary="Attempted without fence",
|
||||
evidence={},
|
||||
)
|
||||
|
||||
prepared = prepare_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence={"inventory_sha256": "a" * 64},
|
||||
lease_claim=claim,
|
||||
)
|
||||
assert prepared.status == RecoveryStatus.PREPARED.value
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
Reference in New Issue
Block a user