govoplan-core/tests/test_recovery_runtime.py

from __future__ import annotations

from datetime import datetime, timedelta, timezone

from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
import pytest

from govoplan_core.core.recovery import (
    RecoveryCheckpoint,
    RecoveryGuaranteeError,
    RecoveryMode,
    RecoveryOperation,
    RecoveryPlan,
    RecoveryStatus,
    verify_recovery_evidence_chain,
)
from govoplan_core.core.recovery_runtime import (
    RecoveryOperationBusy,
    RecoveryOperationStateConflict,
    begin_durable_recovery_operation,
    claim_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import DistributedLease, RuntimeIdentity
from govoplan_core.db.base import Base


def _fixture():
    engine = create_engine("sqlite+pysqlite:///:memory:")
    Base.metadata.create_all(
        engine,
        tables=[
            DistributedLease.__table__,
            RecoveryOperation.__table__,
            RecoveryCheckpoint.__table__,
        ],
    )
    return engine, sessionmaker(bind=engine, expire_on_commit=False)


def _identity(node: str, incarnation: str) -> RuntimeIdentity:
    return RuntimeIdentity(
        installation_id="installation-1",
        node_id=node,
        incarnation=incarnation,
        role="worker",
        software_version="test",
        composition_hash="a" * 64,
    )


def _start(factory, identity, *, key: str = "build-1"):
    return begin_durable_recovery_operation(
        factory,
        identity=identity,
        module_id="campaigns",
        operation_type="build-artifacts",
        idempotency_key=key,
        request={"version_id": "version-1", "write_eml": True},
        recovery_plan=RecoveryPlan(
            mode=RecoveryMode.COMPENSATION,
            preconditions=("validated version is locked",),
            compensation_steps=("delete build object prefix",),
            verification_steps=("compare database and object manifests",),
        ),
        precondition_evidence={"validation_sha256": "b" * 64},
        lease_resource_key="campaign:build:version-1",
        resource_type="campaign_version",
        resource_id="version-1",
    )


def test_durable_operation_commits_before_caller_effect_and_replays_success() -> None:
    engine, factory = _fixture()
    try:
        started = _start(factory, _identity("worker-1", "incarnation-1"))
        assert started.operation is not None
        with factory() as session:
            persisted = session.get(RecoveryOperation, started.operation_id)
            assert persisted is not None
            assert persisted.status == RecoveryStatus.RUNNING.value
            assert persisted.checkpoint_count == 3

        started.operation.checkpoint(
            kind="object-prefix-reserved",
            summary="Build object prefix reserved",
            evidence={"prefix": "campaign-artifacts/build-1/"},
        )
        started.operation.succeed(
            evidence={
                "verified": True,
                "checks": {"database_manifest": "matched", "object_manifest": "matched"},
            }
        )

        replay = _start(factory, _identity("worker-2", "incarnation-2"))
        assert replay.replayed is True
        assert replay.operation is None
        with factory() as session:
            assert verify_recovery_evidence_chain(session, started.operation_id)
    finally:
        engine.dispose()


def test_same_fence_cannot_start_duplicate_running_operation() -> None:
    engine, factory = _fixture()
    try:
        identity = _identity("worker-1", "incarnation-1")
        started = _start(factory, identity)
        with pytest.raises(RecoveryOperationStateConflict, match="already running"):
            _start(factory, identity)
        assert started.operation is not None
        started.operation.release_unresolved()
    finally:
        engine.dispose()


def test_verified_provider_rejection_is_terminal_without_recovery() -> None:
    engine, factory = _fixture()
    try:
        started = _start(factory, _identity("worker-1", "incarnation-1"))
        assert started.operation is not None
        started.operation.reject(
            summary="Provider definitively rejected the request",
            evidence={
                "verified": True,
                "provider_outcome": "rejected",
                "checks": {"provider_response": "definitive-rejection"},
            },
        )
        with factory() as session:
            operation = session.get(RecoveryOperation, started.operation_id)
            assert operation is not None
            assert operation.status == RecoveryStatus.REJECTED.value
            assert operation.completed_at is not None
            assert verify_recovery_evidence_chain(session, operation.id)
    finally:
        engine.dispose()


def test_other_runtime_cannot_use_an_active_fence() -> None:
    engine, factory = _fixture()
    try:
        started = _start(factory, _identity("worker-1", "incarnation-1"))
        with pytest.raises(RecoveryOperationBusy):
            _start(factory, _identity("worker-2", "incarnation-2"), key="build-2")
        assert started.operation is not None
        started.operation.release_unresolved()
    finally:
        engine.dispose()


def test_expired_crash_fence_is_taken_over_as_recovery_required() -> None:
    engine, factory = _fixture()
    try:
        started = _start(factory, _identity("worker-1", "incarnation-1"))
        assert started.operation is not None
        with factory() as session:
            lease = session.execute(select(DistributedLease)).scalar_one()
            lease.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
            session.add(lease)
            session.commit()

        recovery = claim_durable_recovery_operation(
            factory,
            identity=_identity("worker-2", "incarnation-2"),
            operation_id=started.operation_id,
        )
        with factory() as session:
            operation = session.get(RecoveryOperation, started.operation_id)
            assert operation is not None
            assert operation.status == RecoveryStatus.RECOVERY_REQUIRED.value
            assert operation.fencing_token == 2
            assert verify_recovery_evidence_chain(session, operation.id)
        recovery.compensate(
            failure_summary="worker stopped during object publication",
            failure_evidence={"object_prefix": "campaign-artifacts/build-1/"},
            recovery_evidence={
                "verified": True,
                "checks": {"object_prefix_empty": True},
            },
        )
    finally:
        engine.dispose()


def test_tampered_checkpoint_blocks_verified_success() -> None:
    engine, factory = _fixture()
    try:
        started = _start(factory, _identity("worker-1", "incarnation-1"))
        assert started.operation is not None
        with factory() as session:
            checkpoint = session.execute(
                select(RecoveryCheckpoint).order_by(RecoveryCheckpoint.sequence)
            ).scalars().first()
            assert checkpoint is not None
            checkpoint.summary = "tampered"
            session.add(checkpoint)
            session.commit()
        with pytest.raises(RecoveryGuaranteeError, match="chain verification failed"):
            started.operation.succeed(
                evidence={"verified": True, "checks": {"objects": "matched"}}
            )
    finally:
        engine.dispose()