Implement durable module recovery operations
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryOperation,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
plan_recovery_operation,
|
||||
prepare_recovery_operation,
|
||||
record_recovery_checkpoint,
|
||||
start_recovery_operation,
|
||||
transition_recovery_operation,
|
||||
verify_recovery_evidence_chain,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
LeaseClaim,
|
||||
RuntimeIdentity,
|
||||
acquire_lease,
|
||||
release_lease,
|
||||
renew_lease,
|
||||
)
|
||||
|
||||
|
||||
SessionFactory = Callable[[], Session]
|
||||
|
||||
|
||||
class RecoveryOperationBusy(RecoveryGuaranteeError):
|
||||
pass
|
||||
|
||||
|
||||
class RecoveryOperationStateConflict(RecoveryGuaranteeError):
|
||||
def __init__(self, operation_id: str, status: str) -> None:
|
||||
self.operation_id = operation_id
|
||||
self.status = status
|
||||
super().__init__(
|
||||
f"Recovery operation {operation_id} is already {status}; "
|
||||
"reconcile it before starting another effect"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DurableRecoveryStart:
|
||||
operation_id: str
|
||||
status: str
|
||||
replayed: bool
|
||||
operation: DurableRecoveryOperation | None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DurableRecoveryOperation:
|
||||
"""Append checkpoints in independent, committed transactions.
|
||||
|
||||
The caller's business transaction may roll back without erasing evidence
|
||||
that an object, queue, filesystem, or provider effect already occurred.
|
||||
"""
|
||||
|
||||
session_factory: SessionFactory
|
||||
operation_id: str
|
||||
lease_claim: LeaseClaim
|
||||
lease_ttl_seconds: int
|
||||
closed: bool = False
|
||||
|
||||
def checkpoint(
|
||||
self,
|
||||
*,
|
||||
kind: str,
|
||||
summary: str,
|
||||
evidence: dict[str, Any],
|
||||
) -> None:
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
record_recovery_checkpoint(
|
||||
session,
|
||||
operation,
|
||||
kind=kind,
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
session.commit()
|
||||
|
||||
def succeed(self, *, evidence: dict[str, Any]) -> None:
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.SUCCEEDED,
|
||||
kind="verified-success",
|
||||
summary="Operation effects and authoritative state were verified",
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def compensate(
|
||||
self,
|
||||
*,
|
||||
failure_summary: str,
|
||||
failure_evidence: dict[str, Any],
|
||||
recovery_evidence: dict[str, Any],
|
||||
) -> None:
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
if operation.status == RecoveryStatus.RUNNING.value:
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="compensation-required",
|
||||
summary="The started operation requires explicit compensation",
|
||||
evidence=failure_evidence,
|
||||
failure_summary=failure_summary,
|
||||
lease_claim=claim,
|
||||
)
|
||||
if operation.status == RecoveryStatus.RECOVERY_REQUIRED.value:
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERING,
|
||||
kind="compensation-started",
|
||||
summary="Compensation started",
|
||||
evidence={"failure_summary": failure_summary},
|
||||
lease_claim=claim,
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERED,
|
||||
kind="compensation-verified",
|
||||
summary="Compensation restored the declared invariant",
|
||||
evidence=recovery_evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def unresolved(
|
||||
self,
|
||||
*,
|
||||
status: RecoveryStatus,
|
||||
summary: str,
|
||||
evidence: dict[str, Any],
|
||||
failure_summary: str,
|
||||
) -> None:
|
||||
if status not in {
|
||||
RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
RecoveryStatus.RECOVERY_REQUIRED,
|
||||
}:
|
||||
raise ValueError("Unresolved operations require an unresolved status")
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=status,
|
||||
kind="unresolved-effect",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
failure_summary=failure_summary,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def release_unresolved(self) -> None:
|
||||
"""Release authority after a process-local exception.
|
||||
|
||||
This does not alter the operation state. A later recovery claim treats a
|
||||
stale `running` operation according to its declared recovery mode.
|
||||
"""
|
||||
|
||||
if self.closed:
|
||||
return
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
record_recovery_checkpoint(
|
||||
session,
|
||||
operation,
|
||||
kind="authority-released",
|
||||
summary="Execution authority was released without a terminal claim",
|
||||
evidence={"status": operation.status},
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def _locked_and_renewed(
|
||||
self,
|
||||
session: Session,
|
||||
) -> tuple[RecoveryOperation, LeaseClaim]:
|
||||
if self.closed:
|
||||
raise RecoveryGuaranteeError("Recovery operation handle is closed")
|
||||
claim = renew_lease(
|
||||
session,
|
||||
self.lease_claim,
|
||||
ttl_seconds=self.lease_ttl_seconds,
|
||||
)
|
||||
operation = session.execute(
|
||||
select(RecoveryOperation)
|
||||
.where(RecoveryOperation.id == self.operation_id)
|
||||
.with_for_update()
|
||||
).scalar_one()
|
||||
self.lease_claim = claim
|
||||
return operation, claim
|
||||
|
||||
def _verify_chain(self, session: Session) -> None:
|
||||
if not verify_recovery_evidence_chain(session, self.operation_id):
|
||||
raise RecoveryGuaranteeError(
|
||||
"Recovery checkpoint chain verification failed"
|
||||
)
|
||||
|
||||
|
||||
def begin_durable_recovery_operation(
|
||||
session_factory: SessionFactory,
|
||||
*,
|
||||
identity: RuntimeIdentity,
|
||||
module_id: str,
|
||||
operation_type: str,
|
||||
idempotency_key: str,
|
||||
request: dict[str, Any],
|
||||
recovery_plan: RecoveryPlan,
|
||||
precondition_evidence: dict[str, Any],
|
||||
lease_resource_key: str,
|
||||
lease_ttl_seconds: int = 300,
|
||||
resource_type: str | None = None,
|
||||
resource_id: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> DurableRecoveryStart:
|
||||
if lease_ttl_seconds < 1:
|
||||
raise ValueError("Recovery lease TTL must be at least one second")
|
||||
with session_factory() as session:
|
||||
claim = acquire_lease(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
resource_key=lease_resource_key,
|
||||
holder_node_id=identity.node_id,
|
||||
holder_incarnation=identity.incarnation,
|
||||
ttl_seconds=lease_ttl_seconds,
|
||||
metadata={
|
||||
"module_id": module_id,
|
||||
"operation_type": operation_type,
|
||||
},
|
||||
)
|
||||
if claim is None:
|
||||
raise RecoveryOperationBusy(
|
||||
f"Another runtime owns the recovery fence for {lease_resource_key}"
|
||||
)
|
||||
existing = session.execute(
|
||||
select(RecoveryOperation).where(
|
||||
RecoveryOperation.installation_id == identity.installation_id,
|
||||
RecoveryOperation.module_id == module_id,
|
||||
RecoveryOperation.idempotency_key == idempotency_key,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
module_id=module_id,
|
||||
operation_type=operation_type,
|
||||
idempotency_key=idempotency_key,
|
||||
request=request,
|
||||
recovery_plan=recovery_plan,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
lease_claim=claim,
|
||||
metadata=metadata,
|
||||
)
|
||||
if existing is not None:
|
||||
if operation.status == RecoveryStatus.SUCCEEDED.value:
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
return DurableRecoveryStart(
|
||||
operation_id=operation.id,
|
||||
status=operation.status,
|
||||
replayed=True,
|
||||
operation=None,
|
||||
)
|
||||
session.rollback()
|
||||
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||
prepare_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence=precondition_evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
start_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence={"lease_resource_key": lease_resource_key},
|
||||
lease_claim=claim,
|
||||
)
|
||||
if not verify_recovery_evidence_chain(session, operation.id):
|
||||
raise RecoveryGuaranteeError(
|
||||
"Recovery checkpoint chain verification failed before side effects"
|
||||
)
|
||||
session.commit()
|
||||
return DurableRecoveryStart(
|
||||
operation_id=operation.id,
|
||||
status=operation.status,
|
||||
replayed=False,
|
||||
operation=DurableRecoveryOperation(
|
||||
session_factory=session_factory,
|
||||
operation_id=operation.id,
|
||||
lease_claim=claim,
|
||||
lease_ttl_seconds=lease_ttl_seconds,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def claim_durable_recovery_operation(
|
||||
session_factory: SessionFactory,
|
||||
*,
|
||||
identity: RuntimeIdentity,
|
||||
operation_id: str,
|
||||
lease_ttl_seconds: int = 300,
|
||||
) -> DurableRecoveryOperation:
|
||||
with session_factory() as session:
|
||||
candidate = session.get(RecoveryOperation, operation_id)
|
||||
if candidate is None:
|
||||
raise RecoveryGuaranteeError("Recovery operation was not found")
|
||||
if candidate.status in {
|
||||
RecoveryStatus.SUCCEEDED.value,
|
||||
RecoveryStatus.FAILED.value,
|
||||
RecoveryStatus.RECOVERED.value,
|
||||
RecoveryStatus.MANUAL_INTERVENTION.value,
|
||||
}:
|
||||
raise RecoveryOperationStateConflict(candidate.id, candidate.status)
|
||||
if not candidate.lease_resource_key:
|
||||
raise RecoveryGuaranteeError(
|
||||
"Recovery takeover requires an operation-bound lease resource"
|
||||
)
|
||||
claim = acquire_lease(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
resource_key=candidate.lease_resource_key,
|
||||
holder_node_id=identity.node_id,
|
||||
holder_incarnation=identity.incarnation,
|
||||
ttl_seconds=lease_ttl_seconds,
|
||||
metadata={"recovery_operation_id": candidate.id},
|
||||
)
|
||||
if claim is None:
|
||||
raise RecoveryOperationBusy(
|
||||
f"Another runtime owns recovery operation {candidate.id}"
|
||||
)
|
||||
operation = session.execute(
|
||||
select(RecoveryOperation)
|
||||
.where(RecoveryOperation.id == operation_id)
|
||||
.with_for_update()
|
||||
).scalar_one()
|
||||
previous_fence = {
|
||||
"holder_node_id": operation.holder_node_id,
|
||||
"holder_incarnation": operation.holder_incarnation,
|
||||
"fence_number": operation.fencing_token,
|
||||
}
|
||||
operation.holder_node_id = claim.holder_node_id
|
||||
operation.holder_incarnation = claim.holder_incarnation
|
||||
operation.fencing_token = claim.fencing_token
|
||||
session.add(operation)
|
||||
record_recovery_checkpoint(
|
||||
session,
|
||||
operation,
|
||||
kind="fence-takeover",
|
||||
summary="A new runtime claimed explicit recovery authority",
|
||||
evidence={
|
||||
"previous_fence": previous_fence,
|
||||
"new_fence_number": claim.fencing_token,
|
||||
},
|
||||
lease_claim=claim,
|
||||
)
|
||||
if operation.status == RecoveryStatus.RUNNING.value:
|
||||
mode = RecoveryMode(operation.mode)
|
||||
if mode == RecoveryMode.ATOMIC:
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.FAILED,
|
||||
kind="stale-atomic-operation",
|
||||
summary="The stale database-only transaction rolled back",
|
||||
evidence={"previous_fence": previous_fence},
|
||||
failure_summary="Execution authority expired before commit",
|
||||
lease_claim=claim,
|
||||
)
|
||||
elif mode in {RecoveryMode.COMPENSATION, RecoveryMode.SNAPSHOT_RESTORE}:
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="stale-effect-requires-recovery",
|
||||
summary="Execution authority expired after effects may have started",
|
||||
evidence={"previous_fence": previous_fence},
|
||||
failure_summary="Execution authority expired during a non-atomic operation",
|
||||
lease_claim=claim,
|
||||
)
|
||||
else:
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
kind="stale-effect-outcome-unknown",
|
||||
summary="Execution authority expired after an external effect may have started",
|
||||
evidence={"previous_fence": previous_fence},
|
||||
failure_summary="External effect outcome requires reconciliation",
|
||||
lease_claim=claim,
|
||||
)
|
||||
if not verify_recovery_evidence_chain(session, operation.id):
|
||||
raise RecoveryGuaranteeError("Recovery checkpoint chain verification failed")
|
||||
if operation.status == RecoveryStatus.FAILED.value:
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||
session.commit()
|
||||
return DurableRecoveryOperation(
|
||||
session_factory=session_factory,
|
||||
operation_id=operation.id,
|
||||
lease_claim=claim,
|
||||
lease_ttl_seconds=lease_ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DurableRecoveryOperation",
|
||||
"DurableRecoveryStart",
|
||||
"RecoveryOperationBusy",
|
||||
"RecoveryOperationStateConflict",
|
||||
"begin_durable_recovery_operation",
|
||||
"claim_durable_recovery_operation",
|
||||
]
|
||||
Reference in New Issue
Block a user