Extend action recovery reconciliation
This commit is contained in:
@@ -31,6 +31,13 @@ ActionReversibility = Literal[
|
||||
"corrective_only",
|
||||
"irreversible",
|
||||
]
|
||||
ActionRecoveryMode = Literal[
|
||||
"atomic",
|
||||
"compensation",
|
||||
"snapshot_restore",
|
||||
"forward_recovery",
|
||||
"irreversible",
|
||||
]
|
||||
ActionExecutionState = Literal[
|
||||
"pending",
|
||||
"running",
|
||||
@@ -87,6 +94,10 @@ class ActionDefinition:
|
||||
idempotency_strategy: str = "caller_supplied"
|
||||
audit_event_types: tuple[str, ...] = ()
|
||||
preview_required: bool = True
|
||||
recovery_mode: ActionRecoveryMode = "forward_recovery"
|
||||
recovery_verification: tuple[str, ...] = (
|
||||
"verify the provider result and every announced effect before continuation",
|
||||
)
|
||||
contract_version: str = ACTION_EFFECT_CONTRACT_VERSION
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -96,12 +107,24 @@ class ActionDefinition:
|
||||
_require_text(self.description, "Action description")
|
||||
_require_text(self.input_schema_ref, "Action input schema reference")
|
||||
_require_text(self.idempotency_strategy, "Action idempotency strategy")
|
||||
if self.recovery_mode not in {
|
||||
"atomic",
|
||||
"compensation",
|
||||
"snapshot_restore",
|
||||
"forward_recovery",
|
||||
"irreversible",
|
||||
}:
|
||||
raise ValueError("Action recovery mode is not supported")
|
||||
if any(not value.strip() for value in self.required_scopes):
|
||||
raise ValueError("Action scopes must not be empty")
|
||||
if any(not value.strip() for value in self.required_capabilities):
|
||||
raise ValueError("Action capabilities must not be empty")
|
||||
if any(not value.strip() for value in self.expected_effect_keys):
|
||||
raise ValueError("Expected effect keys must not be empty")
|
||||
if not self.recovery_verification or any(
|
||||
not value.strip() for value in self.recovery_verification
|
||||
):
|
||||
raise ValueError("Actions must declare recovery verification steps")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -384,6 +407,7 @@ __all__ = [
|
||||
"AutomationPrincipalResolution",
|
||||
"AutomationSubjectKind",
|
||||
"ActionPreview",
|
||||
"ActionRecoveryMode",
|
||||
"ActionReversibility",
|
||||
"ActionRiskLevel",
|
||||
"EffectDefinition",
|
||||
|
||||
@@ -310,51 +310,40 @@ class DurableRecoveryOperation:
|
||||
"""
|
||||
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
|
||||
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||
if effect_occurred:
|
||||
transition_recovery_operation(
|
||||
try:
|
||||
self._transition_unknown_resolution(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.SUCCEEDED,
|
||||
kind="unknown-outcome-verified-success",
|
||||
summary=summary,
|
||||
effect_occurred=effect_occurred,
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
else:
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="unknown-outcome-recovery-required",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
failure_summary="The external effect was verified absent",
|
||||
lease_claim=claim,
|
||||
)
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERING,
|
||||
kind="unknown-outcome-recovery-started",
|
||||
summary="Recording the verified absence of the external effect",
|
||||
evidence={"effect_occurred": False},
|
||||
lease_claim=claim,
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERED,
|
||||
kind="unknown-outcome-verified-absent",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
self.closed = True
|
||||
|
||||
def commit_unknown_resolution(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
effect_occurred: bool,
|
||||
evidence: dict[str, Any],
|
||||
summary: str,
|
||||
) -> None:
|
||||
"""Commit an operator reconciliation and its domain projection together."""
|
||||
|
||||
try:
|
||||
self._transition_unknown_resolution(
|
||||
session,
|
||||
effect_occurred=effect_occurred,
|
||||
evidence=evidence,
|
||||
summary=summary,
|
||||
)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
self.closed = True
|
||||
|
||||
def release_unresolved(self) -> None:
|
||||
@@ -396,6 +385,7 @@ class DurableRecoveryOperation:
|
||||
select(RecoveryOperation)
|
||||
.where(RecoveryOperation.id == self.operation_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one()
|
||||
self.lease_claim = claim
|
||||
return operation, claim
|
||||
@@ -447,6 +437,59 @@ class DurableRecoveryOperation:
|
||||
raise
|
||||
self.closed = True
|
||||
|
||||
def _transition_unknown_resolution(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
effect_occurred: bool,
|
||||
evidence: dict[str, Any],
|
||||
summary: str,
|
||||
) -> None:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
|
||||
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||
if effect_occurred:
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.SUCCEEDED,
|
||||
kind="unknown-outcome-verified-success",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
else:
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
kind="unknown-outcome-recovery-required",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
failure_summary="The external effect was verified absent",
|
||||
lease_claim=claim,
|
||||
)
|
||||
operation = transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERING,
|
||||
kind="unknown-outcome-recovery-started",
|
||||
summary="Recording the verified absence of the external effect",
|
||||
evidence={"effect_occurred": False},
|
||||
lease_claim=claim,
|
||||
)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.RECOVERED,
|
||||
kind="unknown-outcome-verified-absent",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
|
||||
def _verify_chain(self, session: Session) -> None:
|
||||
if not verify_recovery_evidence_chain(session, self.operation_id):
|
||||
raise RecoveryGuaranteeError(
|
||||
|
||||
Reference in New Issue
Block a user