Resolve unknown recovery outcomes from evidence

This commit is contained in:
2026-08-03 05:00:08 +02:00
parent 5bfbe9a887
commit 7e59a7f2b3
3 changed files with 117 additions and 0 deletions
+8
View File
@@ -26,3 +26,11 @@ Evidence and metadata may contain opaque references, digests, counts, and
provider result codes. They must never contain credentials or resolved secrets. provider result codes. They must never contain credentials or resolved secrets.
Ops is the platform surface for unresolved operation status; owning modules must Ops is the platform surface for unresolved operation status; owning modules must
provide the reconciliation action and business-level explanation. provide the reconciliation action and business-level explanation.
An owning module may reconcile an `outcome_unknown` provider effect through the
claimed durable handle's `resolve_unknown` method. External evidence that the
effect occurred records verified success. Evidence that it did not occur moves
the operation through recovery-required and recovering to verified recovered,
so any later attempt must use a new deliberate idempotency key. The method does
not infer provider state and requires the same terminal verification structure
and hash-chain checks as ordinary completion.
@@ -219,6 +219,69 @@ class DurableRecoveryOperation:
session.commit() session.commit()
self.closed = True self.closed = True
def resolve_unknown(
self,
*,
effect_occurred: bool,
evidence: dict[str, Any],
summary: str,
) -> None:
"""Resolve an externally verified operation with an unknown outcome.
A confirmed provider effect is a verified success. A confirmed absence
of the effect is recorded as forward recovery: the declared invariant
is restored and the original effect may be attempted again under a new
idempotency key.
"""
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
raise RecoveryOperationStateConflict(operation.id, operation.status)
if effect_occurred:
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.SUCCEEDED,
kind="unknown-outcome-verified-success",
summary=summary,
evidence=evidence,
lease_claim=claim,
)
else:
operation = transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERY_REQUIRED,
kind="unknown-outcome-recovery-required",
summary=summary,
evidence=evidence,
failure_summary="The external effect was verified absent",
lease_claim=claim,
)
operation = transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERING,
kind="unknown-outcome-recovery-started",
summary="Recording the verified absence of the external effect",
evidence={"effect_occurred": False},
lease_claim=claim,
)
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERED,
kind="unknown-outcome-verified-absent",
summary=summary,
evidence=evidence,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def release_unresolved(self) -> None: def release_unresolved(self) -> None:
"""Release authority after a process-local exception. """Release authority after a process-local exception.
+46
View File
@@ -203,3 +203,49 @@ def test_tampered_checkpoint_blocks_verified_success() -> None:
) )
finally: finally:
engine.dispose() engine.dispose()
@pytest.mark.parametrize(
("effect_occurred", "expected_status"),
[
(True, RecoveryStatus.SUCCEEDED.value),
(False, RecoveryStatus.RECOVERED.value),
],
)
def test_unknown_provider_outcome_can_be_resolved_from_external_evidence(
effect_occurred: bool,
expected_status: str,
) -> None:
engine, factory = _fixture()
try:
started = _start(factory, _identity("worker-1", "incarnation-1"))
assert started.operation is not None
started.operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary="Provider outcome is unknown",
evidence={"effect_started": True},
failure_summary="Inspect the provider before retrying",
)
recovery = claim_durable_recovery_operation(
factory,
identity=_identity("worker-2", "incarnation-2"),
operation_id=started.operation_id,
)
recovery.resolve_unknown(
effect_occurred=effect_occurred,
summary="Operator verified the provider outcome",
evidence={
"verified": True,
"checks": {"provider_evidence": "case-1"},
"effect_occurred": effect_occurred,
"reference": "case-1",
},
)
with factory() as session:
operation = session.get(RecoveryOperation, started.operation_id)
assert operation is not None
assert operation.status == expected_status
assert verify_recovery_evidence_chain(session, operation.id)
finally:
engine.dispose()