Fence and reconcile workflow runtime effects

src/govoplan_workflow_engine/backend/instance_service.py

@@ -65,6 +65,18 @@ from govoplan_workflow_engine.backend.service import (
     get_definition,
     get_definition_revision,
 )
+from govoplan_workflow_engine.backend.recovery import (
+    WorkflowActionRecovery,
+    WorkflowRecoveryBusy,
+    WorkflowRecoveryConflict,
+    WorkflowRecoveryError,
+    acquire_workflow_state_fence,
+    begin_workflow_action_recovery,
+    canonical_sha256,
+    claim_workflow_action_recovery,
+    reconcile_stale_workflow_action_recovery,
+    release_workflow_state_fence,
+)
 
 
 INSTANCE_START_SCOPE = "workflow:instance:start"
@@ -355,6 +367,54 @@ def reconcile_instance(
             message="The linked Dataflow run no longer exists.",
         )
         return True
+    descriptor_recovery = descriptor.metadata.get("recovery")
+    recovery_requires_attention = bool(
+        isinstance(descriptor_recovery, Mapping)
+        and descriptor_recovery.get("requires_attention")
+    )
+    if descriptor.status == "outcome_unknown" or recovery_requires_attention:
+        previous_state = str(step.handoff.get("state") or "")
+        step.status = "waiting"
+        step.error = (
+            "The Dataflow output outcome is unresolved. Reconcile the run "
+            "before this Workflow can continue."
+        )
+        step.handoff = {
+            "kind": "dataflow_recovery",
+            "state": "outcome_unknown",
+            "run_ref": descriptor.ref,
+            "pipeline_ref": descriptor.pipeline_ref,
+            "action_url": _dataflow_action_url(
+                descriptor.pipeline_ref,
+                descriptor.ref,
+            ),
+            "allowed_actions": ["cancel"],
+            "outcome_unknown": True,
+            "recovery": (
+                dict(descriptor_recovery)
+                if isinstance(descriptor_recovery, Mapping)
+                else {"requires_attention": True}
+            ),
+        }
+        instance.status = "waiting"
+        instance.error = step.error
+        if previous_state != "outcome_unknown":
+            _record_event(
+                session,
+                instance,
+                step=step,
+                kind="workflow.dataflow.outcome_unknown",
+                actor_id=actor_id,
+                payload=dict(step.handoff),
+            )
+            _notify_handoff(
+                session,
+                registry=registry,
+                instance=instance,
+                step=step,
+                subject="Workflow Dataflow outcome requires reconciliation",
+            )
+        return False
     if descriptor.status in {"queued", "retrying", "running"}:
         step.handoff = {
             **dict(step.handoff),
@@ -466,6 +526,113 @@ def resolve_step(
             principal=principal,
             registry=registry,
         )
+    if payload.action in {"confirm_effect", "confirm_absent"}:
+        recovery_payload = step.handoff.get("recovery")
+        if not isinstance(recovery_payload, Mapping):
+            raise WorkflowConflictError(
+                "This handoff has no durable recovery operation to reconcile."
+            )
+        operation_id = str(recovery_payload.get("operation_id") or "").strip()
+        if not operation_id:
+            raise WorkflowConflictError(
+                "This handoff has no durable recovery operation to reconcile."
+            )
+        if not payload.evidence and not payload.comment:
+            raise WorkflowConflictError(
+                "Record an evidence reference or operator comment before "
+                "resolving an unknown provider outcome."
+            )
+        try:
+            recovery = claim_workflow_action_recovery(
+                session,
+                operation_id=operation_id,
+            )
+        except WorkflowRecoveryError as exc:
+            raise WorkflowConflictError(str(exc)) from exc
+        effect_occurred = payload.action == "confirm_effect"
+        evidence = {
+            "verified": True,
+            "checks": {
+                "operator_actor_ref": actor_id,
+                "evidence_refs_sha256": canonical_sha256(list(payload.evidence)),
+                "operator_output_sha256": canonical_sha256(dict(payload.output)),
+                "operator_comment_sha256": canonical_sha256(payload.comment or ""),
+                "effect_occurred": effect_occurred,
+            },
+        }
+        if not effect_occurred:
+            previous_recovery = dict(recovery_payload)
+            call_number = max(1, int(previous_recovery.get("call_number") or 1))
+            step.status = "waiting"
+            step.error = None
+            step.handoff = {
+                **dict(step.handoff),
+                "state": "retryable",
+                "message": (
+                    "The provider effect was verified absent. A deliberate retry "
+                    "is now safe."
+                ),
+                "allowed_actions": ["retry", "cancel"],
+                "outcome_unknown": False,
+                "recovery": {
+                    **previous_recovery,
+                    "status": "recovered",
+                    "requires_attention": False,
+                    "next_call_number": call_number + 1,
+                },
+            }
+            instance.status = "waiting"
+            instance.error = None
+            _record_event(
+                session,
+                instance,
+                step=step,
+                kind="workflow.action.effect_absent",
+                actor_id=actor_id,
+                payload={
+                    "recovery_operation_id": operation_id,
+                    "evidence": list(payload.evidence),
+                },
+            )
+            recovery.commit_unknown_resolution(
+                session,
+                effect_occurred=False,
+                evidence=evidence,
+                summary="Operator verified that the provider effect is absent",
+            )
+            return instance
+        output = {
+            **dict(step.output_),
+            **dict(payload.output),
+            "recovery_decision": "effect_confirmed",
+            "recovery_evidence": list(payload.evidence),
+            "recovery_comment": payload.comment,
+        }
+        next_node_id = _complete_step(
+            session,
+            instance=instance,
+            step=step,
+            graph=graph,
+            port="success",
+            output=output,
+            actor_id=actor_id,
+        )
+        recovery.commit_unknown_resolution(
+            session,
+            effect_occurred=True,
+            evidence=evidence,
+            summary="Operator verified that the provider effect occurred",
+        )
+        _drive_instance(
+            session,
+            instance=instance,
+            graph=graph,
+            next_node_id=next_node_id,
+            principal=principal,
+            registry=registry,
+            actor_id=actor_id,
+        )
+        return instance
     if payload.action == "changes":
         step.handoff = {
             **dict(step.handoff),
@@ -634,12 +801,29 @@ def reconcile_pending_instances(
         "skipped": 0,
     }
     for instance in instances:
+        try:
+            fence = acquire_workflow_state_fence(
+                session,
+                resource_key=f"workflow:instance:{instance.id}",
+            )
+        except RuntimeError:
+            logger.warning(
+                "Workflow instance %s could not acquire a runtime fence",
+                instance.id,
+                exc_info=True,
+            )
+            summary["skipped"] = int(summary["skipped"]) + 1
+            continue
+        if fence is None:
+            summary["skipped"] = int(summary["skipped"]) + 1
+            continue
         step = _current_step(session, instance)
         if step is None or step.node_type not in {
             "workflow.capability",
             "workflow.dataflow",
         }:
             summary["waiting"] = int(summary["waiting"]) + 1
+            release_workflow_state_fence(session, fence)
             continue
         principal = _resolve_instance_principal(
             session,
@@ -648,6 +832,7 @@ def reconcile_pending_instances(
         )
         if principal is None:
             summary["skipped"] = int(summary["skipped"]) + 1
+            release_workflow_state_fence(session, fence)
             continue
         changed = reconcile_instance(
             session,
@@ -661,6 +846,7 @@ def reconcile_pending_instances(
             summary["advanced"] = int(summary["advanced"]) + 1
         else:
             summary["waiting"] = int(summary["waiting"]) + 1
+        release_workflow_state_fence(session, fence)
     session.flush()
     return summary
 
@@ -1075,6 +1261,141 @@ def _execute_capability_step(
         preview_ref=preview.preview_ref,
         metadata=request.metadata,
     )
+    capability_name = str(node.config.get("capability") or "")
+    revision = session.get(
+        WorkflowDefinitionRevision,
+        instance.definition_revision_id,
+    )
+    if revision is None:
+        _set_action_handoff(
+            session,
+            instance=instance,
+            step=step,
+            state="blocked",
+            message="The pinned Workflow revision is unavailable.",
+            action_key=definition.action_key,
+            capability_name=capability_name,
+            registry=registry,
+        )
+        return True
+    try:
+        action_recovery = begin_workflow_action_recovery(
+            session,
+            instance=instance,
+            step=step,
+            revision=revision,
+            definition=definition,
+            capability_name=capability_name,
+            request_idempotency_key=request.idempotency_key,
+            action_input=action_input,
+            preview_payload=preview_payload,
+            backup_reference=(
+                str(node.config.get("recovery_backup_reference") or "").strip()
+                or None
+            ),
+            approval_reference=(
+                str(node.config.get("recovery_approval_reference") or "").strip()
+                or None
+            ),
+        )
+    except WorkflowRecoveryBusy as exc:
+        _set_action_handoff(
+            session,
+            instance=instance,
+            step=step,
+            state="running",
+            message=str(exc),
+            action_key=definition.action_key,
+            capability_name=capability_name,
+            registry=registry,
+        )
+        return False
+    except WorkflowRecoveryConflict as exc:
+        recovery_status = exc.status
+        if recovery_status == "running":
+            try:
+                recovery_status = reconcile_stale_workflow_action_recovery(
+                    session,
+                    operation_id=exc.operation_id,
+                )
+            except WorkflowRecoveryBusy:
+                recovery_status = "running"
+        outcome_unknown = recovery_status in {
+            "outcome_unknown",
+            "recovery_required",
+            "manual_intervention",
+        }
+        safe_to_retry = recovery_status in {"failed", "recovered", "rejected"}
+        previous_recovery = (
+            dict(step.handoff.get("recovery"))
+            if isinstance(step.handoff.get("recovery"), Mapping)
+            else {}
+        )
+        call_number = max(1, int(previous_recovery.get("call_number") or 1))
+        _set_action_handoff(
+            session,
+            instance=instance,
+            step=step,
+            state=(
+                "outcome_unknown"
+                if outcome_unknown
+                else "retryable"
+                if safe_to_retry
+                else recovery_status
+            ),
+            message=(
+                "The provider outcome must be reconciled before this Workflow "
+                "can continue."
+                if outcome_unknown
+                else (
+                    "The stale action was proven not to have committed. A "
+                    "deliberate retry is now safe."
+                    if safe_to_retry
+                    else str(exc)
+                )
+            ),
+            action_key=definition.action_key,
+            capability_name=capability_name,
+            registry=registry,
+            details={
+                "outcome_unknown": outcome_unknown,
+                "recovery": {
+                    **previous_recovery,
+                    "operation_id": exc.operation_id,
+                    "status": recovery_status,
+                    "requires_attention": outcome_unknown,
+                    **(
+                        {"next_call_number": call_number + 1}
+                        if safe_to_retry
+                        else {}
+                    ),
+                },
+            },
+        )
+        return True
+    except WorkflowRecoveryError as exc:
+        _set_action_handoff(
+            session,
+            instance=instance,
+            step=step,
+            state="blocked",
+            message=str(exc),
+            action_key=definition.action_key,
+            capability_name=capability_name,
+            registry=registry,
+            details={"recovery_unavailable": True},
+        )
+        return True
+    if action_recovery.replayed:
+        session.expire_all()
+        return True
+    action_recovery.checkpoint_dispatch(
+        session,
+        instance=instance,
+        step=step,
+        action_key=definition.action_key,
+        capability_name=capability_name,
+    )
     try:
         result = provider.execute_action(
             session,
@@ -1087,38 +1408,85 @@ def _execute_capability_step(
             instance.id,
             step.id,
         )
+        session.rollback()
+        if action_recovery.mode.value == "atomic":
+            _set_action_handoff(
+                session,
+                instance=instance,
+                step=step,
+                state="retryable",
+                message=(
+                    "The atomic module action failed and its database changes "
+                    "were rolled back."
+                ),
+                action_key=definition.action_key,
+                capability_name=capability_name,
+                registry=registry,
+                details={
+                    "preview": preview_payload,
+                    "error_type": type(exc).__name__,
+                    "recovery": {
+                        "operation_id": action_recovery.operation_id,
+                        "mode": action_recovery.mode.value,
+                        "status": "failed",
+                        "call_number": action_recovery.call_number,
+                        "next_call_number": action_recovery.call_number + 1,
+                        "requires_attention": False,
+                    },
+                },
+            )
+            action_recovery.commit_definitive_failure(
+                session,
+                summary="The atomic provider call failed before commit",
+                error_type=type(exc).__name__,
+            )
+            return True
         _set_action_handoff(
             session,
             instance=instance,
             step=step,
-            state="quarantined",
+            state="outcome_unknown",
             message=(
                 "The module action outcome is unknown. Inspect the provider "
-                "before retrying to avoid a duplicate effect."
+                "and record evidence before continuing or retrying."
             ),
             action_key=definition.action_key,
-            capability_name=str(node.config.get("capability") or ""),
+            capability_name=capability_name,
             registry=registry,
             details={
                 "preview": preview_payload,
                 "error_type": type(exc).__name__,
                 "outcome_unknown": True,
+                "recovery": {
+                    "operation_id": action_recovery.operation_id,
+                    "mode": action_recovery.mode.value,
+                    "status": "outcome_unknown",
+                    "call_number": action_recovery.call_number,
+                    "requires_attention": True,
+                },
             },
         )
+        action_recovery.commit_unknown(
+            session,
+            error_type=type(exc).__name__,
+            message=(
+                "Inspect the provider by stable idempotency key before any retry"
+            ),
+        )
         return True
     if not isinstance(result, ActionExecutionResult):
-        _set_action_handoff(
+        return _handle_invalid_action_result(
             session,
             instance=instance,
             step=step,
-            state="quarantined",
             message="The action provider returned an invalid execution result.",
-            action_key=definition.action_key,
-            capability_name=str(node.config.get("capability") or ""),
+            definition=definition,
+            capability_name=capability_name,
             registry=registry,
-            details={"preview": preview_payload},
+            preview_payload=preview_payload,
+            action_recovery=action_recovery,
+            error_type="InvalidActionExecutionResult",
         )
-        return True
     allowed_states = {
         "pending",
         "running",
@@ -1138,29 +1506,34 @@ def _execute_capability_step(
         }
     )
     if result.state not in allowed_states or unknown_effects:
-        _set_action_handoff(
+        return _handle_invalid_action_result(
             session,
             instance=instance,
             step=step,
-            state="quarantined",
             message=(
                 "The action provider returned an unsupported state."
                 if result.state not in allowed_states
                 else "The action provider reported unannounced effects."
             ),
-            action_key=definition.action_key,
-            capability_name=str(node.config.get("capability") or ""),
+            definition=definition,
+            capability_name=capability_name,
             registry=registry,
-            details={
+            preview_payload=preview_payload,
+            action_recovery=action_recovery,
+            error_type=(
+                "UnsupportedActionState"
+                if result.state not in allowed_states
+                else "UnannouncedActionEffect"
+            ),
+            extra_details={
                 "state": result.state,
                 "unknown_effects": unknown_effects,
             },
         )
-        return True
     result_payload = _action_result_payload(result)
     step.output_ = {
         "action_key": definition.action_key,
-        "capability": str(node.config.get("capability") or ""),
+        "capability": capability_name,
         "idempotency_key": request.idempotency_key,
         "preview": preview_payload,
         "execution": result_payload,
@@ -1177,13 +1550,29 @@ def _execute_capability_step(
                 or f"Module action is {result.state.replace('_', ' ')}."
             ),
             action_key=definition.action_key,
-            capability_name=str(node.config.get("capability") or ""),
+            capability_name=capability_name,
             registry=registry,
             details={
                 "preview": preview_payload,
                 "execution": result_payload,
+                "recovery": {
+                    "operation_id": action_recovery.operation_id,
+                    "mode": action_recovery.mode.value,
+                    "status": "succeeded",
+                    "call_number": action_recovery.call_number,
+                    "next_call_number": action_recovery.call_number + 1,
+                    "requires_attention": False,
+                },
             },
         )
+        action_recovery.commit_conclusive_result(
+            session,
+            provider_state=result.state,
+            result_sha256=canonical_sha256(result_payload),
+            observed_effects_sha256=canonical_sha256(
+                result_payload["observed_effects"]
+            ),
+        )
         return True
     _record_event(
         session,
@@ -1193,7 +1582,7 @@ def _execute_capability_step(
         actor_id=actor_id,
         payload={
             "action_key": definition.action_key,
-            "capability": str(node.config.get("capability") or ""),
+            "capability": capability_name,
             "idempotency_key": request.idempotency_key,
             "observed_effects": result_payload["observed_effects"],
             "audit_event_refs": result_payload["audit_event_refs"],
@@ -1211,6 +1600,14 @@ def _execute_capability_step(
         output=dict(step.output_),
         actor_id=actor_id,
     )
+    action_recovery.commit_conclusive_result(
+        session,
+        provider_state=result.state,
+        result_sha256=canonical_sha256(result_payload),
+        observed_effects_sha256=canonical_sha256(
+            result_payload["observed_effects"]
+        ),
+    )
     _drive_instance(
         session,
         instance=instance,
@@ -1399,6 +1796,83 @@ def _action_result_payload(
     }
 
 
+def _handle_invalid_action_result(
+    session: Session,
+    *,
+    instance: WorkflowInstance,
+    step: WorkflowInstanceStep,
+    message: str,
+    definition: ActionDefinition,
+    capability_name: str,
+    registry: object | None,
+    preview_payload: Mapping[str, object],
+    action_recovery: WorkflowActionRecovery,
+    error_type: str,
+    extra_details: Mapping[str, object] | None = None,
+) -> bool:
+    session.rollback()
+    recovery_details = {
+        "operation_id": action_recovery.operation_id,
+        "mode": action_recovery.mode.value,
+        "call_number": action_recovery.call_number,
+    }
+    if action_recovery.mode.value == "atomic":
+        _set_action_handoff(
+            session,
+            instance=instance,
+            step=step,
+            state="retryable",
+            message=f"{message} Atomic database changes were rolled back.",
+            action_key=definition.action_key,
+            capability_name=capability_name,
+            registry=registry,
+            details={
+                "preview": dict(preview_payload),
+                "error_type": error_type,
+                **dict(extra_details or {}),
+                "recovery": {
+                    **recovery_details,
+                    "status": "failed",
+                    "next_call_number": action_recovery.call_number + 1,
+                    "requires_attention": False,
+                },
+            },
+        )
+        action_recovery.commit_definitive_failure(
+            session,
+            summary=message,
+            error_type=error_type,
+        )
+        return True
+    _set_action_handoff(
+        session,
+        instance=instance,
+        step=step,
+        state="outcome_unknown",
+        message=f"{message} Reconcile the provider before retrying.",
+        action_key=definition.action_key,
+        capability_name=capability_name,
+        registry=registry,
+        details={
+            "preview": dict(preview_payload),
+            "error_type": error_type,
+            "outcome_unknown": True,
+            **dict(extra_details or {}),
+            "recovery": {
+                **recovery_details,
+                "status": "outcome_unknown",
+                "requires_attention": True,
+            },
+        },
+    )
+    action_recovery.commit_unknown(
+        session,
+        error_type=error_type,
+        message="Inspect the provider by stable idempotency key before any retry",
+    )
+    return True
+
+
 def _set_action_handoff(
     session: Session,
     *,
@@ -1411,10 +1885,20 @@ def _set_action_handoff(
     registry: object | None,
     details: Mapping[str, object] | None = None,
 ) -> None:
-    allowed_actions = (
-        ["cancel"] if state in {"pending", "running"} else ["retry", "reject", "cancel"]
-    )
     previous = dict(step.handoff)
+    if state in {"pending", "running"}:
+        allowed_actions = ["cancel"]
+    elif state in {"outcome_unknown", "recovery_required"}:
+        allowed_actions = ["confirm_effect", "confirm_absent", "cancel"]
+    elif state == "compensation_required":
+        allowed_actions = ["reject", "cancel"]
+    else:
+        allowed_actions = ["retry", "reject", "cancel"]
+    details_payload = dict(details or {})
+    if "recovery" not in details_payload and isinstance(
+        previous.get("recovery"), Mapping
+    ):
+        details_payload["recovery"] = dict(previous["recovery"])
     step.status = "waiting"
     step.error = message if state not in {"pending", "running"} else None
     step.handoff = {
@@ -1425,7 +1909,7 @@ def _set_action_handoff(
         "capability": capability_name,
         "allowed_actions": allowed_actions,
         "suggested_port": "failure",
-        **dict(details or {}),
+        **details_payload,
     }
     instance.status = "waiting"
     instance.error = step.error