Fence and reconcile workflow runtime effects
This commit is contained in:
@@ -0,0 +1,535 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Mapping
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from govoplan_core.core.automation import ActionDefinition
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryOperation,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
DurableRecoveryOperation,
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
claim_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
LeaseClaim,
|
||||
acquire_lease,
|
||||
assert_lease_fence,
|
||||
process_runtime_identity,
|
||||
release_lease,
|
||||
)
|
||||
from govoplan_workflow_engine.backend.db.models import (
|
||||
WorkflowDefinitionRevision,
|
||||
WorkflowInstance,
|
||||
WorkflowInstanceStep,
|
||||
)
|
||||
|
||||
|
||||
class WorkflowRecoveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class WorkflowRecoveryBusy(WorkflowRecoveryError):
|
||||
pass
|
||||
|
||||
|
||||
class WorkflowRecoveryConflict(WorkflowRecoveryError):
|
||||
def __init__(self, operation_id: str, status: str) -> None:
|
||||
self.operation_id = operation_id
|
||||
self.status = status
|
||||
super().__init__(
|
||||
f"Workflow action recovery is {status}; reconcile it first"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowRecoveryDeclaration:
|
||||
operation_type: str
|
||||
mode: RecoveryMode | None
|
||||
boundaries: tuple[Mapping[str, object], ...]
|
||||
verification: tuple[str, ...]
|
||||
|
||||
|
||||
WORKFLOW_RECOVERY_OPERATIONS = (
|
||||
WorkflowRecoveryDeclaration(
|
||||
operation_type="instance.state-transition",
|
||||
mode=RecoveryMode.ATOMIC,
|
||||
boundaries=(
|
||||
{
|
||||
"name": "worker-claim",
|
||||
"classification": "fenced",
|
||||
"resume": "a current process identity and distributed fence are required",
|
||||
},
|
||||
{
|
||||
"name": "instance-transition",
|
||||
"classification": "atomic",
|
||||
"resume": "the pinned revision, step, event, and instance state commit together",
|
||||
},
|
||||
),
|
||||
verification=(
|
||||
"verify the pinned definition revision and current step",
|
||||
"verify the process still owns the distributed fence before commit",
|
||||
),
|
||||
),
|
||||
WorkflowRecoveryDeclaration(
|
||||
operation_type="activity.external-effect",
|
||||
mode=None,
|
||||
boundaries=(
|
||||
{
|
||||
"name": "action-preview",
|
||||
"classification": "recomputable",
|
||||
"resume": "revalidate capability, authority, input, and preview",
|
||||
},
|
||||
{
|
||||
"name": "provider-dispatch",
|
||||
"classification": "outcome-unknown",
|
||||
"resume": "verify by stable provider idempotency key before retry",
|
||||
},
|
||||
{
|
||||
"name": "workflow-projection",
|
||||
"classification": "verified",
|
||||
"resume": "commit the local projection with terminal recovery evidence",
|
||||
},
|
||||
),
|
||||
verification=(
|
||||
"verify the pinned definition and canonical action request hashes",
|
||||
"verify the provider result and every announced effect",
|
||||
"reconcile an uncertain provider outcome before continuation",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def workflow_session_factory(session: Session) -> sessionmaker[Session]:
|
||||
bind = session.get_bind()
|
||||
if bind is None:
|
||||
raise WorkflowRecoveryError("Workflow recovery requires a database bind")
|
||||
return sessionmaker(bind=bind, expire_on_commit=False)
|
||||
|
||||
|
||||
def canonical_sha256(value: object) -> str:
|
||||
encoded = json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class WorkflowActionRecovery:
|
||||
operation: DurableRecoveryOperation | None
|
||||
operation_id: str
|
||||
mode: RecoveryMode
|
||||
call_number: int
|
||||
replayed: bool
|
||||
|
||||
def checkpoint_dispatch(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
instance: WorkflowInstance,
|
||||
step: WorkflowInstanceStep,
|
||||
action_key: str,
|
||||
capability_name: str,
|
||||
) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
previous = dict(step.handoff or {})
|
||||
step.handoff = {
|
||||
**previous,
|
||||
"kind": "module_action",
|
||||
"state": "running",
|
||||
"action_key": action_key,
|
||||
"capability": capability_name,
|
||||
"allowed_actions": ["cancel"],
|
||||
"recovery": {
|
||||
"operation_id": self.operation_id,
|
||||
"mode": self.mode.value,
|
||||
"status": RecoveryStatus.RUNNING.value,
|
||||
"call_number": self.call_number,
|
||||
"boundary": "provider-dispatch",
|
||||
"requires_attention": False,
|
||||
},
|
||||
}
|
||||
step.status = "running"
|
||||
instance.status = "running"
|
||||
try:
|
||||
session.commit()
|
||||
self.operation.checkpoint(
|
||||
kind="provider-dispatch",
|
||||
summary="Provider dispatch crossed the recoverable effect boundary",
|
||||
evidence={
|
||||
"effect_started": self.mode != RecoveryMode.ATOMIC,
|
||||
"action_key": action_key,
|
||||
"capability": capability_name,
|
||||
"call_number": self.call_number,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
raise WorkflowRecoveryError(
|
||||
"Workflow action dispatch evidence could not be persisted"
|
||||
) from exc
|
||||
|
||||
def commit_conclusive_result(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
provider_state: str,
|
||||
result_sha256: str,
|
||||
observed_effects_sha256: str,
|
||||
) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
evidence = {
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"provider_state": provider_state,
|
||||
"result_sha256": result_sha256,
|
||||
"observed_effects_sha256": observed_effects_sha256,
|
||||
"call_number": self.call_number,
|
||||
},
|
||||
}
|
||||
if self.mode == RecoveryMode.ATOMIC:
|
||||
self.operation.commit_atomic_success(session, evidence=evidence)
|
||||
else:
|
||||
self.operation.commit_verified_success(session, evidence=evidence)
|
||||
|
||||
def commit_unknown(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
error_type: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
if self.operation is None:
|
||||
raise WorkflowRecoveryError(
|
||||
"A replayed action cannot acquire an unknown outcome"
|
||||
)
|
||||
try:
|
||||
session.commit()
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The provider acknowledgement was not conclusive",
|
||||
evidence={
|
||||
"effect_started": self.mode != RecoveryMode.ATOMIC,
|
||||
"error_type": error_type,
|
||||
"call_number": self.call_number,
|
||||
},
|
||||
failure_summary=message,
|
||||
)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
raise WorkflowRecoveryError(
|
||||
"Workflow action uncertainty could not be recorded"
|
||||
) from exc
|
||||
|
||||
def commit_definitive_failure(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
summary: str,
|
||||
error_type: str,
|
||||
) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
evidence = {
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"effect_started": False,
|
||||
"error_type": error_type,
|
||||
"call_number": self.call_number,
|
||||
},
|
||||
}
|
||||
if self.mode == RecoveryMode.ATOMIC:
|
||||
self.operation.commit_atomic_failure(
|
||||
session,
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
)
|
||||
else:
|
||||
self.operation.fail(summary=summary, evidence=evidence)
|
||||
session.commit()
|
||||
|
||||
|
||||
def begin_workflow_action_recovery(
|
||||
session: Session,
|
||||
*,
|
||||
instance: WorkflowInstance,
|
||||
step: WorkflowInstanceStep,
|
||||
revision: WorkflowDefinitionRevision,
|
||||
definition: ActionDefinition,
|
||||
capability_name: str,
|
||||
request_idempotency_key: str,
|
||||
action_input: Mapping[str, object],
|
||||
preview_payload: Mapping[str, object],
|
||||
backup_reference: str | None = None,
|
||||
approval_reference: str | None = None,
|
||||
) -> WorkflowActionRecovery:
|
||||
previous_recovery = (
|
||||
step.handoff.get("recovery")
|
||||
if isinstance(step.handoff, Mapping)
|
||||
else None
|
||||
)
|
||||
call_number = 1
|
||||
if isinstance(previous_recovery, Mapping):
|
||||
call_number = max(
|
||||
1,
|
||||
int(
|
||||
previous_recovery.get("next_call_number")
|
||||
or previous_recovery.get("call_number")
|
||||
or 1
|
||||
),
|
||||
)
|
||||
mode = RecoveryMode(definition.recovery_mode)
|
||||
plan = _action_recovery_plan(
|
||||
definition,
|
||||
mode=mode,
|
||||
backup_reference=backup_reference,
|
||||
approval_reference=approval_reference,
|
||||
)
|
||||
request_sha256 = canonical_sha256(dict(action_input))
|
||||
preview_sha256 = canonical_sha256(dict(preview_payload))
|
||||
access_context_sha256 = canonical_sha256(instance.authorization_)
|
||||
action_contract_sha256 = canonical_sha256(
|
||||
{
|
||||
"action_key": definition.action_key,
|
||||
"owner_module": definition.owner_module,
|
||||
"contract_version": definition.contract_version,
|
||||
"recovery_mode": definition.recovery_mode,
|
||||
"recovery_verification": list(definition.recovery_verification),
|
||||
"expected_effect_keys": list(definition.expected_effect_keys),
|
||||
}
|
||||
)
|
||||
session.commit()
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
workflow_session_factory(session),
|
||||
identity=process_runtime_identity(),
|
||||
module_id="workflow_engine",
|
||||
operation_type="activity.external-effect",
|
||||
idempotency_key=(
|
||||
f"workflow-step:{step.id}:action-call:{call_number}"
|
||||
),
|
||||
request={
|
||||
"tenant_id": instance.tenant_id,
|
||||
"instance_id": instance.id,
|
||||
"step_id": step.id,
|
||||
"node_id": step.node_id,
|
||||
"definition_revision_id": revision.id,
|
||||
"definition_hash": revision.content_hash,
|
||||
"action_key": definition.action_key,
|
||||
"capability": capability_name,
|
||||
"action_contract_sha256": action_contract_sha256,
|
||||
"action_input_sha256": request_sha256,
|
||||
"preview_sha256": preview_sha256,
|
||||
"provider_idempotency_sha256": canonical_sha256(
|
||||
request_idempotency_key
|
||||
),
|
||||
"call_number": call_number,
|
||||
},
|
||||
recovery_plan=plan,
|
||||
precondition_evidence={
|
||||
"definition_hash": revision.content_hash,
|
||||
"action_contract_sha256": action_contract_sha256,
|
||||
"action_input_sha256": request_sha256,
|
||||
"preview_sha256": preview_sha256,
|
||||
"access_context_sha256": access_context_sha256,
|
||||
"call_number": call_number,
|
||||
},
|
||||
lease_resource_key=f"workflow:step:{step.id}",
|
||||
lease_ttl_seconds=300,
|
||||
resource_type="workflow_instance_step",
|
||||
resource_id=step.id,
|
||||
metadata={
|
||||
"resources": ["postgresql", "module-provider"],
|
||||
"workflow_instance_id": instance.id,
|
||||
"action_key": definition.action_key,
|
||||
"capability": capability_name,
|
||||
},
|
||||
)
|
||||
except RecoveryOperationBusy as exc:
|
||||
raise WorkflowRecoveryBusy(
|
||||
"Another runtime owns this Workflow action step"
|
||||
) from exc
|
||||
except RecoveryOperationStateConflict as exc:
|
||||
raise WorkflowRecoveryConflict(exc.operation_id, exc.status) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise WorkflowRecoveryError(
|
||||
"The recovery ledger is unavailable; the module action did not start"
|
||||
) from exc
|
||||
return WorkflowActionRecovery(
|
||||
operation=started.operation,
|
||||
operation_id=started.operation_id,
|
||||
mode=mode,
|
||||
call_number=call_number,
|
||||
replayed=started.replayed,
|
||||
)
|
||||
|
||||
|
||||
def claim_workflow_action_recovery(
|
||||
session: Session,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> DurableRecoveryOperation:
|
||||
try:
|
||||
return claim_durable_recovery_operation(
|
||||
workflow_session_factory(session),
|
||||
identity=process_runtime_identity(),
|
||||
operation_id=operation_id,
|
||||
lease_ttl_seconds=300,
|
||||
)
|
||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
raise WorkflowRecoveryError(
|
||||
"The Workflow action outcome cannot currently be reconciled"
|
||||
) from exc
|
||||
|
||||
|
||||
def reconcile_stale_workflow_action_recovery(
|
||||
session: Session,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> str:
|
||||
try:
|
||||
handle = claim_durable_recovery_operation(
|
||||
workflow_session_factory(session),
|
||||
identity=process_runtime_identity(),
|
||||
operation_id=operation_id,
|
||||
lease_ttl_seconds=300,
|
||||
)
|
||||
except RecoveryOperationBusy as exc:
|
||||
raise WorkflowRecoveryBusy(
|
||||
"Another runtime still owns this Workflow action step"
|
||||
) from exc
|
||||
except RecoveryOperationStateConflict as exc:
|
||||
return exc.status
|
||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
raise WorkflowRecoveryError(
|
||||
"The stale Workflow action fence could not be reconciled"
|
||||
) from exc
|
||||
handle.release_unresolved()
|
||||
session.expire_all()
|
||||
operation = session.get(RecoveryOperation, operation_id)
|
||||
if operation is None:
|
||||
raise WorkflowRecoveryError(
|
||||
"The reconciled Workflow recovery operation is unavailable"
|
||||
)
|
||||
return str(operation.status)
|
||||
|
||||
|
||||
def workflow_action_recovery_state(
|
||||
session: Session,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> dict[str, object] | None:
|
||||
operation = session.get(RecoveryOperation, operation_id)
|
||||
if operation is None or operation.module_id != "workflow_engine":
|
||||
return None
|
||||
return {
|
||||
"operation_id": operation.id,
|
||||
"mode": operation.mode,
|
||||
"status": operation.status,
|
||||
"requires_attention": operation.status
|
||||
in {
|
||||
RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||
RecoveryStatus.MANUAL_INTERVENTION.value,
|
||||
},
|
||||
"failure_summary": operation.failure_summary,
|
||||
}
|
||||
|
||||
|
||||
def acquire_workflow_state_fence(
|
||||
session: Session,
|
||||
*,
|
||||
resource_key: str,
|
||||
ttl_seconds: int = 120,
|
||||
) -> LeaseClaim | None:
|
||||
identity = process_runtime_identity()
|
||||
return acquire_lease(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
resource_key=resource_key,
|
||||
holder_node_id=identity.node_id,
|
||||
holder_incarnation=identity.incarnation,
|
||||
ttl_seconds=ttl_seconds,
|
||||
metadata={"module_id": "workflow_engine", "kind": "state-transition"},
|
||||
)
|
||||
|
||||
|
||||
def release_workflow_state_fence(session: Session, claim: LeaseClaim) -> None:
|
||||
assert_lease_fence(session, claim)
|
||||
release_lease(session, claim)
|
||||
|
||||
|
||||
def _action_recovery_plan(
|
||||
definition: ActionDefinition,
|
||||
*,
|
||||
mode: RecoveryMode,
|
||||
backup_reference: str | None,
|
||||
approval_reference: str | None,
|
||||
) -> RecoveryPlan:
|
||||
if mode == RecoveryMode.SNAPSHOT_RESTORE and not backup_reference:
|
||||
raise WorkflowRecoveryError(
|
||||
"Snapshot-restore actions require a pinned recovery backup reference"
|
||||
)
|
||||
if mode == RecoveryMode.IRREVERSIBLE and not approval_reference:
|
||||
raise WorkflowRecoveryError(
|
||||
"Irreversible actions require a pinned approval reference"
|
||||
)
|
||||
return RecoveryPlan(
|
||||
mode=mode,
|
||||
preconditions=(
|
||||
"the pinned Workflow revision and action contract hashes are present",
|
||||
"the current authority and provider capability were revalidated",
|
||||
"the action step owns a distributed execution fence",
|
||||
),
|
||||
compensation_steps=(
|
||||
"invoke the provider-declared compensation action",
|
||||
"verify the domain invariant after compensation",
|
||||
)
|
||||
if mode == RecoveryMode.COMPENSATION
|
||||
else (),
|
||||
forward_recovery_steps=(
|
||||
"inspect the provider by stable idempotency key",
|
||||
"record whether the announced effect occurred",
|
||||
"continue or retry only after the outcome is proven",
|
||||
)
|
||||
if mode == RecoveryMode.FORWARD_RECOVERY
|
||||
else (),
|
||||
verification_steps=tuple(definition.recovery_verification),
|
||||
backup_reference=backup_reference,
|
||||
approval_reference=approval_reference,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"WORKFLOW_RECOVERY_OPERATIONS",
|
||||
"WorkflowActionRecovery",
|
||||
"WorkflowRecoveryDeclaration",
|
||||
"WorkflowRecoveryBusy",
|
||||
"WorkflowRecoveryConflict",
|
||||
"WorkflowRecoveryError",
|
||||
"acquire_workflow_state_fence",
|
||||
"begin_workflow_action_recovery",
|
||||
"canonical_sha256",
|
||||
"claim_workflow_action_recovery",
|
||||
"release_workflow_state_fence",
|
||||
"reconcile_stale_workflow_action_recovery",
|
||||
"workflow_action_recovery_state",
|
||||
"workflow_session_factory",
|
||||
]
|
||||
Reference in New Issue
Block a user