GovOPlaN/govoplan-workflow-engine
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Fence and reconcile workflow runtime effects

Browse Source
This commit is contained in:
zemion
2026-08-03 06:37:45 +02:00
parent a1cea1d162
commit fed8fbe6c3
10 changed files with 1669 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_instance_service.py
+475 -1
View File
@@ -4,7 +4,7 @@ from dataclasses import replace
from datetime import UTC, datetime, timedelta
import unittest
from sqlalchemy import create_engine
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session, sessionmaker
from govoplan_core.auth import ApiPrincipal
@@ -32,7 +32,19 @@ from govoplan_core.core.institutional import (
ServiceLaunchRequest,
TemporalRevision,
)
from govoplan_core.core.recovery import (
RecoveryCheckpoint,
RecoveryOperation,
RecoveryStatus,
verify_recovery_evidence_chain,
)
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
bind_process_runtime_identity,
)
from govoplan_core.db.base import Base
from govoplan_core.db.base import utcnow
from govoplan_workflow_engine.backend.db.models import (
WorkflowDefinition,
WorkflowDefinitionRevision,
@@ -45,12 +57,17 @@ from govoplan_workflow_engine.backend.db.models import (
)
from govoplan_workflow_engine.backend.instance_service import (
SqlWorkflowRuntimeWorker,
_action_preview_payload,
cancel_instance,
instance_response,
reconcile_instance,
resolve_step,
start_instance,
)
from govoplan_workflow_engine.backend.recovery import (
acquire_workflow_state_fence,
begin_workflow_action_recovery,
)
from govoplan_workflow_engine.backend.schemas import (
BpmnRevisionInput,
WorkflowDefinitionCreateRequest,
@@ -97,6 +114,17 @@ def principal() -> ApiPrincipal:
)
def runtime_identity() -> RuntimeIdentity:
return RuntimeIdentity(
installation_id="workflow-engine-tests",
node_id="workflow-worker",
incarnation="workflow-worker-incarnation",
role="worker",
software_version="test",
composition_hash="c" * 64,
)
def runtime_graph() -> WorkflowGraph:
return WorkflowGraph(
nodes=[
@@ -278,6 +306,20 @@ class FakeDataflowLifecycle:
error="Data quality gate failed.",
)
def mark_outcome_unknown(self, run_ref: str) -> None:
self.runs[run_ref] = replace(
self.runs[run_ref],
status="outcome_unknown",
error="Output acknowledgement was lost.",
metadata={
"recovery": {
"operation_id": "recovery:dataflow:1",
"status": "outcome_unknown",
"requires_attention": True,
}
},
)
class FakeAutomationProvider:
def __init__(self) -> None:
@@ -337,6 +379,8 @@ class FakeActionProvider:
def execute_action(self, _session, _principal, *, request):
self.requests.append(request)
state = self.states.pop(0) if self.states else "completed"
if state == "exception":
raise RuntimeError("Provider acknowledgement was lost.")
if state != "completed":
return ActionExecutionResult(
state=state,
@@ -396,6 +440,9 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
Base.metadata.create_all(
self.engine,
tables=[
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
WorkflowDefinition.__table__,
WorkflowDefinitionRevision.__table__,
WorkflowInstance.__table__,
@@ -408,6 +455,7 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
)
self.Session = sessionmaker(bind=self.engine)
self.session: Session = self.Session()
bind_process_runtime_identity(runtime_identity())
self.dataflow = FakeDataflowLifecycle()
self.registry = Registry(self.dataflow)
self.definition = create_definition(
@@ -432,6 +480,7 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
self.session.commit()
def tearDown(self) -> None:
bind_process_runtime_identity(None)
self.session.close()
Base.metadata.drop_all(
self.engine,
@@ -444,6 +493,9 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
WorkflowInstance.__table__,
WorkflowDefinitionRevision.__table__,
WorkflowDefinition.__table__,
RecoveryCheckpoint.__table__,
RecoveryOperation.__table__,
DistributedLease.__table__,
],
)
self.engine.dispose()
@@ -687,6 +739,51 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
"workflow.action.completed",
{event.kind for event in response.events},
)
operation = self.session.scalar(
select(RecoveryOperation).where(
RecoveryOperation.resource_id == response.steps[1].id
)
)
assert operation is not None
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
self.assertTrue(verify_recovery_evidence_chain(self.session, operation.id))
def test_missing_optional_action_provider_blocks_without_recovery_effect(
self,
) -> None:
definition = create_definition(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=WorkflowDefinitionCreateRequest(
name="Unavailable action provider",
graph=action_graph(),
),
)
activate_definition(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
)
with self.assertRaisesRegex(WorkflowConflictError, "not available"):
start_instance(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
principal=principal(),
registry=self.registry,
payload=WorkflowInstanceStartRequest(
idempotency_key="missing-action-provider",
input={"case_id": "case-missing"},
),
)
self.assertEqual(
0,
self.session.scalar(select(func.count(RecoveryOperation.id))),
)
def test_retryable_module_action_reuses_the_idempotency_key(self) -> None:
action = FakeActionProvider("retryable", "completed")
@@ -739,6 +836,383 @@ class WorkflowInstanceServiceTests(unittest.TestCase):
action.requests[1].idempotency_key,
)
def test_unknown_module_action_requires_evidence_before_retry(self) -> None:
action = FakeActionProvider("exception", "completed")
registry = Registry(self.dataflow, action=action)
definition = create_definition(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=WorkflowDefinitionCreateRequest(
name="Uncertain action",
graph=action_graph(),
),
)
activate_definition(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
)
instance, _replayed = start_instance(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowInstanceStartRequest(
idempotency_key="uncertain-action",
input={"case_id": "case-unknown"},
),
)
waiting = instance_response(self.session, instance)
current = waiting.steps[-1]
self.assertEqual("outcome_unknown", current.handoff["state"])
self.assertNotIn("retry", current.handoff["allowed_actions"])
self.assertEqual(
{"confirm_effect", "confirm_absent", "cancel"},
set(current.handoff["allowed_actions"]),
)
operation = self.session.get(
RecoveryOperation,
current.handoff["recovery"]["operation_id"],
)
assert operation is not None
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
with self.assertRaisesRegex(WorkflowConflictError, "evidence reference"):
resolve_step(
self.session,
tenant_id="tenant-1",
instance_id=instance.id,
step_id=current.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowStepActionRequest(action="confirm_absent"),
)
resolved = resolve_step(
self.session,
tenant_id="tenant-1",
instance_id=instance.id,
step_id=current.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowStepActionRequest(
action="confirm_absent",
evidence=["provider-search:case-unknown"],
),
)
current = next(step for step in resolved.steps if step.id == current.id)
self.assertEqual("retryable", current.handoff["state"])
self.session.refresh(operation)
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
completed = resolve_step(
self.session,
tenant_id="tenant-1",
instance_id=instance.id,
step_id=current.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowStepActionRequest(action="retry"),
)
self.assertEqual("completed", completed.status)
self.assertEqual(2, len(action.requests))
self.assertEqual(
action.requests[0].idempotency_key,
action.requests[1].idempotency_key,
)
def test_confirmed_unknown_effect_advances_without_provider_replay(self) -> None:
action = FakeActionProvider("exception")
registry = Registry(self.dataflow, action=action)
definition = create_definition(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=WorkflowDefinitionCreateRequest(
name="Confirmed uncertain action",
graph=action_graph(),
),
)
activate_definition(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
)
instance, _replayed = start_instance(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowInstanceStartRequest(
idempotency_key="confirmed-uncertain-action",
input={"case_id": "case-confirmed"},
),
)
current = instance_response(self.session, instance).steps[-1]
operation_id = current.handoff["recovery"]["operation_id"]
resolved = resolve_step(
self.session,
tenant_id="tenant-1",
instance_id=instance.id,
step_id=current.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowStepActionRequest(
action="confirm_effect",
output={"case_ref": "case:confirmed"},
evidence=["provider-receipt:case-confirmed"],
),
)
self.assertEqual("completed", resolved.status)
self.assertEqual(1, len(action.requests))
operation = self.session.get(RecoveryOperation, operation_id)
assert operation is not None
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
def test_tampered_action_evidence_prevents_workflow_completion(self) -> None:
class TamperingProvider(FakeActionProvider):
def execute_action(self, session, principal, *, request):
result = super().execute_action(
session,
principal,
request=request,
)
checkpoint = session.scalar(
select(RecoveryCheckpoint).order_by(
RecoveryCheckpoint.sequence
)
)
assert checkpoint is not None
checkpoint.summary = "tampered provider evidence"
session.commit()
return result
action = TamperingProvider()
registry = Registry(self.dataflow, action=action)
definition = create_definition(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=WorkflowDefinitionCreateRequest(
name="Tampered action",
graph=action_graph(),
),
)
activate_definition(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
)
with self.assertRaisesRegex(ValueError, "chain verification failed"):
start_instance(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowInstanceStartRequest(
idempotency_key="tampered-action",
input={"case_id": "case-tampered"},
),
)
operation = self.session.scalar(
select(RecoveryOperation).where(
RecoveryOperation.module_id == "workflow_engine"
)
)
assert operation is not None
self.assertEqual(RecoveryStatus.RUNNING.value, operation.status)
self.assertFalse(verify_recovery_evidence_chain(self.session, operation.id))
def test_dataflow_unknown_outcome_blocks_workflow_until_resolved(self) -> None:
instance = self._start("dataflow-unknown")
self.dataflow.mark_outcome_unknown("run:1")
changed = reconcile_instance(
self.session,
instance=instance,
principal=principal(),
registry=self.registry,
)
self.assertFalse(changed)
self.assertEqual("waiting", instance.status)
current = next(
step for step in instance.steps if step.id == instance.current_step_id
)
self.assertEqual("dataflow_recovery", current.handoff["kind"])
self.assertEqual(["cancel"], current.handoff["allowed_actions"])
self.assertTrue(current.handoff["recovery"]["requires_attention"])
self.dataflow.finish("run:1")
changed = reconcile_instance(
self.session,
instance=instance,
principal=principal(),
registry=self.registry,
)
self.assertTrue(changed)
self.assertEqual("completed", instance.status)
def test_worker_does_not_advance_an_instance_owned_by_another_runtime(
self,
) -> None:
instance = self._start("fenced-instance")
self.session.commit()
self.dataflow.finish("run:1")
fence = acquire_workflow_state_fence(
self.session,
resource_key=f"workflow:instance:{instance.id}",
)
assert fence is not None
self.session.commit()
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="workflow-engine-tests",
node_id="other-worker",
incarnation="other-worker-incarnation",
role="worker",
software_version="test",
composition_hash="e" * 64,
)
)
worker = SqlWorkflowRuntimeWorker(
registry=Registry(self.dataflow, FakeAutomationProvider()),
)
summary = worker.reconcile_pending(self.session)
self.assertEqual(0, summary["advanced"])
self.assertEqual(1, summary["skipped"])
self.session.refresh(instance)
self.assertEqual("waiting", instance.status)
lease = self.session.scalar(
select(DistributedLease).where(
DistributedLease.resource_key == f"workflow:instance:{instance.id}"
)
)
assert lease is not None
lease.expires_at = utcnow() - timedelta(seconds=1)
self.session.commit()
summary = worker.reconcile_pending(self.session)
self.assertEqual(1, summary["advanced"])
self.session.refresh(instance)
self.assertEqual("completed", instance.status)
bind_process_runtime_identity(runtime_identity())
def test_stale_action_attempt_becomes_unknown_instead_of_replaying(
self,
) -> None:
action = FakeActionProvider("retryable", "completed")
registry = Registry(self.dataflow, action=action)
definition = create_definition(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=WorkflowDefinitionCreateRequest(
name="Stale action attempt",
graph=action_graph(),
),
)
activate_definition(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
)
instance, _replayed = start_instance(
self.session,
tenant_id="tenant-1",
definition_id=definition.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowInstanceStartRequest(
idempotency_key="stale-action-attempt",
input={"case_id": "case-stale"},
),
)
current = instance_response(self.session, instance).steps[-1]
revision = self.session.get(
WorkflowDefinitionRevision,
instance.definition_revision_id,
)
assert revision is not None
request = action.requests[0]
preview = action.preview_action(
self.session,
principal(),
request=request,
)
started = begin_workflow_action_recovery(
self.session,
instance=instance,
step=current,
revision=revision,
definition=action.action,
capability_name="test.actions",
request_idempotency_key=request.idempotency_key,
action_input={"case_id": "case-stale"},
preview_payload=_action_preview_payload(preview),
)
self.assertFalse(started.replayed)
lease = self.session.scalar(
select(DistributedLease).where(
DistributedLease.resource_key == f"workflow:step:{current.id}"
)
)
assert lease is not None
lease.expires_at = utcnow() - timedelta(seconds=1)
self.session.commit()
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="workflow-engine-tests",
node_id="takeover-worker",
incarnation="takeover-worker-incarnation",
role="worker",
software_version="test",
composition_hash="f" * 64,
)
)
resolved = resolve_step(
self.session,
tenant_id="tenant-1",
instance_id=instance.id,
step_id=current.id,
actor_id="account-1",
principal=principal(),
registry=registry,
payload=WorkflowStepActionRequest(action="retry"),
)
current = next(step for step in resolved.steps if step.id == current.id)
self.assertEqual("outcome_unknown", current.handoff["state"])
self.assertNotIn("retry", current.handoff["allowed_actions"])
self.assertEqual(1, len(action.requests))
operation = self.session.get(RecoveryOperation, started.operation_id)
assert operation is not None
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
bind_process_runtime_identity(runtime_identity())
def test_automated_dataflow_failure_policy_fails_without_handoff(
self,
) -> None: