Fence and reconcile Dataflow runs

tests/test_service.py

@@ -17,6 +17,18 @@ from govoplan_core.core.datasources import (
     DatasourceMaterialization,
     DatasourcePublicationResult,
 )
+from govoplan_core.core.recovery import (
+    RecoveryCheckpoint,
+    RecoveryMode,
+    RecoveryOperation,
+    RecoveryStatus,
+    verify_recovery_evidence_chain,
+)
+from govoplan_core.core.runtime_coordination import (
+    DistributedLease,
+    RuntimeIdentity,
+    bind_process_runtime_identity,
+)
 from govoplan_core.db.base import Base
 from govoplan_dataflow.backend.backends.duckdb import DuckDbExecutionBackend
 from govoplan_dataflow.backend.db.models import (
@@ -99,6 +111,17 @@ def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
     )
 
 
+def runtime_identity() -> RuntimeIdentity:
+    return RuntimeIdentity(
+        installation_id="dataflow-service-tests",
+        node_id="service-node",
+        incarnation="service-incarnation",
+        role="api",
+        software_version="test",
+        composition_hash="a" * 64,
+    )
+
+
 class FakePublicationProvider:
     def __init__(self) -> None:
         self.requests = []
@@ -128,6 +151,25 @@ class FakePublicationProvider:
         )
 
 
+class FailingPublicationProvider(FakePublicationProvider):
+    def publish_rows(self, _session, _principal, *, request):
+        self.requests.append(request)
+        raise RuntimeError("connection closed after dispatch")
+
+
+class TamperingPublicationProvider(FakePublicationProvider):
+    def publish_rows(self, session, principal, *, request):
+        checkpoint = session.scalar(
+            select(RecoveryCheckpoint)
+            .order_by(RecoveryCheckpoint.sequence)
+            .limit(1)
+        )
+        assert checkpoint is not None
+        checkpoint.summary = "tampered"
+        session.commit()
+        return super().publish_rows(session, principal, request=request)
+
+
 class FakeRegistry:
     def __init__(self, publication_provider: FakePublicationProvider) -> None:
         self.publication_provider = publication_provider
@@ -159,6 +201,9 @@ class DataflowServiceTests(unittest.TestCase):
         Base.metadata.create_all(
             self.engine,
             tables=[
+                DistributedLease.__table__,
+                RecoveryOperation.__table__,
+                RecoveryCheckpoint.__table__,
                 DataflowPipeline.__table__,
                 DataflowPipelineRevision.__table__,
                 DataflowRun.__table__,
@@ -166,8 +211,10 @@ class DataflowServiceTests(unittest.TestCase):
         )
         self.Session = sessionmaker(bind=self.engine)
         self.session: Session = self.Session()
+        bind_process_runtime_identity(runtime_identity())
 
     def tearDown(self) -> None:
+        bind_process_runtime_identity(None)
         self.session.close()
         Base.metadata.drop_all(
             self.engine,
@@ -175,6 +222,9 @@ class DataflowServiceTests(unittest.TestCase):
                 DataflowRun.__table__,
                 DataflowPipelineRevision.__table__,
                 DataflowPipeline.__table__,
+                RecoveryCheckpoint.__table__,
+                RecoveryOperation.__table__,
+                DistributedLease.__table__,
             ],
         )
         self.engine.dispose()
@@ -435,6 +485,26 @@ class DataflowServiceTests(unittest.TestCase):
             list(publication_provider.requests[0].rows),
         )
         self.assertEqual(1, len(publication_provider.requests))
+        operation = self.session.scalar(
+            select(RecoveryOperation).where(
+                RecoveryOperation.resource_id == first.id
+            )
+        )
+        assert operation is not None
+        self.assertEqual(RecoveryMode.FORWARD_RECOVERY.value, operation.mode)
+        self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
+        self.assertTrue(
+            verify_recovery_evidence_chain(self.session, operation.id)
+        )
+        checkpoint_kinds = list(
+            self.session.scalars(
+                select(RecoveryCheckpoint.kind)
+                .where(RecoveryCheckpoint.operation_id == operation.id)
+                .order_by(RecoveryCheckpoint.sequence)
+            )
+        )
+        self.assertIn("output-publication-dispatch", checkpoint_kinds)
+        self.assertIn("verified-success", checkpoint_kinds)
 
     def test_run_idempotency_key_rejects_changed_parameters(self) -> None:
         pipeline = self._create()
@@ -490,6 +560,96 @@ class DataflowServiceTests(unittest.TestCase):
 
         self.assertEqual("failed", run.status)
         self.assertIn("Datasources publication capability", run.error)
+        operation = self.session.scalar(
+            select(RecoveryOperation).where(
+                RecoveryOperation.resource_id == run.id
+            )
+        )
+        assert operation is not None
+        self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
+
+    def test_provider_failure_after_dispatch_requires_reconciliation(self) -> None:
+        pipeline = self._create()
+        provider = FailingPublicationProvider()
+        request = DataflowRunRequest(
+            pipeline_ref=f"pipeline:{pipeline.id}",
+            revision=1,
+            idempotency_key="uncertain-publication",
+            publication=DataflowPublicationTarget(
+                name="Uncertain output",
+                source_name="uncertain_output",
+            ),
+        )
+
+        run, replayed = start_pipeline_run(
+            self.session,
+            tenant_id="tenant-1",
+            actor_id="user-1",
+            principal=principal(),
+            registry=FakeRegistry(provider),
+            request=request,
+        )
+        replay, second_replayed = start_pipeline_run(
+            self.session,
+            tenant_id="tenant-1",
+            actor_id="user-1",
+            principal=principal(),
+            registry=FakeRegistry(provider),
+            request=request,
+        )
+
+        self.assertFalse(replayed)
+        self.assertTrue(second_replayed)
+        self.assertEqual(run.id, replay.id)
+        self.assertEqual("outcome_unknown", run.status)
+        self.assertEqual(1, len(provider.requests))
+        operation = self.session.scalar(
+            select(RecoveryOperation).where(
+                RecoveryOperation.resource_id == run.id
+            )
+        )
+        assert operation is not None
+        self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
+
+    def test_tampered_recovery_chain_prevents_verified_publication(self) -> None:
+        pipeline = self._create()
+        provider = TamperingPublicationProvider()
+
+        with self.assertRaises(DataflowConflictError):
+            start_pipeline_run(
+                self.session,
+                tenant_id="tenant-1",
+                actor_id="user-1",
+                principal=principal(),
+                registry=FakeRegistry(provider),
+                request=DataflowRunRequest(
+                    pipeline_ref=f"pipeline:{pipeline.id}",
+                    revision=1,
+                    idempotency_key="tampered-publication",
+                    publication=DataflowPublicationTarget(
+                        name="Tampered output",
+                        source_name="tampered_output",
+                    ),
+                ),
+            )
+
+        run = self.session.scalar(
+            select(DataflowRun).where(
+                DataflowRun.idempotency_key == "tampered-publication"
+            )
+        )
+        assert run is not None
+        operation = self.session.scalar(
+            select(RecoveryOperation).where(
+                RecoveryOperation.resource_id == run.id
+            )
+        )
+        assert operation is not None
+        self.assertEqual("outcome_unknown", run.status)
+        self.assertIsNone(run.output_publication_ref)
+        self.assertFalse(
+            verify_recovery_evidence_chain(self.session, operation.id)
+        )
 
 
 if __name__ == "__main__":