feat(datasources): publish durable artifact outputs

tests/test_lifecycle.py

@@ -11,6 +11,7 @@ from govoplan_core.core.change_sequence import ChangeSequenceEntry
 from govoplan_core.core.datasources import (
     CAPABILITY_DATASOURCE_ORIGINS,
     DatasourceAccessError,
+    DatasourceArtifactReference,
     DatasourceField,
     DatasourceGovernance,
     DatasourceOrigin,
@@ -44,6 +45,7 @@ from govoplan_datasources.backend.service import (
     SqlDatasourceProvider,
 )
 from govoplan_datasources.backend.payloads import (
+    ExternalArtifactPayloadBackend,
     create_database_rows_payload,
     finalize_payload_deletion,
     mark_unreferenced_payload_for_deletion,
@@ -166,6 +168,56 @@ class FakeRegistry:
         return self.origin_provider
 
 
+class FakeArtifactBackend:
+    backend = "test_artifact"
+
+    def __init__(self) -> None:
+        self.verified: list[str] = []
+        self.deleted: list[str] = []
+
+    def read_rows(
+        self,
+        _session,
+        *,
+        tenant_id: str,
+        artifact: DatasourceArtifactReference,
+        offset: int,
+        limit: int,
+    ):
+        self.assert_tenant(tenant_id)
+        stop = min(artifact.row_count, offset + limit)
+        return tuple(
+            {"id": index, "result": "match"}
+            for index in range(offset, stop)
+        )
+
+    def verify(
+        self,
+        _session,
+        *,
+        tenant_id: str,
+        artifact: DatasourceArtifactReference,
+    ) -> None:
+        self.assert_tenant(tenant_id)
+        if not artifact.locator.startswith("artifact:"):
+            raise DatasourceUnavailableError("Unknown test artifact.")
+        self.verified.append(artifact.locator)
+
+    def delete(
+        self,
+        _session,
+        *,
+        tenant_id: str,
+        artifact: DatasourceArtifactReference,
+    ) -> None:
+        self.assert_tenant(tenant_id)
+        self.deleted.append(artifact.locator)
+
+    def assert_tenant(self, tenant_id: str) -> None:
+        if tenant_id != "tenant-1":
+            raise AssertionError("Artifact backend crossed a tenant boundary.")
+
+
 class DatasourceLifecycleTests(unittest.TestCase):
     def setUp(self) -> None:
         self.engine = create_engine("sqlite:///:memory:")
@@ -185,8 +237,10 @@ class DatasourceLifecycleTests(unittest.TestCase):
         self.Session = sessionmaker(bind=self.engine)
         self.session = self.Session()
         self.origins = FakeOriginProvider()
+        self.artifacts = FakeArtifactBackend()
         self.provider = SqlDatasourceProvider(
             registry=FakeRegistry(self.origins),
+            payload_backends=(ExternalArtifactPayloadBackend(self.artifacts),),
         )
 
     def tearDown(self) -> None:
@@ -667,6 +721,147 @@ class DatasourceLifecycleTests(unittest.TestCase):
             self.session.query(DatasourcePublicationRecord).count(),
         )
 
+    def test_artifact_publication_pins_large_payload_and_supports_bounded_reads(
+        self,
+    ) -> None:
+        artifact = DatasourceArtifactReference(
+            backend="test_artifact",
+            locator="artifact:monthly-output",
+            checksum="a" * 64,
+            row_count=25_000,
+            byte_count=12_000_000,
+            schema=(
+                DatasourceField("id", "integer", nullable=False),
+                DatasourceField("result", "string", nullable=False),
+            ),
+            fingerprint="b" * 64,
+        )
+
+        published = self.provider.publish_rows(
+            self.session,
+            principal(scopes=(SOURCE_WRITE_SCOPE, CATALOGUE_READ_SCOPE)),
+            request=DatasourcePublicationRequest(
+                producer_module="dataflow",
+                producer_run_ref="dataflow-run:large-output",
+                idempotency_key="large-output",
+                name="Large output",
+                source_name="large_output",
+                artifact=artifact,
+            ),
+        )
+        preview = self.provider.read_datasource(
+            self.session,
+            principal(scopes=(CATALOGUE_READ_SCOPE,)),
+            request=DatasourceReadRequest(
+                datasource_ref=published.datasource.ref,
+                offset=10,
+                limit=3,
+            ),
+        )
+
+        self.assertEqual("published", published.status)
+        self.assertEqual(25_000, published.materialization.row_count)
+        self.assertEqual(
+            [{"id": 10, "result": "match"},
+             {"id": 11, "result": "match"},
+             {"id": 12, "result": "match"}],
+            list(preview.rows),
+        )
+        self.assertEqual(
+            ["artifact:monthly-output", "artifact:monthly-output"],
+            self.artifacts.verified,
+        )
+
+    def test_unattested_artifact_quality_rules_require_review_without_becoming_current(
+        self,
+    ) -> None:
+        published = self.provider.publish_rows(
+            self.session,
+            principal(scopes=(SOURCE_WRITE_SCOPE,)),
+            request=DatasourcePublicationRequest(
+                producer_module="reporting",
+                producer_run_ref="report-run:review",
+                idempotency_key="review-output",
+                name="Review output",
+                source_name="review_output",
+                artifact=DatasourceArtifactReference(
+                    backend="test_artifact",
+                    locator="artifact:review-output",
+                    checksum="c" * 64,
+                    row_count=2,
+                    byte_count=128,
+                    schema=(DatasourceField("id", "integer", False),),
+                    fingerprint="d" * 64,
+                ),
+                governance=DatasourceGovernance(
+                    quality_policy={
+                        "version": "unique-id-v1",
+                        "rules": [
+                            {
+                                "id": "unique-id",
+                                "type": "unique",
+                                "fields": ["id"],
+                            }
+                        ],
+                    }
+                ),
+            ),
+        )
+        record = self.session.get(
+            DatasourceRecord,
+            published.datasource.ref.removeprefix("datasource:"),
+        )
+
+        self.assertEqual("review_required", published.status)
+        self.assertEqual("review_required", published.materialization.state)
+        self.assertIsNotNone(record)
+        assert record is not None
+        self.assertIsNone(record.current_materialization_id)
+
+    def test_artifact_warning_is_a_notification_ready_terminal_state(self) -> None:
+        published = self.provider.publish_rows(
+            self.session,
+            principal(scopes=(SOURCE_WRITE_SCOPE,)),
+            request=DatasourcePublicationRequest(
+                producer_module="dataflow",
+                producer_run_ref="dataflow-run:warning",
+                idempotency_key="warning-output",
+                name="Warning output",
+                source_name="warning_output",
+                artifact=DatasourceArtifactReference(
+                    backend="test_artifact",
+                    locator="artifact:warning-output",
+                    checksum="e" * 64,
+                    row_count=1,
+                    byte_count=64,
+                    schema=(DatasourceField("id", "integer", False),),
+                    fingerprint="f" * 64,
+                    validation={
+                        "status": "warning",
+                        "warnings": [
+                            {
+                                "severity": "warning",
+                                "code": "producer.partial_match",
+                                "message": "One source used a fallback match.",
+                            }
+                        ],
+                    },
+                ),
+            ),
+        )
+        record = self.session.get(
+            DatasourcePublicationRecord,
+            published.ref.removeprefix("publication:"),
+        )
+
+        self.assertEqual("published_with_warnings", published.status)
+        self.assertIsNotNone(record)
+        assert record is not None
+        self.assertEqual(
+            "published_with_warnings",
+            record.status,
+        )
+
     def test_publication_idempotency_key_rejects_different_output(self) -> None:
         producer = principal(scopes=(SOURCE_WRITE_SCOPE,))
         base = DatasourcePublicationRequest(