feat: add idempotent producer publication

This commit is contained in:
2026-07-28 13:47:53 +02:00
parent 1cb6228442
commit 8ae7dace99
9 changed files with 530 additions and 4 deletions

View File

@@ -14,12 +14,15 @@ from govoplan_core.core.datasources import (
DatasourceOrigin,
DatasourceOriginReadRequest,
DatasourceOriginReadResult,
DatasourcePublicationRequest,
DatasourceReadRequest,
DatasourceStageInput,
DatasourceValidationError,
)
from govoplan_core.db.base import Base, utcnow
from govoplan_datasources.backend.db.models import (
DatasourceMaterializationRecord,
DatasourcePublicationRecord,
DatasourceRecord,
DatasourceStageRecord,
)
@@ -136,6 +139,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
DatasourceRecord.__table__,
DatasourceMaterializationRecord.__table__,
DatasourceStageRecord.__table__,
DatasourcePublicationRecord.__table__,
],
)
self.Session = sessionmaker(bind=self.engine)
@@ -151,6 +155,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
self.engine,
tables=[
DatasourceStageRecord.__table__,
DatasourcePublicationRecord.__table__,
DatasourceMaterializationRecord.__table__,
DatasourceRecord.__table__,
],
@@ -346,6 +351,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
tuple(item.ref for item in self.provider.list_stages(self.session, writer)),
)
def test_producer_publication_is_atomic_idempotent_and_addressable(self) -> None:
producer = principal(scopes=())
request = DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="monthly-output-2026-07",
name="Monthly comparison result",
source_name="monthly_comparison_result",
rows=(
{"case_id": "A-1", "result": "match"},
{"case_id": "A-2", "result": "review"},
),
freeze=True,
frozen_label="July 2026",
provenance={"pipeline_revision": 3},
)
first = self.provider.publish_rows(
self.session,
producer,
request=request,
)
replay = self.provider.publish_rows(
self.session,
producer,
request=request,
)
second = self.provider.publish_rows(
self.session,
producer,
request=DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-2",
idempotency_key="monthly-output-2026-08",
target_datasource_ref=first.datasource.ref,
rows=({"case_id": "A-3", "result": "match"},),
),
)
self.session.commit()
self.assertFalse(first.replayed)
self.assertTrue(replay.replayed)
self.assertEqual(first.ref, replay.ref)
self.assertEqual(first.materialization.ref, replay.materialization.ref)
self.assertIsNotNone(first.materialization.frozen_at)
self.assertEqual(1, first.materialization.revision)
self.assertEqual(2, second.materialization.revision)
self.assertEqual(first.datasource.ref, second.datasource.ref)
self.assertEqual(
2,
self.session.query(DatasourcePublicationRecord).count(),
)
def test_publication_idempotency_key_rejects_different_output(self) -> None:
producer = principal(scopes=())
base = DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="stable-key",
name="Result",
source_name="result",
rows=({"id": 1},),
)
self.provider.publish_rows(self.session, producer, request=base)
with self.assertRaises(DatasourceValidationError):
self.provider.publish_rows(
self.session,
producer,
request=DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="stable-key",
name="Result",
source_name="result",
rows=({"id": 2},),
),
)
if __name__ == "__main__":
unittest.main()

View File

@@ -5,6 +5,7 @@ import unittest
from govoplan_core.core.datasources import (
CAPABILITY_DATASOURCE_CATALOGUE,
CAPABILITY_DATASOURCE_LIFECYCLE,
CAPABILITY_DATASOURCE_PUBLICATION,
)
from govoplan_datasources.backend.manifest import get_manifest
@@ -16,6 +17,10 @@ class DatasourceManifestTests(unittest.TestCase):
self.assertEqual(manifest.id, "datasources")
self.assertIn(CAPABILITY_DATASOURCE_CATALOGUE, manifest.capability_factories)
self.assertIn(CAPABILITY_DATASOURCE_LIFECYCLE, manifest.capability_factories)
self.assertIn(
CAPABILITY_DATASOURCE_PUBLICATION,
manifest.capability_factories,
)
self.assertIn(
"datasources.materializations",
{item.name for item in manifest.provides_interfaces},

View File

@@ -24,13 +24,14 @@ class DatasourceMigrationTests(unittest.TestCase):
try:
with engine.connect() as connection:
self.assertIn(
"f3a8d2c7b1e0",
"c4e9f1a7d2b6",
set(MigrationContext.configure(connection).get_current_heads()),
)
self.assertEqual(
{
"datasource_catalogue",
"datasource_materializations",
"datasource_publications",
"datasource_stages",
},
{