feat: add idempotent producer publication

This commit is contained in:
2026-07-28 13:47:53 +02:00
parent 1cb6228442
commit 8ae7dace99
9 changed files with 530 additions and 4 deletions

View File

@@ -14,12 +14,15 @@ from govoplan_core.core.datasources import (
DatasourceOrigin,
DatasourceOriginReadRequest,
DatasourceOriginReadResult,
DatasourcePublicationRequest,
DatasourceReadRequest,
DatasourceStageInput,
DatasourceValidationError,
)
from govoplan_core.db.base import Base, utcnow
from govoplan_datasources.backend.db.models import (
DatasourceMaterializationRecord,
DatasourcePublicationRecord,
DatasourceRecord,
DatasourceStageRecord,
)
@@ -136,6 +139,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
DatasourceRecord.__table__,
DatasourceMaterializationRecord.__table__,
DatasourceStageRecord.__table__,
DatasourcePublicationRecord.__table__,
],
)
self.Session = sessionmaker(bind=self.engine)
@@ -151,6 +155,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
self.engine,
tables=[
DatasourceStageRecord.__table__,
DatasourcePublicationRecord.__table__,
DatasourceMaterializationRecord.__table__,
DatasourceRecord.__table__,
],
@@ -346,6 +351,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
tuple(item.ref for item in self.provider.list_stages(self.session, writer)),
)
def test_producer_publication_is_atomic_idempotent_and_addressable(self) -> None:
producer = principal(scopes=())
request = DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="monthly-output-2026-07",
name="Monthly comparison result",
source_name="monthly_comparison_result",
rows=(
{"case_id": "A-1", "result": "match"},
{"case_id": "A-2", "result": "review"},
),
freeze=True,
frozen_label="July 2026",
provenance={"pipeline_revision": 3},
)
first = self.provider.publish_rows(
self.session,
producer,
request=request,
)
replay = self.provider.publish_rows(
self.session,
producer,
request=request,
)
second = self.provider.publish_rows(
self.session,
producer,
request=DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-2",
idempotency_key="monthly-output-2026-08",
target_datasource_ref=first.datasource.ref,
rows=({"case_id": "A-3", "result": "match"},),
),
)
self.session.commit()
self.assertFalse(first.replayed)
self.assertTrue(replay.replayed)
self.assertEqual(first.ref, replay.ref)
self.assertEqual(first.materialization.ref, replay.materialization.ref)
self.assertIsNotNone(first.materialization.frozen_at)
self.assertEqual(1, first.materialization.revision)
self.assertEqual(2, second.materialization.revision)
self.assertEqual(first.datasource.ref, second.datasource.ref)
self.assertEqual(
2,
self.session.query(DatasourcePublicationRecord).count(),
)
def test_publication_idempotency_key_rejects_different_output(self) -> None:
producer = principal(scopes=())
base = DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="stable-key",
name="Result",
source_name="result",
rows=({"id": 1},),
)
self.provider.publish_rows(self.session, producer, request=base)
with self.assertRaises(DatasourceValidationError):
self.provider.publish_rows(
self.session,
producer,
request=DatasourcePublicationRequest(
producer_module="dataflow",
producer_run_ref="dataflow-run:run-1",
idempotency_key="stable-key",
name="Result",
source_name="result",
rows=({"id": 2},),
),
)
if __name__ == "__main__":
unittest.main()