feat: add idempotent producer publication
This commit is contained in:
@@ -14,12 +14,15 @@ from govoplan_core.core.datasources import (
|
||||
DatasourceOrigin,
|
||||
DatasourceOriginReadRequest,
|
||||
DatasourceOriginReadResult,
|
||||
DatasourcePublicationRequest,
|
||||
DatasourceReadRequest,
|
||||
DatasourceStageInput,
|
||||
DatasourceValidationError,
|
||||
)
|
||||
from govoplan_core.db.base import Base, utcnow
|
||||
from govoplan_datasources.backend.db.models import (
|
||||
DatasourceMaterializationRecord,
|
||||
DatasourcePublicationRecord,
|
||||
DatasourceRecord,
|
||||
DatasourceStageRecord,
|
||||
)
|
||||
@@ -136,6 +139,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
DatasourceRecord.__table__,
|
||||
DatasourceMaterializationRecord.__table__,
|
||||
DatasourceStageRecord.__table__,
|
||||
DatasourcePublicationRecord.__table__,
|
||||
],
|
||||
)
|
||||
self.Session = sessionmaker(bind=self.engine)
|
||||
@@ -151,6 +155,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
self.engine,
|
||||
tables=[
|
||||
DatasourceStageRecord.__table__,
|
||||
DatasourcePublicationRecord.__table__,
|
||||
DatasourceMaterializationRecord.__table__,
|
||||
DatasourceRecord.__table__,
|
||||
],
|
||||
@@ -346,6 +351,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
tuple(item.ref for item in self.provider.list_stages(self.session, writer)),
|
||||
)
|
||||
|
||||
def test_producer_publication_is_atomic_idempotent_and_addressable(self) -> None:
|
||||
producer = principal(scopes=())
|
||||
request = DatasourcePublicationRequest(
|
||||
producer_module="dataflow",
|
||||
producer_run_ref="dataflow-run:run-1",
|
||||
idempotency_key="monthly-output-2026-07",
|
||||
name="Monthly comparison result",
|
||||
source_name="monthly_comparison_result",
|
||||
rows=(
|
||||
{"case_id": "A-1", "result": "match"},
|
||||
{"case_id": "A-2", "result": "review"},
|
||||
),
|
||||
freeze=True,
|
||||
frozen_label="July 2026",
|
||||
provenance={"pipeline_revision": 3},
|
||||
)
|
||||
|
||||
first = self.provider.publish_rows(
|
||||
self.session,
|
||||
producer,
|
||||
request=request,
|
||||
)
|
||||
replay = self.provider.publish_rows(
|
||||
self.session,
|
||||
producer,
|
||||
request=request,
|
||||
)
|
||||
second = self.provider.publish_rows(
|
||||
self.session,
|
||||
producer,
|
||||
request=DatasourcePublicationRequest(
|
||||
producer_module="dataflow",
|
||||
producer_run_ref="dataflow-run:run-2",
|
||||
idempotency_key="monthly-output-2026-08",
|
||||
target_datasource_ref=first.datasource.ref,
|
||||
rows=({"case_id": "A-3", "result": "match"},),
|
||||
),
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
self.assertFalse(first.replayed)
|
||||
self.assertTrue(replay.replayed)
|
||||
self.assertEqual(first.ref, replay.ref)
|
||||
self.assertEqual(first.materialization.ref, replay.materialization.ref)
|
||||
self.assertIsNotNone(first.materialization.frozen_at)
|
||||
self.assertEqual(1, first.materialization.revision)
|
||||
self.assertEqual(2, second.materialization.revision)
|
||||
self.assertEqual(first.datasource.ref, second.datasource.ref)
|
||||
self.assertEqual(
|
||||
2,
|
||||
self.session.query(DatasourcePublicationRecord).count(),
|
||||
)
|
||||
|
||||
def test_publication_idempotency_key_rejects_different_output(self) -> None:
|
||||
producer = principal(scopes=())
|
||||
base = DatasourcePublicationRequest(
|
||||
producer_module="dataflow",
|
||||
producer_run_ref="dataflow-run:run-1",
|
||||
idempotency_key="stable-key",
|
||||
name="Result",
|
||||
source_name="result",
|
||||
rows=({"id": 1},),
|
||||
)
|
||||
self.provider.publish_rows(self.session, producer, request=base)
|
||||
|
||||
with self.assertRaises(DatasourceValidationError):
|
||||
self.provider.publish_rows(
|
||||
self.session,
|
||||
producer,
|
||||
request=DatasourcePublicationRequest(
|
||||
producer_module="dataflow",
|
||||
producer_run_ref="dataflow-run:run-1",
|
||||
idempotency_key="stable-key",
|
||||
name="Result",
|
||||
source_name="result",
|
||||
rows=({"id": 2},),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user