feat(core): dispatch durable dataflow runs
This commit is contained in:
@@ -4,9 +4,12 @@ import unittest
|
||||
|
||||
from govoplan_core.core.dataflows import (
|
||||
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
|
||||
CAPABILITY_DATAFLOW_RUN_WORKER,
|
||||
DataflowRunDescriptor,
|
||||
DataflowRunLifecycleProvider,
|
||||
DataflowRunWorker,
|
||||
dataflow_run_lifecycle,
|
||||
dataflow_run_worker,
|
||||
)
|
||||
from govoplan_core.core.modules import ModuleContext, ModuleManifest
|
||||
from govoplan_core.core.registry import PlatformRegistry
|
||||
@@ -49,10 +52,27 @@ class _Provider:
|
||||
)
|
||||
|
||||
|
||||
class _Worker:
|
||||
def dispatch_pending(
|
||||
self,
|
||||
session,
|
||||
*,
|
||||
now=None,
|
||||
limit=10,
|
||||
worker_id=None,
|
||||
):
|
||||
return {"claimed": 0}
|
||||
|
||||
def purge_expired(self, session, *, now=None, limit=500):
|
||||
return {"purged": 0}
|
||||
|
||||
|
||||
class DataflowContractTests(unittest.TestCase):
|
||||
def test_run_lifecycle_is_runtime_checkable_and_resolved(self) -> None:
|
||||
provider = _Provider()
|
||||
worker = _Worker()
|
||||
self.assertIsInstance(provider, DataflowRunLifecycleProvider)
|
||||
self.assertIsInstance(worker, DataflowRunWorker)
|
||||
registry = PlatformRegistry()
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
@@ -61,6 +81,7 @@ class DataflowContractTests(unittest.TestCase):
|
||||
version="test",
|
||||
capability_factories={
|
||||
CAPABILITY_DATAFLOW_RUN_LIFECYCLE: lambda context: provider,
|
||||
CAPABILITY_DATAFLOW_RUN_WORKER: lambda context: worker,
|
||||
},
|
||||
)
|
||||
)
|
||||
@@ -69,7 +90,9 @@ class DataflowContractTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIs(provider, dataflow_run_lifecycle(registry))
|
||||
self.assertIs(worker, dataflow_run_worker(registry))
|
||||
self.assertIsNone(dataflow_run_lifecycle(PlatformRegistry()))
|
||||
self.assertIsNone(dataflow_run_worker(PlatformRegistry()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
91
tests/test_dataflow_run_worker.py
Normal file
91
tests/test_dataflow_run_worker.py
Normal file
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
from govoplan_core.celery_app import (
|
||||
celery,
|
||||
dispatch_dataflow_runs,
|
||||
purge_dataflow_runs,
|
||||
)
|
||||
|
||||
|
||||
class DataflowRunWorkerTests(unittest.TestCase):
|
||||
def test_dispatch_commits_worker_outcome(self) -> None:
|
||||
session = MagicMock()
|
||||
database = MagicMock()
|
||||
database.SessionLocal.return_value.__enter__.return_value = session
|
||||
provider = MagicMock()
|
||||
provider.dispatch_pending.return_value = {
|
||||
"claimed": 1,
|
||||
"succeeded": 1,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"govoplan_core.celery_app._dataflow_run_worker",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"govoplan_core.db.session.get_database",
|
||||
return_value=database,
|
||||
),
|
||||
):
|
||||
result = dispatch_dataflow_runs.run(7)
|
||||
|
||||
provider.dispatch_pending.assert_called_once_with(
|
||||
session,
|
||||
limit=7,
|
||||
worker_id=ANY,
|
||||
)
|
||||
session.commit.assert_called_once_with()
|
||||
self.assertEqual(1, result["succeeded"])
|
||||
|
||||
def test_retention_commits_worker_outcome(self) -> None:
|
||||
session = MagicMock()
|
||||
database = MagicMock()
|
||||
database.SessionLocal.return_value.__enter__.return_value = session
|
||||
provider = MagicMock()
|
||||
provider.purge_expired.return_value = {"purged": 2}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"govoplan_core.celery_app._dataflow_run_worker",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"govoplan_core.db.session.get_database",
|
||||
return_value=database,
|
||||
),
|
||||
):
|
||||
result = purge_dataflow_runs.run(25)
|
||||
|
||||
provider.purge_expired.assert_called_once_with(session, limit=25)
|
||||
session.commit.assert_called_once_with()
|
||||
self.assertEqual(2, result["purged"])
|
||||
|
||||
def test_routes_and_periodic_jobs_are_registered(self) -> None:
|
||||
self.assertEqual(
|
||||
celery.conf.task_routes["govoplan.dataflow.dispatch_runs"],
|
||||
{"queue": "dataflow"},
|
||||
)
|
||||
self.assertEqual(
|
||||
celery.conf.task_routes["govoplan.dataflow.purge_runs"],
|
||||
{"queue": "dataflow"},
|
||||
)
|
||||
self.assertEqual(
|
||||
"govoplan.dataflow.dispatch_runs",
|
||||
celery.conf.beat_schedule[
|
||||
"dataflow-runs-every-five-seconds"
|
||||
]["task"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"govoplan.dataflow.purge_runs",
|
||||
celery.conf.beat_schedule[
|
||||
"dataflow-run-retention-daily"
|
||||
]["task"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user