Initialize governed Dataflow module
This commit is contained in:
125
tests/test_graph_and_sql.py
Normal file
125
tests/test_graph_and_sql.py
Normal file
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from govoplan_dataflow.backend.executor import execute_preview
|
||||
from govoplan_dataflow.backend.schemas import (
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
GraphPosition,
|
||||
PipelineGraph,
|
||||
)
|
||||
from govoplan_dataflow.backend.sql_compiler import SqlCompilationError, compile_sql, render_sql
|
||||
|
||||
|
||||
def inline_source() -> GraphNode:
|
||||
return GraphNode(
|
||||
id="source",
|
||||
type="source.inline",
|
||||
label="Monthly input",
|
||||
position=GraphPosition(x=40, y=160),
|
||||
config={
|
||||
"source_name": "monthly_files",
|
||||
"rows": [
|
||||
{"department": "A", "status": "open", "amount": 10},
|
||||
{"department": "A", "status": "closed", "amount": 5},
|
||||
{"department": "B", "status": "open", "amount": 20},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class DataflowGraphAndSqlTests(unittest.TestCase):
|
||||
def test_compiles_renders_and_executes_grouped_query(self) -> None:
|
||||
sql = """
|
||||
SELECT department, COUNT(*) AS records, SUM(amount) AS total
|
||||
FROM monthly_files
|
||||
WHERE status = 'open'
|
||||
GROUP BY department
|
||||
ORDER BY total DESC
|
||||
LIMIT 50
|
||||
"""
|
||||
graph, normalized, diagnostics = compile_sql(sql, source_nodes=[inline_source()])
|
||||
|
||||
self.assertEqual([], diagnostics)
|
||||
self.assertEqual(
|
||||
["source.inline", "filter", "aggregate", "sort", "limit", "output"],
|
||||
[node.type for node in graph.nodes],
|
||||
)
|
||||
self.assertEqual(normalized, render_sql(graph)[0])
|
||||
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
self.assertEqual(
|
||||
[
|
||||
{"department": "B", "records": 1, "total": 20},
|
||||
{"department": "A", "records": 1, "total": 10},
|
||||
],
|
||||
result.rows,
|
||||
)
|
||||
self.assertEqual(2, result.total_rows)
|
||||
self.assertFalse(result.truncated)
|
||||
self.assertEqual(3, result.input_row_count)
|
||||
self.assertEqual(6, len(result.node_diagnostics))
|
||||
self.assertEqual(1, len(result.source_fingerprints))
|
||||
|
||||
def test_rejects_effectful_or_multi_statement_sql(self) -> None:
|
||||
for sql in (
|
||||
"DELETE FROM monthly_files",
|
||||
"SELECT * FROM monthly_files; DROP TABLE monthly_files",
|
||||
"INSERT INTO target SELECT * FROM monthly_files",
|
||||
):
|
||||
with self.subTest(sql=sql), self.assertRaises(SqlCompilationError):
|
||||
compile_sql(sql, source_nodes=[inline_source()])
|
||||
|
||||
def test_rejects_join_until_comparison_slice(self) -> None:
|
||||
with self.assertRaisesRegex(SqlCompilationError, "JOIN support"):
|
||||
compile_sql(
|
||||
"SELECT * FROM monthly_files JOIN other ON monthly_files.id = other.id",
|
||||
source_nodes=[inline_source()],
|
||||
)
|
||||
|
||||
def test_graph_validation_rejects_cycle(self) -> None:
|
||||
source = inline_source()
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=300, y=160),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[source, output],
|
||||
edges=[
|
||||
GraphEdge(id="forward", source="source", target="output"),
|
||||
GraphEdge(id="back", source="output", target="source"),
|
||||
],
|
||||
)
|
||||
|
||||
from govoplan_dataflow.backend.graph import validate_graph
|
||||
|
||||
codes = {item.code for item in validate_graph(graph)}
|
||||
self.assertIn("graph.cycle", codes)
|
||||
|
||||
def test_preview_truncates_response_without_changing_pipeline_limit(self) -> None:
|
||||
source = inline_source()
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=300, y=160),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[source, output],
|
||||
edges=[GraphEdge(id="edge", source="source", target="output")],
|
||||
)
|
||||
|
||||
result = execute_preview(graph, row_limit=2)
|
||||
|
||||
self.assertEqual(2, len(result.rows))
|
||||
self.assertEqual(3, result.total_rows)
|
||||
self.assertTrue(result.truncated)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
34
tests/test_manifest.py
Normal file
34
tests/test_manifest.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest
|
||||
from govoplan_dataflow.backend.manifest import get_manifest
|
||||
|
||||
|
||||
class DataflowManifestTests(unittest.TestCase):
|
||||
def test_manifest_declares_independent_dataflow_boundary(self) -> None:
|
||||
manifest = get_manifest()
|
||||
|
||||
self.assertIsInstance(manifest, ModuleManifest)
|
||||
self.assertEqual(manifest.id, "dataflow")
|
||||
self.assertEqual(manifest.dependencies, ())
|
||||
self.assertIn("connectors", manifest.optional_dependencies)
|
||||
self.assertIn("reporting", manifest.optional_dependencies)
|
||||
self.assertIn("workflow", manifest.optional_dependencies)
|
||||
self.assertEqual(manifest.frontend.package_name, "@govoplan/dataflow-webui")
|
||||
self.assertIsNotNone(manifest.route_factory)
|
||||
self.assertIsNotNone(manifest.migration_spec)
|
||||
self.assertEqual(
|
||||
{
|
||||
"dataflow.pipeline_catalog",
|
||||
"dataflow.pipeline_preview",
|
||||
"dataflow.run_lifecycle",
|
||||
"dataflow.dataset_output",
|
||||
},
|
||||
{item.name for item in manifest.provides_interfaces},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
47
tests/test_migrations.py
Normal file
47
tests/test_migrations.py
Normal file
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from sqlalchemy import create_engine, inspect
|
||||
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
from govoplan_dataflow.backend.manifest import get_manifest
|
||||
|
||||
|
||||
class DataflowMigrationTests(unittest.TestCase):
|
||||
def test_baseline_creates_dataflow_tables_and_head(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-dataflow-migration-") as directory:
|
||||
url = f"sqlite:///{Path(directory) / 'dataflow.db'}"
|
||||
migrate_database(
|
||||
database_url=url,
|
||||
enabled_modules=("dataflow",),
|
||||
manifest_factories=(get_manifest,),
|
||||
)
|
||||
engine = create_engine(url)
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
self.assertIn(
|
||||
"d4f7a1c8e2b0",
|
||||
set(MigrationContext.configure(connection).get_current_heads()),
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"dataflow_pipelines",
|
||||
"dataflow_pipeline_revisions",
|
||||
"dataflow_runs",
|
||||
},
|
||||
{
|
||||
name
|
||||
for name in inspect(connection).get_table_names()
|
||||
if name.startswith("dataflow_")
|
||||
},
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
198
tests/test_service.py
Normal file
198
tests/test_service.py
Normal file
@@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine, func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_dataflow.backend.db.models import (
|
||||
DataflowPipeline,
|
||||
DataflowPipelineRevision,
|
||||
DataflowRun,
|
||||
)
|
||||
from govoplan_dataflow.backend.schemas import (
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
GraphPosition,
|
||||
PipelineCreateRequest,
|
||||
PipelineGraph,
|
||||
PipelinePreviewRequest,
|
||||
PipelineUpdateRequest,
|
||||
)
|
||||
from govoplan_dataflow.backend.service import (
|
||||
DataflowConflictError,
|
||||
DataflowNotFoundError,
|
||||
create_pipeline,
|
||||
get_pipeline,
|
||||
preview_pipeline,
|
||||
update_pipeline,
|
||||
)
|
||||
|
||||
|
||||
def sample_graph(*, minimum: int = 10) -> PipelineGraph:
|
||||
return PipelineGraph(
|
||||
nodes=[
|
||||
GraphNode(
|
||||
id="source",
|
||||
type="source.inline",
|
||||
label="Monthly input",
|
||||
position=GraphPosition(x=40, y=160),
|
||||
config={
|
||||
"source_name": "monthly_files",
|
||||
"rows": [
|
||||
{"id": 1, "amount": 5},
|
||||
{"id": 2, "amount": 15},
|
||||
{"id": 3, "amount": 25},
|
||||
],
|
||||
},
|
||||
),
|
||||
GraphNode(
|
||||
id="filter",
|
||||
type="filter",
|
||||
label="Minimum amount",
|
||||
position=GraphPosition(x=260, y=160),
|
||||
config={"column": "amount", "operator": "gte", "value": minimum},
|
||||
),
|
||||
GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=480, y=160),
|
||||
config={},
|
||||
),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(id="source-filter", source="source", target="filter"),
|
||||
GraphEdge(id="filter-output", source="filter", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class DataflowServiceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=[
|
||||
DataflowPipeline.__table__,
|
||||
DataflowPipelineRevision.__table__,
|
||||
DataflowRun.__table__,
|
||||
],
|
||||
)
|
||||
self.Session = sessionmaker(bind=self.engine)
|
||||
self.session: Session = self.Session()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
Base.metadata.drop_all(
|
||||
self.engine,
|
||||
tables=[
|
||||
DataflowRun.__table__,
|
||||
DataflowPipelineRevision.__table__,
|
||||
DataflowPipeline.__table__,
|
||||
],
|
||||
)
|
||||
self.engine.dispose()
|
||||
|
||||
def _create(self, *, tenant_id: str = "tenant-1") -> DataflowPipeline:
|
||||
pipeline = create_pipeline(
|
||||
self.session,
|
||||
tenant_id=tenant_id,
|
||||
actor_id="user-1",
|
||||
payload=PipelineCreateRequest(
|
||||
name="Monthly comparison",
|
||||
description="First governed pipeline",
|
||||
graph=sample_graph(),
|
||||
editor_mode="graph",
|
||||
),
|
||||
)
|
||||
self.session.commit()
|
||||
return pipeline
|
||||
|
||||
def test_create_and_update_make_immutable_revisions(self) -> None:
|
||||
pipeline = self._create()
|
||||
|
||||
updated = update_pipeline(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
pipeline_id=pipeline.id,
|
||||
actor_id="user-2",
|
||||
payload=PipelineUpdateRequest(
|
||||
name="Monthly comparison",
|
||||
description="Changed threshold",
|
||||
graph=sample_graph(minimum=20),
|
||||
editor_mode="graph",
|
||||
expected_revision=1,
|
||||
),
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
revisions = list(
|
||||
self.session.scalars(
|
||||
select(DataflowPipelineRevision)
|
||||
.where(DataflowPipelineRevision.pipeline_id == pipeline.id)
|
||||
.order_by(DataflowPipelineRevision.revision)
|
||||
)
|
||||
)
|
||||
self.assertEqual(2, updated.current_revision)
|
||||
self.assertEqual([1, 2], [item.revision for item in revisions])
|
||||
self.assertNotEqual(revisions[0].content_hash, revisions[1].content_hash)
|
||||
self.assertEqual(10, revisions[0].graph["nodes"][1]["config"]["value"])
|
||||
self.assertEqual(20, revisions[1].graph["nodes"][1]["config"]["value"])
|
||||
|
||||
def test_stale_revision_is_rejected(self) -> None:
|
||||
pipeline = self._create()
|
||||
|
||||
with self.assertRaises(DataflowConflictError):
|
||||
update_pipeline(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
pipeline_id=pipeline.id,
|
||||
actor_id="user-2",
|
||||
payload=PipelineUpdateRequest(
|
||||
name="Stale edit",
|
||||
graph=sample_graph(minimum=30),
|
||||
editor_mode="graph",
|
||||
expected_revision=2,
|
||||
),
|
||||
)
|
||||
|
||||
def test_pipeline_lookup_is_tenant_isolated(self) -> None:
|
||||
pipeline = self._create()
|
||||
|
||||
with self.assertRaises(DataflowNotFoundError):
|
||||
get_pipeline(
|
||||
self.session,
|
||||
tenant_id="tenant-2",
|
||||
pipeline_id=pipeline.id,
|
||||
)
|
||||
|
||||
def test_saved_preview_records_lineage_but_not_result_rows(self) -> None:
|
||||
pipeline = self._create()
|
||||
|
||||
response = preview_pipeline(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
actor_id="user-1",
|
||||
payload=PipelinePreviewRequest(pipeline_id=pipeline.id, row_limit=1),
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
run = self.session.scalar(select(DataflowRun).where(DataflowRun.id == response.run_id))
|
||||
self.assertEqual("succeeded", response.status)
|
||||
self.assertEqual([{"id": 2, "amount": 15}], response.rows)
|
||||
self.assertEqual(2, response.total_rows)
|
||||
self.assertTrue(response.truncated)
|
||||
self.assertEqual(2, run.output_row_count)
|
||||
self.assertEqual(3, run.input_row_count)
|
||||
self.assertEqual(1, len(run.source_fingerprints))
|
||||
self.assertFalse(hasattr(run, "result_rows"))
|
||||
self.assertEqual(
|
||||
1,
|
||||
self.session.scalar(select(func.count()).select_from(DataflowRun)),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user