Expand governed Dataflow editor and node library
This commit is contained in:
@@ -2,7 +2,13 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from govoplan_dataflow.backend.executor import PipelineExecutionError, execute_preview
|
||||
from govoplan_dataflow.backend.executor import (
|
||||
MAX_SOURCE_ROWS,
|
||||
PipelineExecutionError,
|
||||
ResolvedSource,
|
||||
execute_preview,
|
||||
)
|
||||
from govoplan_dataflow.backend.graph import validate_graph
|
||||
from govoplan_dataflow.backend.schemas import (
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
@@ -29,7 +35,66 @@ def inline_source() -> GraphNode:
|
||||
)
|
||||
|
||||
|
||||
def lookup_source() -> GraphNode:
|
||||
return GraphNode(
|
||||
id="lookup",
|
||||
type="source.inline",
|
||||
label="Department lookup",
|
||||
position=GraphPosition(x=40, y=300),
|
||||
config={
|
||||
"source_name": "department_lookup",
|
||||
"rows": [
|
||||
{"department": "A", "label": "Administration"},
|
||||
{"department": "B", "label": "Building services"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class DataflowGraphAndSqlTests(unittest.TestCase):
|
||||
def test_rejects_invalid_or_duplicate_logical_source_names(self) -> None:
|
||||
invalid_source = inline_source().model_copy(deep=True)
|
||||
invalid_source.config["source_name"] = "monthly cases"
|
||||
duplicate_source = lookup_source().model_copy(deep=True)
|
||||
duplicate_source.config["source_name"] = "MONTHLY_FILES"
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
invalid_source,
|
||||
duplicate_source,
|
||||
GraphNode(
|
||||
id="union",
|
||||
type="combine.union",
|
||||
label="Append rows",
|
||||
position=GraphPosition(x=250, y=160),
|
||||
config={"mode": "all"},
|
||||
),
|
||||
GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=450, y=160),
|
||||
config={},
|
||||
),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(id="edge-1", source="source", target="union"),
|
||||
GraphEdge(id="edge-2", source="lookup", target="union"),
|
||||
GraphEdge(id="edge-3", source="union", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
diagnostics = validate_graph(graph)
|
||||
|
||||
self.assertTrue(any(item.code == "source.name_invalid" for item in diagnostics))
|
||||
invalid_source.config["source_name"] = "monthly_files"
|
||||
diagnostics = validate_graph(
|
||||
graph.model_copy(
|
||||
update={"nodes": [invalid_source, *graph.nodes[1:]]},
|
||||
deep=True,
|
||||
)
|
||||
)
|
||||
self.assertTrue(any(item.code == "source.duplicate_name" for item in diagnostics))
|
||||
|
||||
def test_compiles_renders_and_executes_grouped_query(self) -> None:
|
||||
sql = """
|
||||
SELECT department, COUNT(*) AS records, SUM(amount) AS total
|
||||
@@ -71,13 +136,207 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
||||
with self.subTest(sql=sql), self.assertRaises(SqlCompilationError):
|
||||
compile_sql(sql, source_nodes=[inline_source()])
|
||||
|
||||
def test_rejects_join_until_comparison_slice(self) -> None:
|
||||
with self.assertRaisesRegex(SqlCompilationError, "JOIN support"):
|
||||
def test_compiles_renders_and_executes_two_source_join(self) -> None:
|
||||
sql = """
|
||||
SELECT monthly_files.department, lookup.label
|
||||
FROM monthly_files
|
||||
LEFT JOIN department_lookup AS lookup
|
||||
ON monthly_files.department = lookup.department
|
||||
ORDER BY monthly_files.department
|
||||
"""
|
||||
|
||||
graph, _, diagnostics = compile_sql(
|
||||
sql,
|
||||
source_nodes=[inline_source(), lookup_source()],
|
||||
)
|
||||
|
||||
self.assertEqual([], diagnostics)
|
||||
self.assertEqual(
|
||||
[
|
||||
"source.inline",
|
||||
"source.inline",
|
||||
"combine.join",
|
||||
"select",
|
||||
"sort",
|
||||
"output",
|
||||
],
|
||||
[node.type for node in graph.nodes],
|
||||
)
|
||||
self.assertEqual("lookup_", graph.nodes[2].config["right_prefix"])
|
||||
rendered, render_diagnostics = render_sql(graph)
|
||||
self.assertEqual([], render_diagnostics)
|
||||
roundtrip, _, _ = compile_sql(
|
||||
rendered,
|
||||
source_nodes=[inline_source(), lookup_source()],
|
||||
)
|
||||
self.assertEqual(
|
||||
graph.model_dump(mode="json"),
|
||||
roundtrip.model_dump(mode="json"),
|
||||
)
|
||||
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
self.assertEqual(
|
||||
[
|
||||
{"department": "A", "lookup_label": "Administration"},
|
||||
{"department": "A", "lookup_label": "Administration"},
|
||||
{"department": "B", "lookup_label": "Building services"},
|
||||
],
|
||||
result.rows,
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(SqlCompilationError, "source-qualified"):
|
||||
compile_sql(
|
||||
"SELECT * FROM monthly_files JOIN other ON monthly_files.id = other.id",
|
||||
source_nodes=[inline_source()],
|
||||
"""
|
||||
SELECT label
|
||||
FROM monthly_files
|
||||
JOIN department_lookup AS lookup
|
||||
ON monthly_files.department = lookup.department
|
||||
""",
|
||||
source_nodes=[inline_source(), lookup_source()],
|
||||
)
|
||||
|
||||
def test_distinct_and_derived_columns_are_deterministic(self) -> None:
|
||||
source = inline_source().model_copy(
|
||||
update={
|
||||
"config": {
|
||||
"source_name": "monthly_files",
|
||||
"rows": [
|
||||
{"first": " Ada ", "last": "Lovelace"},
|
||||
{"first": " Ada ", "last": "Lovelace"},
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
derive = GraphNode(
|
||||
id="derive",
|
||||
type="derive",
|
||||
label="Display name",
|
||||
position=GraphPosition(x=260, y=160),
|
||||
config={
|
||||
"target_column": "display_name",
|
||||
"operation": "concat",
|
||||
"source_columns": ["first", "last"],
|
||||
"separator": " ",
|
||||
},
|
||||
)
|
||||
distinct = GraphNode(
|
||||
id="distinct",
|
||||
type="distinct",
|
||||
label="Unique people",
|
||||
position=GraphPosition(x=480, y=160),
|
||||
config={"columns": ["display_name"]},
|
||||
)
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=700, y=160),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[source, derive, distinct, output],
|
||||
edges=[
|
||||
GraphEdge(id="source-derive", source=source.id, target=derive.id),
|
||||
GraphEdge(id="derive-distinct", source=derive.id, target=distinct.id),
|
||||
GraphEdge(id="distinct-output", source=distinct.id, target=output.id),
|
||||
],
|
||||
)
|
||||
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
|
||||
self.assertEqual(
|
||||
[{"first": " Ada ", "last": "Lovelace", "display_name": " Ada Lovelace"}],
|
||||
result.rows,
|
||||
)
|
||||
|
||||
def test_union_can_keep_or_remove_duplicate_rows(self) -> None:
|
||||
first = inline_source().model_copy(
|
||||
update={
|
||||
"id": "first",
|
||||
"config": {"source_name": "first", "rows": [{"id": 1}, {"id": 2}]},
|
||||
}
|
||||
)
|
||||
second = inline_source().model_copy(
|
||||
update={
|
||||
"id": "second",
|
||||
"config": {"source_name": "second", "rows": [{"id": 2}, {"id": 3}]},
|
||||
}
|
||||
)
|
||||
union = GraphNode(
|
||||
id="union",
|
||||
type="combine.union",
|
||||
label="Append",
|
||||
position=GraphPosition(x=300, y=200),
|
||||
config={"mode": "distinct"},
|
||||
)
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=520, y=200),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[first, second, union, output],
|
||||
edges=[
|
||||
GraphEdge(id="first-union", source=first.id, target=union.id),
|
||||
GraphEdge(id="second-union", source=second.id, target=union.id),
|
||||
GraphEdge(id="union-output", source=union.id, target=output.id),
|
||||
],
|
||||
)
|
||||
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
|
||||
self.assertEqual([{"id": 1}, {"id": 2}, {"id": 3}], result.rows)
|
||||
|
||||
def test_connector_source_uses_bounded_resolver_and_records_lineage(self) -> None:
|
||||
source = GraphNode(
|
||||
id="connector",
|
||||
type="source.reference",
|
||||
label="Monthly import",
|
||||
position=GraphPosition(x=40, y=160),
|
||||
config={
|
||||
"source_ref": "snapshot:source-1",
|
||||
"source_name": "monthly_import",
|
||||
"expected_fingerprint": "sha256:expected",
|
||||
},
|
||||
)
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=280, y=160),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[source, output],
|
||||
edges=[GraphEdge(id="source-output", source=source.id, target=output.id)],
|
||||
)
|
||||
calls: list[tuple[str, int]] = []
|
||||
|
||||
def resolve(node: GraphNode, limit: int) -> ResolvedSource:
|
||||
calls.append((node.id, limit))
|
||||
return ResolvedSource(
|
||||
rows=({"id": 1}, {"id": 2}),
|
||||
source_ref="snapshot:source-1",
|
||||
provider="snapshot",
|
||||
fingerprint="sha256:expected",
|
||||
total_rows=20,
|
||||
truncated=True,
|
||||
)
|
||||
|
||||
result = execute_preview(
|
||||
graph,
|
||||
row_limit=100,
|
||||
source_resolver=resolve,
|
||||
)
|
||||
|
||||
self.assertEqual([("connector", MAX_SOURCE_ROWS)], calls)
|
||||
self.assertEqual([{"id": 1}, {"id": 2}], result.rows)
|
||||
self.assertEqual(20, result.source_fingerprints[0]["row_count"])
|
||||
self.assertTrue(result.source_fingerprints[0]["truncated"])
|
||||
self.assertEqual("source.preview_truncated", result.diagnostics[0].code)
|
||||
|
||||
def test_graph_validation_rejects_cycle(self) -> None:
|
||||
source = inline_source()
|
||||
output = GraphNode(
|
||||
@@ -95,11 +354,78 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
from govoplan_dataflow.backend.graph import validate_graph
|
||||
|
||||
codes = {item.code for item in validate_graph(graph)}
|
||||
self.assertIn("graph.cycle", codes)
|
||||
|
||||
def test_schema_validation_reports_unknown_and_colliding_columns(self) -> None:
|
||||
source = inline_source().model_copy(
|
||||
update={
|
||||
"config": {
|
||||
"source_name": "monthly_files",
|
||||
"rows": [{"id": 1, "right_id": "reserved"}],
|
||||
}
|
||||
}
|
||||
)
|
||||
lookup = lookup_source().model_copy(
|
||||
update={
|
||||
"config": {
|
||||
"source_name": "department_lookup",
|
||||
"rows": [{"id": 1}],
|
||||
}
|
||||
}
|
||||
)
|
||||
join = GraphNode(
|
||||
id="join",
|
||||
type="combine.join",
|
||||
label="Join",
|
||||
position=GraphPosition(x=280, y=200),
|
||||
config={
|
||||
"join_type": "inner",
|
||||
"left_keys": ["missing_id"],
|
||||
"right_keys": ["id"],
|
||||
"right_prefix": "right_",
|
||||
},
|
||||
)
|
||||
output = GraphNode(
|
||||
id="output",
|
||||
type="output",
|
||||
label="Output",
|
||||
position=GraphPosition(x=500, y=200),
|
||||
config={},
|
||||
)
|
||||
graph = PipelineGraph(
|
||||
nodes=[source, lookup, join, output],
|
||||
edges=[
|
||||
GraphEdge(
|
||||
id="source-join",
|
||||
source=source.id,
|
||||
target=join.id,
|
||||
target_port="left",
|
||||
),
|
||||
GraphEdge(
|
||||
id="lookup-join",
|
||||
source=lookup.id,
|
||||
target=join.id,
|
||||
target_port="right",
|
||||
),
|
||||
GraphEdge(id="join-output", source=join.id, target=output.id),
|
||||
],
|
||||
)
|
||||
|
||||
diagnostics = validate_graph(graph)
|
||||
|
||||
self.assertTrue(
|
||||
any(
|
||||
item.code == "schema.unknown_column"
|
||||
and item.node_id == join.id
|
||||
and item.field == "left_keys"
|
||||
for item in diagnostics
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
any(item.code == "join.output_collision" for item in diagnostics)
|
||||
)
|
||||
|
||||
def test_preview_truncates_response_without_changing_pipeline_limit(self) -> None:
|
||||
source = inline_source()
|
||||
output = GraphNode(
|
||||
@@ -202,6 +528,14 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual("filter-1", raised.exception.node_id)
|
||||
self.assertIn("Cannot apply", str(raised.exception))
|
||||
self.assertEqual(
|
||||
[("source", "succeeded"), ("filter-1", "failed")],
|
||||
[
|
||||
(diagnostic.node_id, diagnostic.status)
|
||||
for diagnostic in raised.exception.node_diagnostics
|
||||
],
|
||||
)
|
||||
self.assertEqual(1, len(raised.exception.source_fingerprints))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user