feat(dataflow): model production SQL flow patterns
This commit is contained in:
@@ -21,10 +21,12 @@ class DataflowNodeLibraryTests(unittest.TestCase):
|
||||
"select",
|
||||
"derive",
|
||||
"expression",
|
||||
"calculate",
|
||||
"convert",
|
||||
"replace",
|
||||
"aggregate",
|
||||
"sort",
|
||||
"window.rank",
|
||||
"limit",
|
||||
"quality.rules",
|
||||
"reconcile.compare",
|
||||
|
||||
@@ -57,6 +57,196 @@ class DataflowOperatorTests(unittest.TestCase):
|
||||
with self.assertRaises(ExpressionError):
|
||||
parse_expression("(select secret from credentials)")
|
||||
|
||||
def test_expression_vocabulary_covers_conditional_text_and_dates(self) -> None:
|
||||
row = {
|
||||
"code": "A-007",
|
||||
"name": " Ada 42 ",
|
||||
"amount": 12,
|
||||
"created_on": "2026-07-30",
|
||||
"started_on": "2026-07-01",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
"00007",
|
||||
evaluate_expression(
|
||||
"lpad(split_part(code, '-', 2), 5, '0')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"Ada",
|
||||
evaluate_expression(
|
||||
"regexp_replace(name, '[^A-Za-z]', '', 'g')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertTrue(
|
||||
evaluate_expression(
|
||||
"code like 'A-%' and amount between 10 and 20",
|
||||
row,
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
"30.07.2026",
|
||||
evaluate_expression(
|
||||
"to_char(created_on, 'DD.MM.YYYY')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
29,
|
||||
evaluate_expression(
|
||||
"date_diff('day', started_on, created_on)",
|
||||
row,
|
||||
),
|
||||
)
|
||||
|
||||
def test_ordered_calculations_and_partitioned_rank_execute(self) -> None:
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"source",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "records",
|
||||
"rows": [
|
||||
{"group": "A", "amount": 10},
|
||||
{"group": "A", "amount": 20},
|
||||
{"group": "A", "amount": 20},
|
||||
{"group": "B", "amount": 5},
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"calculate",
|
||||
"calculate",
|
||||
{
|
||||
"calculations": [
|
||||
{
|
||||
"target_column": "gross",
|
||||
"expression": "amount * 2",
|
||||
"result_type": "integer",
|
||||
},
|
||||
{
|
||||
"target_column": "band",
|
||||
"expression": (
|
||||
"case when gross >= 40 then 'high' "
|
||||
"else 'standard' end"
|
||||
),
|
||||
"result_type": "string",
|
||||
},
|
||||
]
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
node(
|
||||
"rank",
|
||||
"window.rank",
|
||||
{
|
||||
"method": "rank",
|
||||
"target_column": "group_rank",
|
||||
"partition_by": ["group"],
|
||||
"order_by": [
|
||||
{"column": "gross", "direction": "desc"}
|
||||
],
|
||||
},
|
||||
x=400,
|
||||
),
|
||||
node("output", "output", {}, x=600),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(id="e1", source="source", target="calculate"),
|
||||
GraphEdge(id="e2", source="calculate", target="rank"),
|
||||
GraphEdge(id="e3", source="rank", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
[item for item in validate_graph(graph) if item.severity == "error"]
|
||||
)
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
|
||||
self.assertEqual([3, 1, 1, 1], [
|
||||
item["group_rank"] for item in result.rows
|
||||
])
|
||||
self.assertEqual(
|
||||
["standard", "high", "high", "standard"],
|
||||
[item["band"] for item in result.rows],
|
||||
)
|
||||
|
||||
def test_semi_and_anti_joins_return_left_rows_once(self) -> None:
|
||||
for join_type, expected in (
|
||||
("semi", [{"id": "A"}, {"id": "C"}]),
|
||||
("anti", [{"id": "B"}]),
|
||||
):
|
||||
with self.subTest(join_type=join_type):
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"left",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "left_records",
|
||||
"rows": [{"id": "A"}, {"id": "B"}, {"id": "C"}],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"right",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "right_records",
|
||||
"rows": [
|
||||
{"id": "A"},
|
||||
{"id": "A"},
|
||||
{"id": "C"},
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"join",
|
||||
"combine.join",
|
||||
{
|
||||
"join_type": join_type,
|
||||
"left_keys": ["id"],
|
||||
"right_keys": ["id"],
|
||||
"right_prefix": "right_",
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
node("output", "output", {}, x=400),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(
|
||||
id="e1",
|
||||
source="left",
|
||||
target="join",
|
||||
target_port="left",
|
||||
),
|
||||
GraphEdge(
|
||||
id="e2",
|
||||
source="right",
|
||||
target="join",
|
||||
target_port="right",
|
||||
),
|
||||
GraphEdge(id="e3", source="join", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
[
|
||||
item
|
||||
for item in validate_graph(graph)
|
||||
if item.severity == "error"
|
||||
]
|
||||
)
|
||||
self.assertEqual(
|
||||
expected,
|
||||
execute_preview(graph, row_limit=100).rows,
|
||||
)
|
||||
|
||||
def test_conversion_expression_and_quality_nodes_execute(self) -> None:
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
@@ -362,6 +552,89 @@ class DataflowOperatorTests(unittest.TestCase):
|
||||
|
||||
self.assertIn("LOWER(name) AS normalized_name", sql)
|
||||
|
||||
def test_calculation_and_rank_nodes_render_as_sql(self) -> None:
|
||||
for transform, expected_sql in (
|
||||
(
|
||||
node(
|
||||
"calculate",
|
||||
"calculate",
|
||||
{
|
||||
"calculations": [
|
||||
{
|
||||
"target_column": "gross",
|
||||
"expression": "amount * 2",
|
||||
"result_type": "integer",
|
||||
},
|
||||
{
|
||||
"target_column": "band",
|
||||
"expression": (
|
||||
"case when amount >= 10 then 'high' "
|
||||
"else 'standard' end"
|
||||
),
|
||||
"result_type": "string",
|
||||
},
|
||||
]
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
("amount * 2 AS gross", "CASE WHEN amount >= 10"),
|
||||
),
|
||||
(
|
||||
node(
|
||||
"rank",
|
||||
"window.rank",
|
||||
{
|
||||
"method": "row_number",
|
||||
"target_column": "position",
|
||||
"partition_by": ["group"],
|
||||
"order_by": [
|
||||
{"column": "amount", "direction": "desc"}
|
||||
],
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
(
|
||||
"ROW_NUMBER() OVER",
|
||||
"PARTITION BY group ORDER BY amount DESC",
|
||||
),
|
||||
),
|
||||
):
|
||||
with self.subTest(node_type=transform.type):
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"source",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "records",
|
||||
"rows": [
|
||||
{"group": "A", "amount": 12}
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
transform,
|
||||
node("output", "output", {}, x=400),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(
|
||||
id="e1",
|
||||
source="source",
|
||||
target=transform.id,
|
||||
),
|
||||
GraphEdge(
|
||||
id="e2",
|
||||
source=transform.id,
|
||||
target="output",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
sql, _ = render_sql(graph)
|
||||
|
||||
for fragment in expected_sql:
|
||||
self.assertIn(fragment, sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -164,7 +164,46 @@ class ExecutionBackendTests(unittest.TestCase):
|
||||
DuckDbExecutionBackend().available(),
|
||||
"analytics extra is not installed",
|
||||
)
|
||||
def test_duckdb_matches_reference_for_aggregate_join_and_union(self) -> None:
|
||||
def test_duckdb_matches_reference_for_supported_operators(self) -> None:
|
||||
calculation_graph = simple_graph()
|
||||
calculation_graph.nodes[1] = calculation_graph.nodes[1].model_copy(
|
||||
update={
|
||||
"type": "calculate",
|
||||
"label": "Calculated fields",
|
||||
"config": {
|
||||
"calculations": [
|
||||
{
|
||||
"target_column": "double_amount",
|
||||
"expression": "amount * 2",
|
||||
"result_type": "integer",
|
||||
},
|
||||
{
|
||||
"target_column": "amount_band",
|
||||
"expression": (
|
||||
"case when amount >= 10 then 'high' "
|
||||
"else 'standard' end"
|
||||
),
|
||||
"result_type": "string",
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
rank_graph = simple_graph()
|
||||
rank_graph.nodes[1] = rank_graph.nodes[1].model_copy(
|
||||
update={
|
||||
"type": "window.rank",
|
||||
"label": "Department rank",
|
||||
"config": {
|
||||
"method": "rank",
|
||||
"target_column": "department_rank",
|
||||
"partition_by": ["department"],
|
||||
"order_by": [
|
||||
{"column": "amount", "direction": "desc"}
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
graphs = [
|
||||
compile_sql(
|
||||
"""
|
||||
@@ -214,6 +253,42 @@ class ExecutionBackendTests(unittest.TestCase):
|
||||
),
|
||||
],
|
||||
)[0],
|
||||
compile_sql(
|
||||
"""
|
||||
SELECT records.department, records.amount
|
||||
FROM records
|
||||
SEMI JOIN labels
|
||||
ON records.department = labels.department
|
||||
ORDER BY records.amount
|
||||
""",
|
||||
source_nodes=[
|
||||
inline_source(),
|
||||
inline_source(
|
||||
"labels-source",
|
||||
"labels",
|
||||
[{"department": "a"}],
|
||||
),
|
||||
],
|
||||
)[0],
|
||||
compile_sql(
|
||||
"""
|
||||
SELECT records.department, records.amount
|
||||
FROM records
|
||||
ANTI JOIN labels
|
||||
ON records.department = labels.department
|
||||
ORDER BY records.amount
|
||||
""",
|
||||
source_nodes=[
|
||||
inline_source(),
|
||||
inline_source(
|
||||
"labels-source",
|
||||
"labels",
|
||||
[{"department": "a"}],
|
||||
),
|
||||
],
|
||||
)[0],
|
||||
calculation_graph,
|
||||
rank_graph,
|
||||
]
|
||||
for graph in graphs:
|
||||
with self.subTest(nodes=[node.type for node in graph.nodes]):
|
||||
|
||||
Reference in New Issue
Block a user