feat(dataflow): model production SQL flow patterns
This commit is contained in:
@@ -57,6 +57,196 @@ class DataflowOperatorTests(unittest.TestCase):
|
||||
with self.assertRaises(ExpressionError):
|
||||
parse_expression("(select secret from credentials)")
|
||||
|
||||
def test_expression_vocabulary_covers_conditional_text_and_dates(self) -> None:
|
||||
row = {
|
||||
"code": "A-007",
|
||||
"name": " Ada 42 ",
|
||||
"amount": 12,
|
||||
"created_on": "2026-07-30",
|
||||
"started_on": "2026-07-01",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
"00007",
|
||||
evaluate_expression(
|
||||
"lpad(split_part(code, '-', 2), 5, '0')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"Ada",
|
||||
evaluate_expression(
|
||||
"regexp_replace(name, '[^A-Za-z]', '', 'g')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertTrue(
|
||||
evaluate_expression(
|
||||
"code like 'A-%' and amount between 10 and 20",
|
||||
row,
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
"30.07.2026",
|
||||
evaluate_expression(
|
||||
"to_char(created_on, 'DD.MM.YYYY')",
|
||||
row,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
29,
|
||||
evaluate_expression(
|
||||
"date_diff('day', started_on, created_on)",
|
||||
row,
|
||||
),
|
||||
)
|
||||
|
||||
def test_ordered_calculations_and_partitioned_rank_execute(self) -> None:
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"source",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "records",
|
||||
"rows": [
|
||||
{"group": "A", "amount": 10},
|
||||
{"group": "A", "amount": 20},
|
||||
{"group": "A", "amount": 20},
|
||||
{"group": "B", "amount": 5},
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"calculate",
|
||||
"calculate",
|
||||
{
|
||||
"calculations": [
|
||||
{
|
||||
"target_column": "gross",
|
||||
"expression": "amount * 2",
|
||||
"result_type": "integer",
|
||||
},
|
||||
{
|
||||
"target_column": "band",
|
||||
"expression": (
|
||||
"case when gross >= 40 then 'high' "
|
||||
"else 'standard' end"
|
||||
),
|
||||
"result_type": "string",
|
||||
},
|
||||
]
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
node(
|
||||
"rank",
|
||||
"window.rank",
|
||||
{
|
||||
"method": "rank",
|
||||
"target_column": "group_rank",
|
||||
"partition_by": ["group"],
|
||||
"order_by": [
|
||||
{"column": "gross", "direction": "desc"}
|
||||
],
|
||||
},
|
||||
x=400,
|
||||
),
|
||||
node("output", "output", {}, x=600),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(id="e1", source="source", target="calculate"),
|
||||
GraphEdge(id="e2", source="calculate", target="rank"),
|
||||
GraphEdge(id="e3", source="rank", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
[item for item in validate_graph(graph) if item.severity == "error"]
|
||||
)
|
||||
result = execute_preview(graph, row_limit=100)
|
||||
|
||||
self.assertEqual([3, 1, 1, 1], [
|
||||
item["group_rank"] for item in result.rows
|
||||
])
|
||||
self.assertEqual(
|
||||
["standard", "high", "high", "standard"],
|
||||
[item["band"] for item in result.rows],
|
||||
)
|
||||
|
||||
def test_semi_and_anti_joins_return_left_rows_once(self) -> None:
|
||||
for join_type, expected in (
|
||||
("semi", [{"id": "A"}, {"id": "C"}]),
|
||||
("anti", [{"id": "B"}]),
|
||||
):
|
||||
with self.subTest(join_type=join_type):
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"left",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "left_records",
|
||||
"rows": [{"id": "A"}, {"id": "B"}, {"id": "C"}],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"right",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "right_records",
|
||||
"rows": [
|
||||
{"id": "A"},
|
||||
{"id": "A"},
|
||||
{"id": "C"},
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
node(
|
||||
"join",
|
||||
"combine.join",
|
||||
{
|
||||
"join_type": join_type,
|
||||
"left_keys": ["id"],
|
||||
"right_keys": ["id"],
|
||||
"right_prefix": "right_",
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
node("output", "output", {}, x=400),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(
|
||||
id="e1",
|
||||
source="left",
|
||||
target="join",
|
||||
target_port="left",
|
||||
),
|
||||
GraphEdge(
|
||||
id="e2",
|
||||
source="right",
|
||||
target="join",
|
||||
target_port="right",
|
||||
),
|
||||
GraphEdge(id="e3", source="join", target="output"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
[
|
||||
item
|
||||
for item in validate_graph(graph)
|
||||
if item.severity == "error"
|
||||
]
|
||||
)
|
||||
self.assertEqual(
|
||||
expected,
|
||||
execute_preview(graph, row_limit=100).rows,
|
||||
)
|
||||
|
||||
def test_conversion_expression_and_quality_nodes_execute(self) -> None:
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
@@ -362,6 +552,89 @@ class DataflowOperatorTests(unittest.TestCase):
|
||||
|
||||
self.assertIn("LOWER(name) AS normalized_name", sql)
|
||||
|
||||
def test_calculation_and_rank_nodes_render_as_sql(self) -> None:
|
||||
for transform, expected_sql in (
|
||||
(
|
||||
node(
|
||||
"calculate",
|
||||
"calculate",
|
||||
{
|
||||
"calculations": [
|
||||
{
|
||||
"target_column": "gross",
|
||||
"expression": "amount * 2",
|
||||
"result_type": "integer",
|
||||
},
|
||||
{
|
||||
"target_column": "band",
|
||||
"expression": (
|
||||
"case when amount >= 10 then 'high' "
|
||||
"else 'standard' end"
|
||||
),
|
||||
"result_type": "string",
|
||||
},
|
||||
]
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
("amount * 2 AS gross", "CASE WHEN amount >= 10"),
|
||||
),
|
||||
(
|
||||
node(
|
||||
"rank",
|
||||
"window.rank",
|
||||
{
|
||||
"method": "row_number",
|
||||
"target_column": "position",
|
||||
"partition_by": ["group"],
|
||||
"order_by": [
|
||||
{"column": "amount", "direction": "desc"}
|
||||
],
|
||||
},
|
||||
x=200,
|
||||
),
|
||||
(
|
||||
"ROW_NUMBER() OVER",
|
||||
"PARTITION BY group ORDER BY amount DESC",
|
||||
),
|
||||
),
|
||||
):
|
||||
with self.subTest(node_type=transform.type):
|
||||
graph = PipelineGraph(
|
||||
nodes=[
|
||||
node(
|
||||
"source",
|
||||
"source.inline",
|
||||
{
|
||||
"source_name": "records",
|
||||
"rows": [
|
||||
{"group": "A", "amount": 12}
|
||||
],
|
||||
},
|
||||
x=0,
|
||||
),
|
||||
transform,
|
||||
node("output", "output", {}, x=400),
|
||||
],
|
||||
edges=[
|
||||
GraphEdge(
|
||||
id="e1",
|
||||
source="source",
|
||||
target=transform.id,
|
||||
),
|
||||
GraphEdge(
|
||||
id="e2",
|
||||
source=transform.id,
|
||||
target="output",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
sql, _ = render_sql(graph)
|
||||
|
||||
for fragment in expected_sql:
|
||||
self.assertIn(fragment, sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user