from __future__ import annotations import unittest from govoplan_dataflow.backend.executor import execute_preview from govoplan_dataflow.backend.expressions import ( ExpressionError, evaluate_expression, infer_expression_type, parse_expression, ) from govoplan_dataflow.backend.graph import validate_graph from govoplan_dataflow.backend.node_library import NODE_TYPES from govoplan_dataflow.backend.operator_registry import OPERATOR_REGISTRY from govoplan_dataflow.backend.schemas import ( GraphEdge, GraphNode, GraphPosition, PipelineGraph, ) from govoplan_dataflow.backend.sql_compiler import render_sql def node( node_id: str, node_type: str, config: dict, *, x: int, ) -> GraphNode: return GraphNode( id=node_id, type=node_type, label=node_id, position=GraphPosition(x=x, y=100), config=config, ) class DataflowOperatorTests(unittest.TestCase): def test_typed_expression_is_safe_and_deterministic(self) -> None: parsed = parse_expression( "case when amount >= 10 then lower(trim(name)) else 'small' end" ) self.assertEqual( "ada", evaluate_expression(parsed, {"amount": 12, "name": " ADA "}), ) self.assertEqual( "string", infer_expression_type( parsed, {"amount": "integer", "name": "string"}, ), ) with self.assertRaises(ExpressionError): parse_expression("(select secret from credentials)") def test_expression_vocabulary_covers_conditional_text_and_dates(self) -> None: row = { "code": "A-007", "name": " Ada 42 ", "amount": 12, "created_on": "2026-07-30", "started_on": "2026-07-01", } self.assertEqual( "00007", evaluate_expression( "lpad(split_part(code, '-', 2), 5, '0')", row, ), ) self.assertEqual( "Ada", evaluate_expression( "regexp_replace(name, '[^A-Za-z]', '', 'g')", row, ), ) self.assertTrue( evaluate_expression( "code like 'A-%' and amount between 10 and 20", row, ) ) self.assertEqual( "30.07.2026", evaluate_expression( "to_char(created_on, 'DD.MM.YYYY')", row, ), ) self.assertEqual( 29, evaluate_expression( "date_diff('day', started_on, created_on)", row, ), ) def test_ordered_calculations_and_partitioned_rank_execute(self) -> None: graph = PipelineGraph( nodes=[ node( "source", "source.inline", { "source_name": "records", "rows": [ {"group": "A", "amount": 10}, {"group": "A", "amount": 20}, {"group": "A", "amount": 20}, {"group": "B", "amount": 5}, ], }, x=0, ), node( "calculate", "calculate", { "calculations": [ { "target_column": "gross", "expression": "amount * 2", "result_type": "integer", }, { "target_column": "band", "expression": ( "case when gross >= 40 then 'high' " "else 'standard' end" ), "result_type": "string", }, ] }, x=200, ), node( "rank", "window.rank", { "method": "rank", "target_column": "group_rank", "partition_by": ["group"], "order_by": [ {"column": "gross", "direction": "desc"} ], }, x=400, ), node("output", "output", {}, x=600), ], edges=[ GraphEdge(id="e1", source="source", target="calculate"), GraphEdge(id="e2", source="calculate", target="rank"), GraphEdge(id="e3", source="rank", target="output"), ], ) self.assertFalse( [item for item in validate_graph(graph) if item.severity == "error"] ) result = execute_preview(graph, row_limit=100) self.assertEqual([3, 1, 1, 1], [ item["group_rank"] for item in result.rows ]) self.assertEqual( ["standard", "high", "high", "standard"], [item["band"] for item in result.rows], ) def test_semi_and_anti_joins_return_left_rows_once(self) -> None: for join_type, expected in ( ("semi", [{"id": "A"}, {"id": "C"}]), ("anti", [{"id": "B"}]), ): with self.subTest(join_type=join_type): graph = PipelineGraph( nodes=[ node( "left", "source.inline", { "source_name": "left_records", "rows": [{"id": "A"}, {"id": "B"}, {"id": "C"}], }, x=0, ), node( "right", "source.inline", { "source_name": "right_records", "rows": [ {"id": "A"}, {"id": "A"}, {"id": "C"}, ], }, x=0, ), node( "join", "combine.join", { "join_type": join_type, "left_keys": ["id"], "right_keys": ["id"], "right_prefix": "right_", }, x=200, ), node("output", "output", {}, x=400), ], edges=[ GraphEdge( id="e1", source="left", target="join", target_port="left", ), GraphEdge( id="e2", source="right", target="join", target_port="right", ), GraphEdge(id="e3", source="join", target="output"), ], ) self.assertFalse( [ item for item in validate_graph(graph) if item.severity == "error" ] ) self.assertEqual( expected, execute_preview(graph, row_limit=100).rows, ) def test_conversion_expression_and_quality_nodes_execute(self) -> None: graph = PipelineGraph( nodes=[ node( "source", "source.inline", { "source_name": "input", "rows": [ {"id": "1", "amount": "12.50", "name": " Ada "}, {"id": "2", "amount": "bad", "name": ""}, ], }, x=0, ), node( "convert", "convert", { "source_column": "amount", "target_column": "amount_number", "target_type": "number", "on_error": "null", }, x=200, ), node( "expression", "expression", { "target_column": "normalized_name", "expression": "lower(trim(name))", "result_type": "string", }, x=400, ), node( "quality", "quality.rules", { "rules": [ { "id": "amount-required", "column": "amount_number", "operator": "not_null", }, { "id": "name-required", "column": "normalized_name", "operator": "not_null", }, ], "action": "annotate", }, x=600, ), node("output", "output", {}, x=800), ], edges=[ GraphEdge(id="e1", source="source", target="convert"), GraphEdge(id="e2", source="convert", target="expression"), GraphEdge(id="e3", source="expression", target="quality"), GraphEdge(id="e4", source="quality", target="output"), ], ) self.assertFalse( [item for item in validate_graph(graph) if item.severity == "error"] ) result = execute_preview(graph, row_limit=100) self.assertEqual("ada", result.rows[0]["normalized_name"]) self.assertTrue(result.rows[0]["_quality_valid"]) self.assertFalse(result.rows[1]["_quality_valid"]) self.assertEqual( ["amount-required", "name-required"], result.rows[1]["_quality_errors"], ) def test_reconciliation_reports_changed_and_missing_rows(self) -> None: graph = PipelineGraph( nodes=[ node( "expected", "source.inline", { "source_name": "expected", "rows": [ {"id": "1", "amount": 10}, {"id": "2", "amount": 20}, ], }, x=0, ), node( "observed", "source.inline", { "source_name": "observed", "rows": [ {"id": "1", "amount": 11}, {"id": "3", "amount": 30}, ], }, x=0, ), node( "reconcile", "reconcile.compare", { "left_keys": ["id"], "right_keys": ["id"], "compare_columns": ["amount"], "right_prefix": "observed_", }, x=300, ), node("output", "output", {}, x=600), ], edges=[ GraphEdge( id="e1", source="expected", target="reconcile", target_port="left", ), GraphEdge( id="e2", source="observed", target="reconcile", target_port="right", ), GraphEdge(id="e3", source="reconcile", target="output"), ], ) result = execute_preview(graph, row_limit=100) self.assertEqual( ["changed", "missing_observed", "missing_expected"], [item["_reconciliation_status"] for item in result.rows], ) def test_parameterized_subflow_runs_a_pinned_graph(self) -> None: nested = { "schema_version": 1, "nodes": [ { "id": "input", "type": "source.inline", "label": "Input", "position": {"x": 0, "y": 100}, "config": { "source_name": "subflow_input", "rows": [], "input_binding": True, }, }, { "id": "filter", "type": "filter.expression", "label": "Minimum", "position": {"x": 200, "y": 100}, "config": {"expression": "amount >= ${minimum}"}, }, { "id": "output", "type": "output", "label": "Output", "position": {"x": 400, "y": 100}, "config": {}, }, ], "edges": [ {"id": "e1", "source": "input", "target": "filter"}, {"id": "e2", "source": "filter", "target": "output"}, ], } graph = PipelineGraph( nodes=[ node( "source", "source.inline", { "source_name": "source", "rows": [{"amount": 5}, {"amount": 15}], }, x=0, ), node( "subflow", "subflow", { "template_ref": "minimum-filter", "template_version": "1", "parameters": {"minimum": 10}, "graph": nested, }, x=300, ), node("output", "output", {}, x=600), ], edges=[ GraphEdge(id="e1", source="source", target="subflow"), GraphEdge(id="e2", source="subflow", target="output"), ], ) result = execute_preview(graph, row_limit=100) self.assertEqual([{"amount": 15}], result.rows) def test_registry_covers_validation_execution_and_sql_rendering(self) -> None: # Importing the runtime modules above registers each independent facet. coverage = OPERATOR_REGISTRY.coverage() self.assertEqual(set(NODE_TYPES), set(coverage)) for node_type, facets in coverage.items(): self.assertIn("config_validator", facets, node_type) self.assertIn("schema_propagator", facets, node_type) self.assertIn("executor", facets, node_type) self.assertIn("sql_renderer", facets, node_type) def test_maximum_size_linear_graph_validates_and_executes(self) -> None: nodes = [ node( "source", "source.inline", { "source_name": "bounded_input", "rows": [{"value": 1}], }, x=0, ) ] edges: list[GraphEdge] = [] previous_id = "source" for index in range(1, 99): node_id = f"limit-{index}" nodes.append( node( node_id, "limit", {"count": 1}, x=index * 10, ) ) edges.append( GraphEdge( id=f"edge-{index}", source=previous_id, target=node_id, ) ) previous_id = node_id nodes.append(node("output", "output", {}, x=990)) edges.append( GraphEdge( id="edge-99", source=previous_id, target="output", ) ) graph = PipelineGraph(nodes=nodes, edges=edges) self.assertEqual([], validate_graph(graph)) result = execute_preview(graph, row_limit=1) self.assertEqual([{"value": 1}], result.rows) self.assertEqual(100, len(result.node_diagnostics)) def test_expression_node_renders_through_registered_sql_compiler(self) -> None: graph = PipelineGraph( nodes=[ node( "source", "source.inline", { "source_name": "records", "rows": [{"name": "Ada"}], }, x=0, ), node( "expression", "expression", { "target_column": "normalized_name", "expression": "lower(name)", "result_type": "string", }, x=200, ), node("output", "output", {}, x=400), ], edges=[ GraphEdge(id="e1", source="source", target="expression"), GraphEdge(id="e2", source="expression", target="output"), ], ) sql, _ = render_sql(graph) self.assertIn("LOWER(name) AS normalized_name", sql) def test_calculation_and_rank_nodes_render_as_sql(self) -> None: for transform, expected_sql in ( ( node( "calculate", "calculate", { "calculations": [ { "target_column": "gross", "expression": "amount * 2", "result_type": "integer", }, { "target_column": "band", "expression": ( "case when amount >= 10 then 'high' " "else 'standard' end" ), "result_type": "string", }, ] }, x=200, ), ("amount * 2 AS gross", "CASE WHEN amount >= 10"), ), ( node( "rank", "window.rank", { "method": "row_number", "target_column": "position", "partition_by": ["group"], "order_by": [ {"column": "amount", "direction": "desc"} ], }, x=200, ), ( "ROW_NUMBER() OVER", "PARTITION BY group ORDER BY amount DESC", ), ), ): with self.subTest(node_type=transform.type): graph = PipelineGraph( nodes=[ node( "source", "source.inline", { "source_name": "records", "rows": [ {"group": "A", "amount": 12} ], }, x=0, ), transform, node("output", "output", {}, x=400), ], edges=[ GraphEdge( id="e1", source="source", target=transform.id, ), GraphEdge( id="e2", source=transform.id, target="output", ), ], ) sql, _ = render_sql(graph) for fragment in expected_sql: self.assertIn(fragment, sql) if __name__ == "__main__": unittest.main()