Expand governed Dataflow editor and node library

tests/test_graph_and_sql.py

@@ -2,7 +2,13 @@ from __future__ import annotations
 
 import unittest
 
-from govoplan_dataflow.backend.executor import PipelineExecutionError, execute_preview
+from govoplan_dataflow.backend.executor import (
+    MAX_SOURCE_ROWS,
+    PipelineExecutionError,
+    ResolvedSource,
+    execute_preview,
+)
+from govoplan_dataflow.backend.graph import validate_graph
 from govoplan_dataflow.backend.schemas import (
     GraphEdge,
     GraphNode,
@@ -29,7 +35,66 @@ def inline_source() -> GraphNode:
     )
 
 
+def lookup_source() -> GraphNode:
+    return GraphNode(
+        id="lookup",
+        type="source.inline",
+        label="Department lookup",
+        position=GraphPosition(x=40, y=300),
+        config={
+            "source_name": "department_lookup",
+            "rows": [
+                {"department": "A", "label": "Administration"},
+                {"department": "B", "label": "Building services"},
+            ],
+        },
+    )
+
+
 class DataflowGraphAndSqlTests(unittest.TestCase):
+    def test_rejects_invalid_or_duplicate_logical_source_names(self) -> None:
+        invalid_source = inline_source().model_copy(deep=True)
+        invalid_source.config["source_name"] = "monthly cases"
+        duplicate_source = lookup_source().model_copy(deep=True)
+        duplicate_source.config["source_name"] = "MONTHLY_FILES"
+        graph = PipelineGraph(
+            nodes=[
+                invalid_source,
+                duplicate_source,
+                GraphNode(
+                    id="union",
+                    type="combine.union",
+                    label="Append rows",
+                    position=GraphPosition(x=250, y=160),
+                    config={"mode": "all"},
+                ),
+                GraphNode(
+                    id="output",
+                    type="output",
+                    label="Output",
+                    position=GraphPosition(x=450, y=160),
+                    config={},
+                ),
+            ],
+            edges=[
+                GraphEdge(id="edge-1", source="source", target="union"),
+                GraphEdge(id="edge-2", source="lookup", target="union"),
+                GraphEdge(id="edge-3", source="union", target="output"),
+            ],
+        )
+
+        diagnostics = validate_graph(graph)
+
+        self.assertTrue(any(item.code == "source.name_invalid" for item in diagnostics))
+        invalid_source.config["source_name"] = "monthly_files"
+        diagnostics = validate_graph(
+            graph.model_copy(
+                update={"nodes": [invalid_source, *graph.nodes[1:]]},
+                deep=True,
+            )
+        )
+        self.assertTrue(any(item.code == "source.duplicate_name" for item in diagnostics))
+
     def test_compiles_renders_and_executes_grouped_query(self) -> None:
         sql = """
             SELECT department, COUNT(*) AS records, SUM(amount) AS total
@@ -71,13 +136,207 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
             with self.subTest(sql=sql), self.assertRaises(SqlCompilationError):
                 compile_sql(sql, source_nodes=[inline_source()])
 
-    def test_rejects_join_until_comparison_slice(self) -> None:
-        with self.assertRaisesRegex(SqlCompilationError, "JOIN support"):
+    def test_compiles_renders_and_executes_two_source_join(self) -> None:
+        sql = """
+            SELECT monthly_files.department, lookup.label
+            FROM monthly_files
+            LEFT JOIN department_lookup AS lookup
+              ON monthly_files.department = lookup.department
+            ORDER BY monthly_files.department
+        """
+
+        graph, _, diagnostics = compile_sql(
+            sql,
+            source_nodes=[inline_source(), lookup_source()],
+        )
+
+        self.assertEqual([], diagnostics)
+        self.assertEqual(
+            [
+                "source.inline",
+                "source.inline",
+                "combine.join",
+                "select",
+                "sort",
+                "output",
+            ],
+            [node.type for node in graph.nodes],
+        )
+        self.assertEqual("lookup_", graph.nodes[2].config["right_prefix"])
+        rendered, render_diagnostics = render_sql(graph)
+        self.assertEqual([], render_diagnostics)
+        roundtrip, _, _ = compile_sql(
+            rendered,
+            source_nodes=[inline_source(), lookup_source()],
+        )
+        self.assertEqual(
+            graph.model_dump(mode="json"),
+            roundtrip.model_dump(mode="json"),
+        )
+
+        result = execute_preview(graph, row_limit=100)
+        self.assertEqual(
+            [
+                {"department": "A", "lookup_label": "Administration"},
+                {"department": "A", "lookup_label": "Administration"},
+                {"department": "B", "lookup_label": "Building services"},
+            ],
+            result.rows,
+        )
+
+        with self.assertRaisesRegex(SqlCompilationError, "source-qualified"):
             compile_sql(
-                "SELECT * FROM monthly_files JOIN other ON monthly_files.id = other.id",
-                source_nodes=[inline_source()],
+                """
+                SELECT label
+                FROM monthly_files
+                JOIN department_lookup AS lookup
+                  ON monthly_files.department = lookup.department
+                """,
+                source_nodes=[inline_source(), lookup_source()],
             )
 
+    def test_distinct_and_derived_columns_are_deterministic(self) -> None:
+        source = inline_source().model_copy(
+            update={
+                "config": {
+                    "source_name": "monthly_files",
+                    "rows": [
+                        {"first": " Ada ", "last": "Lovelace"},
+                        {"first": " Ada ", "last": "Lovelace"},
+                    ],
+                }
+            }
+        )
+        derive = GraphNode(
+            id="derive",
+            type="derive",
+            label="Display name",
+            position=GraphPosition(x=260, y=160),
+            config={
+                "target_column": "display_name",
+                "operation": "concat",
+                "source_columns": ["first", "last"],
+                "separator": " ",
+            },
+        )
+        distinct = GraphNode(
+            id="distinct",
+            type="distinct",
+            label="Unique people",
+            position=GraphPosition(x=480, y=160),
+            config={"columns": ["display_name"]},
+        )
+        output = GraphNode(
+            id="output",
+            type="output",
+            label="Output",
+            position=GraphPosition(x=700, y=160),
+            config={},
+        )
+        graph = PipelineGraph(
+            nodes=[source, derive, distinct, output],
+            edges=[
+                GraphEdge(id="source-derive", source=source.id, target=derive.id),
+                GraphEdge(id="derive-distinct", source=derive.id, target=distinct.id),
+                GraphEdge(id="distinct-output", source=distinct.id, target=output.id),
+            ],
+        )
+
+        result = execute_preview(graph, row_limit=100)
+
+        self.assertEqual(
+            [{"first": " Ada ", "last": "Lovelace", "display_name": " Ada  Lovelace"}],
+            result.rows,
+        )
+
+    def test_union_can_keep_or_remove_duplicate_rows(self) -> None:
+        first = inline_source().model_copy(
+            update={
+                "id": "first",
+                "config": {"source_name": "first", "rows": [{"id": 1}, {"id": 2}]},
+            }
+        )
+        second = inline_source().model_copy(
+            update={
+                "id": "second",
+                "config": {"source_name": "second", "rows": [{"id": 2}, {"id": 3}]},
+            }
+        )
+        union = GraphNode(
+            id="union",
+            type="combine.union",
+            label="Append",
+            position=GraphPosition(x=300, y=200),
+            config={"mode": "distinct"},
+        )
+        output = GraphNode(
+            id="output",
+            type="output",
+            label="Output",
+            position=GraphPosition(x=520, y=200),
+            config={},
+        )
+        graph = PipelineGraph(
+            nodes=[first, second, union, output],
+            edges=[
+                GraphEdge(id="first-union", source=first.id, target=union.id),
+                GraphEdge(id="second-union", source=second.id, target=union.id),
+                GraphEdge(id="union-output", source=union.id, target=output.id),
+            ],
+        )
+
+        result = execute_preview(graph, row_limit=100)
+
+        self.assertEqual([{"id": 1}, {"id": 2}, {"id": 3}], result.rows)
+
+    def test_connector_source_uses_bounded_resolver_and_records_lineage(self) -> None:
+        source = GraphNode(
+            id="connector",
+            type="source.reference",
+            label="Monthly import",
+            position=GraphPosition(x=40, y=160),
+            config={
+                "source_ref": "snapshot:source-1",
+                "source_name": "monthly_import",
+                "expected_fingerprint": "sha256:expected",
+            },
+        )
+        output = GraphNode(
+            id="output",
+            type="output",
+            label="Output",
+            position=GraphPosition(x=280, y=160),
+            config={},
+        )
+        graph = PipelineGraph(
+            nodes=[source, output],
+            edges=[GraphEdge(id="source-output", source=source.id, target=output.id)],
+        )
+        calls: list[tuple[str, int]] = []
+
+        def resolve(node: GraphNode, limit: int) -> ResolvedSource:
+            calls.append((node.id, limit))
+            return ResolvedSource(
+                rows=({"id": 1}, {"id": 2}),
+                source_ref="snapshot:source-1",
+                provider="snapshot",
+                fingerprint="sha256:expected",
+                total_rows=20,
+                truncated=True,
+            )
+
+        result = execute_preview(
+            graph,
+            row_limit=100,
+            source_resolver=resolve,
+        )
+
+        self.assertEqual([("connector", MAX_SOURCE_ROWS)], calls)
+        self.assertEqual([{"id": 1}, {"id": 2}], result.rows)
+        self.assertEqual(20, result.source_fingerprints[0]["row_count"])
+        self.assertTrue(result.source_fingerprints[0]["truncated"])
+        self.assertEqual("source.preview_truncated", result.diagnostics[0].code)
+
     def test_graph_validation_rejects_cycle(self) -> None:
         source = inline_source()
         output = GraphNode(
@@ -95,11 +354,78 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
             ],
         )
 
-        from govoplan_dataflow.backend.graph import validate_graph
-
         codes = {item.code for item in validate_graph(graph)}
         self.assertIn("graph.cycle", codes)
 
+    def test_schema_validation_reports_unknown_and_colliding_columns(self) -> None:
+        source = inline_source().model_copy(
+            update={
+                "config": {
+                    "source_name": "monthly_files",
+                    "rows": [{"id": 1, "right_id": "reserved"}],
+                }
+            }
+        )
+        lookup = lookup_source().model_copy(
+            update={
+                "config": {
+                    "source_name": "department_lookup",
+                    "rows": [{"id": 1}],
+                }
+            }
+        )
+        join = GraphNode(
+            id="join",
+            type="combine.join",
+            label="Join",
+            position=GraphPosition(x=280, y=200),
+            config={
+                "join_type": "inner",
+                "left_keys": ["missing_id"],
+                "right_keys": ["id"],
+                "right_prefix": "right_",
+            },
+        )
+        output = GraphNode(
+            id="output",
+            type="output",
+            label="Output",
+            position=GraphPosition(x=500, y=200),
+            config={},
+        )
+        graph = PipelineGraph(
+            nodes=[source, lookup, join, output],
+            edges=[
+                GraphEdge(
+                    id="source-join",
+                    source=source.id,
+                    target=join.id,
+                    target_port="left",
+                ),
+                GraphEdge(
+                    id="lookup-join",
+                    source=lookup.id,
+                    target=join.id,
+                    target_port="right",
+                ),
+                GraphEdge(id="join-output", source=join.id, target=output.id),
+            ],
+        )
+
+        diagnostics = validate_graph(graph)
+
+        self.assertTrue(
+            any(
+                item.code == "schema.unknown_column"
+                and item.node_id == join.id
+                and item.field == "left_keys"
+                for item in diagnostics
+            )
+        )
+        self.assertTrue(
+            any(item.code == "join.output_collision" for item in diagnostics)
+        )
+
     def test_preview_truncates_response_without_changing_pipeline_limit(self) -> None:
         source = inline_source()
         output = GraphNode(
@@ -202,6 +528,14 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
 
         self.assertEqual("filter-1", raised.exception.node_id)
         self.assertIn("Cannot apply", str(raised.exception))
+        self.assertEqual(
+            [("source", "succeeded"), ("filter-1", "failed")],
+            [
+                (diagnostic.node_id, diagnostic.status)
+                for diagnostic in raised.exception.node_diagnostics
+            ],
+        )
+        self.assertEqual(1, len(raised.exception.source_fingerprints))
 
 
 if __name__ == "__main__":

tests/test_node_library.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import unittest
+
+from govoplan_dataflow.backend.node_library import CATEGORY_LABELS, NODE_LIBRARY, NODE_TYPES
+
+
+class DataflowNodeLibraryTests(unittest.TestCase):
+    def test_library_has_unique_executable_nodes_in_every_category(self) -> None:
+        self.assertEqual(len(NODE_LIBRARY), len(NODE_TYPES))
+        self.assertEqual(set(CATEGORY_LABELS), {item.category for item in NODE_LIBRARY})
+        self.assertEqual(
+            {
+                "source.inline",
+                "source.reference",
+                "combine.union",
+                "combine.join",
+                "filter",
+                "distinct",
+                "select",
+                "derive",
+                "aggregate",
+                "sort",
+                "limit",
+                "output",
+            },
+            set(NODE_TYPES),
+        )
+
+    def test_join_and_union_publish_their_connection_contract(self) -> None:
+        join = NODE_TYPES["combine.join"]
+        union = NODE_TYPES["combine.union"]
+
+        self.assertEqual(["left", "right"], [port.id for port in join.input_ports])
+        self.assertTrue(union.input_ports[0].multiple)
+        self.assertEqual(2, union.input_ports[0].minimum_connections)
+        self.assertEqual([], list(NODE_TYPES["output"].output_ports))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_schemas.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import unittest
+
+from govoplan_dataflow.backend.schemas import TabularSnapshotCreateRequest
+
+
+class DataflowSchemaTests(unittest.TestCase):
+    def test_snapshot_request_requires_the_selected_format_payload(self) -> None:
+        json_request = TabularSnapshotCreateRequest(
+            name="Monthly cases",
+            source_name="monthly_cases",
+            format="json",
+            rows=[{"case_id": "0012"}],
+        )
+        csv_request = TabularSnapshotCreateRequest(
+            name="Monthly cases",
+            source_name="monthly_cases_csv",
+            format="csv",
+            csv_text="case_id;amount\n0012;7.5\n",
+            delimiter=";",
+        )
+
+        self.assertEqual([{"case_id": "0012"}], json_request.rows)
+        self.assertEqual(";", csv_request.delimiter)
+        with self.assertRaises(ValueError):
+            TabularSnapshotCreateRequest(
+                name="Missing rows",
+                source_name="missing_rows",
+                format="json",
+            )
+        with self.assertRaises(ValueError):
+            TabularSnapshotCreateRequest(
+                name="Ambiguous",
+                source_name="ambiguous",
+                format="csv",
+                rows=[],
+                csv_text="id\n1\n",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_service.py

@@ -220,12 +220,49 @@ class DataflowServiceTests(unittest.TestCase):
         self.assertEqual(2, run.output_row_count)
         self.assertEqual(3, run.input_row_count)
         self.assertEqual(1, len(run.source_fingerprints))
+        self.assertEqual(3, response.input_row_count)
+        self.assertEqual(run.source_fingerprints, response.source_fingerprints)
         self.assertFalse(hasattr(run, "result_rows"))
         self.assertEqual(
             1,
             self.session.scalar(select(func.count()).select_from(DataflowRun)),
         )
 
+    def test_failed_preview_keeps_upstream_lineage_and_failed_node_diagnostic(self) -> None:
+        graph = sample_graph()
+        graph.nodes[0].config["rows"] = [{"id": 1, "amount": "not-a-number"}]
+        pipeline = create_pipeline(
+            self.session,
+            tenant_id="tenant-1",
+            actor_id="user-1",
+            payload=PipelineCreateRequest(
+                name="Invalid runtime value",
+                graph=graph,
+                editor_mode="graph",
+            ),
+        )
+        self.session.commit()
+
+        response = preview_pipeline(
+            self.session,
+            tenant_id="tenant-1",
+            actor_id="user-1",
+            payload=PipelinePreviewRequest(pipeline_id=pipeline.id),
+        )
+        self.session.commit()
+
+        run = self.session.scalar(select(DataflowRun).where(DataflowRun.id == response.run_id))
+        self.assertEqual("failed", response.status)
+        self.assertEqual(
+            [("source", "succeeded"), ("filter", "failed")],
+            [(item.node_id, item.status) for item in response.node_diagnostics],
+        )
+        self.assertEqual(1, run.input_row_count)
+        self.assertEqual(1, len(run.source_fingerprints))
+        self.assertEqual(1, response.input_row_count)
+        self.assertEqual(run.source_fingerprints, response.source_fingerprints)
+        self.assertEqual("preview.execution", run.diagnostics[-1]["code"])
+
 
 if __name__ == "__main__":
     unittest.main()