Fix union SQL and selection previews

This commit is contained in:
2026-07-28 17:54:28 +02:00
parent 521829a7fc
commit 74f1210a32
9 changed files with 498 additions and 141 deletions

View File

@@ -45,18 +45,20 @@ fails visibly instead of silently changing a run.
## SQL And Preview Safety ## SQL And Preview Safety
The SQL workbench parses one `SELECT` statement into the canonical graph. The The SQL workbench parses one `SELECT` or column-aligned `UNION BY NAME`
dialect supports projection, aliases, filters, grouping, aggregate functions, statement into the canonical graph. The dialect supports projection, aliases,
sorting, limits, `DISTINCT`, and one two-source equi-join. It rejects DDL, DML, filters, grouping, aggregate functions, sorting, limits, `DISTINCT`, append,
subqueries, arbitrary functions, file access, and unchecked pass-through and one two-source equi-join. It rejects DDL, DML, arbitrary subqueries,
execution. arbitrary functions, file access, and unchecked pass-through execution.
Preview reads at most 250 rows per source and enforces time, intermediate-row, Preview reads at most 250 rows per source and enforces time, intermediate-row,
result-byte, graph-node, and response-row bounds. Saved previews record the result-byte, graph-node, and response-row bounds. Saved previews record the
pipeline revision, executor version, source fingerprints, node diagnostics, pipeline revision, executor version, source fingerprints, node diagnostics,
and output summary, but not source or result rows. A preview may return one and output summary, but not source or result rows. A preview may return one
explicitly requested intermediate node state under the same response bound; explicitly requested intermediate node state under the same response bound;
that row payload remains ephemeral and is not added to run evidence. that row payload remains ephemeral and is not added to run evidence. The WebUI
targets the selected graph node and supports remembered automatic or
manual-refresh preview modes.
Saved revisions can also be started through the versioned Saved revisions can also be started through the versioned
`dataflow.runLifecycle` capability or the Run dialog. The first runner is `dataflow.runLifecycle` capability or the Run dialog. The first runner is

View File

@@ -45,7 +45,9 @@ class GraphNode(BaseModel):
class GraphEdge(BaseModel): class GraphEdge(BaseModel):
id: str = Field(min_length=1, max_length=120, pattern=r"^[A-Za-z0-9_.:-]+$") # Older WebUI releases composed edge IDs from both node IDs. Keep those
# definitions readable while new clients use short opaque edge IDs.
id: str = Field(min_length=1, max_length=255, pattern=r"^[A-Za-z0-9_.:-]+$")
source: str = Field(min_length=1, max_length=100) source: str = Field(min_length=1, max_length=100)
target: str = Field(min_length=1, max_length=100) target: str = Field(min_length=1, max_length=100)
source_port: str = Field(default="output", min_length=1, max_length=80) source_port: str = Field(default="output", min_length=1, max_length=80)

View File

@@ -40,15 +40,30 @@ def compile_sql(
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.parse", f"SQL could not be parsed{location}: {detail.get('description') or exc}")] [_sql_error("sql.parse", f"SQL could not be parsed{location}: {detail.get('description') or exc}")]
) from exc ) from exc
if len(statements) != 1 or not isinstance(statements[0], exp.Select): if len(statements) != 1 or not isinstance(statements[0], (exp.Select, exp.Union)):
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.select_only", "Dataflow SQL accepts exactly one SELECT statement.")] [
_sql_error(
"sql.select_only",
"Dataflow SQL accepts exactly one SELECT or UNION BY NAME statement.",
) )
query = statements[0] ]
_reject_unsupported_query_shape(query) )
query, union_expression = _select_query(statements[0])
from_clause = query.args.get("from_") from_clause = query.args.get("from_")
if from_clause is None or not isinstance(from_clause.this, exp.Table): union_subquery = (
from_clause.this
if from_clause is not None
and isinstance(from_clause.this, exp.Subquery)
and isinstance(from_clause.this.this, exp.Union)
else None
)
_reject_unsupported_query_shape(query, allowed_subquery=union_subquery)
if from_clause is None or (
union_expression is None
and not isinstance(from_clause.this, exp.Table)
):
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.source_required", "SELECT requires a logical tabular source.")] [_sql_error("sql.source_required", "SELECT requires a logical tabular source.")]
) )
@@ -57,7 +72,65 @@ def compile_sql(
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.join_count", "Dataflow SQL currently supports one two-source join.")] [_sql_error("sql.join_count", "Dataflow SQL currently supports one two-source join.")]
) )
source_node_list = list(source_nodes)
nodes: list[GraphNode] = []
edges: list[GraphEdge] = []
qualifier_prefixes: dict[str, str] | None = None
previous_node_id: str
edge_sequence = 0
def next_edge_id() -> str:
nonlocal edge_sequence
edge_sequence += 1
return f"edge-{edge_sequence}"
if union_expression is not None:
if joins:
raise SqlCompilationError(
[_sql_error("sql.union_join", "JOIN outside UNION BY NAME is not supported.")]
)
union_tables, union_mode = _union_tables(union_expression)
for index, table in enumerate(union_tables, start=1):
source = _source_node(
table,
source_node_list,
fallback_id=f"source-{index}",
position=GraphPosition(x=60, y=80 + ((index - 1) * 140)),
)
if any(existing.id == source.id for existing in nodes):
raise SqlCompilationError(
[
_sql_error(
"sql.source_identity",
"UNION inputs must use different graph nodes.",
)
]
)
nodes.append(source)
union_node = GraphNode(
id="union",
type="combine.union",
label="Append rows",
position=GraphPosition(x=300, y=80 + ((len(nodes) - 1) * 70)),
config={"mode": union_mode},
)
nodes.append(union_node)
edges.extend(
GraphEdge(
id=next_edge_id(),
source=source.id,
target=union_node.id,
)
for source in nodes
if source.type.startswith("source.")
)
previous_node_id = union_node.id
else:
left_table = from_clause.this left_table = from_clause.this
if not isinstance(left_table, exp.Table):
raise SqlCompilationError(
[_sql_error("sql.source_required", "SELECT requires a logical tabular source.")]
)
right_table = joins[0].this if joins else None right_table = joins[0].this if joins else None
if right_table is not None and not isinstance(right_table, exp.Table): if right_table is not None and not isinstance(right_table, exp.Table):
raise SqlCompilationError( raise SqlCompilationError(
@@ -69,18 +142,14 @@ def compile_sql(
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.qualified_source", "Use logical source names without a catalog or schema.")] [_sql_error("sql.qualified_source", "Use logical source names without a catalog or schema.")]
) )
source_node_list = list(source_nodes)
left_source = _source_node( left_source = _source_node(
left_table, left_table,
source_node_list, source_node_list,
fallback_id="source-left" if right_table is not None else "source", fallback_id="source-left" if right_table is not None else "source",
position=GraphPosition(x=60, y=120 if right_table is not None else 180), position=GraphPosition(x=60, y=120 if right_table is not None else 180),
) )
nodes = [left_source] nodes.append(left_source)
edges: list[GraphEdge] = []
previous_node_id = left_source.id previous_node_id = left_source.id
qualifier_prefixes: dict[str, str] | None = None
if isinstance(right_table, exp.Table): if isinstance(right_table, exp.Table):
right_source = _source_node( right_source = _source_node(
right_table, right_table,
@@ -106,13 +175,13 @@ def compile_sql(
edges.extend( edges.extend(
( (
GraphEdge( GraphEdge(
id=f"edge-{left_source.id}-{join_node.id}-left", id=next_edge_id(),
source=left_source.id, source=left_source.id,
target=join_node.id, target=join_node.id,
target_port="left", target_port="left",
), ),
GraphEdge( GraphEdge(
id=f"edge-{right_source.id}-{join_node.id}-right", id=next_edge_id(),
source=right_source.id, source=right_source.id,
target=join_node.id, target=join_node.id,
target_port="right", target_port="right",
@@ -131,7 +200,7 @@ def compile_sql(
nodes.append(node) nodes.append(node)
edges.append( edges.append(
GraphEdge( GraphEdge(
id=f"edge-{previous_node_id}-{node.id}", id=next_edge_id(),
source=previous_node_id, source=previous_node_id,
target=node.id, target=node.id,
) )
@@ -322,17 +391,67 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
node_by_id = {node.id: node for node in graph.nodes} node_by_id = {node.id: node for node in graph.nodes}
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")] source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
join_nodes = [node for node in graph.nodes if node.type == "combine.join"] join_nodes = [node for node in graph.nodes if node.type == "combine.join"]
union_nodes = [node for node in graph.nodes if node.type == "combine.union"]
if len(join_nodes) > 1: if len(join_nodes) > 1:
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.join_count", "Only one two-source join can be rendered as SQL.")] [_sql_error("sql.join_count", "Only one two-source join can be rendered as SQL.")]
) )
if len(union_nodes) > 1:
raise SqlCompilationError(
[_sql_error("sql.union_count", "Only one append node can be rendered as SQL.")]
)
if join_nodes and union_nodes:
raise SqlCompilationError(
[_sql_error("sql.combine_count", "JOIN and UNION cannot yet be combined in Dataflow SQL.")]
)
left_source: GraphNode left_source: GraphNode | None = None
right_source: GraphNode | None = None right_source: GraphNode | None = None
join_node = join_nodes[0] if join_nodes else None join_node = join_nodes[0] if join_nodes else None
union_node = union_nodes[0] if union_nodes else None
union_expression: exp.Expression | None = None
left_source_name: str | None = None
right_prefix: str | None = None right_prefix: str | None = None
right_alias: str | None = None right_alias: str | None = None
if join_node is not None: right_source_name: str | None = None
if union_node is not None:
inputs = graph_inputs_by_port(graph).get(union_node.id, {})
union_sources = [
node_by_id[source_id]
for source_id in inputs.get("input", [])
if source_id in node_by_id
]
if (
len(union_sources) != len(source_nodes)
or any(not node.type.startswith("source.") for node in union_sources)
or {node.id for node in union_sources} != {node.id for node in source_nodes}
):
raise SqlCompilationError(
[
_node_sql_error(
union_node.id,
"sql.union_shape",
"SQL rendering currently requires each append input to connect directly to a source.",
)
]
)
source_names = [
str(node.config.get("source_name", "")).strip()
for node in union_sources
]
if any(not source_name for source_name in source_names):
raise SqlCompilationError(
[_sql_error("source.name_required", "Every source needs a logical SQL name.")]
)
union_expression = exp.select("*").from_(exp.to_table(source_names[0]))
for source_name in source_names[1:]:
union_expression = exp.Union(
this=union_expression,
expression=exp.select("*").from_(exp.to_table(source_name)),
distinct=union_node.config.get("mode") == "distinct",
by_name=True,
)
elif join_node is not None:
inputs = graph_inputs_by_port(graph).get(join_node.id, {}) inputs = graph_inputs_by_port(graph).get(join_node.id, {})
left_source = node_by_id[inputs["left"][0]] left_source = node_by_id[inputs["left"][0]]
right_source = node_by_id[inputs["right"][0]] right_source = node_by_id[inputs["right"][0]]
@@ -345,6 +464,11 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
[_sql_error("sql.source_count", "SQL rendering needs one source or one two-source join.")] [_sql_error("sql.source_count", "SQL rendering needs one source or one two-source join.")]
) )
if union_node is None:
if left_source is None:
raise SqlCompilationError(
[_sql_error("sql.source_required", "SQL rendering needs a source.")]
)
left_source_name = str(left_source.config.get("source_name", "")).strip() left_source_name = str(left_source.config.get("source_name", "")).strip()
right_source_name = ( right_source_name = (
str(right_source.config.get("source_name", "")).strip() str(right_source.config.get("source_name", "")).strip()
@@ -368,7 +492,7 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
[_sql_error("sql.column", "A right-side column name is missing.")] [_sql_error("sql.column", "A right-side column name is missing.")]
) )
return exp.column(right_name, table=right_alias) return exp.column(right_name, table=right_alias)
if right_source is not None: if right_source is not None and left_source_name:
return exp.column(name, table=left_source_name) return exp.column(name, table=left_source_name)
return exp.column(name) return exp.column(name)
@@ -382,7 +506,7 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
for node_id in ordered: for node_id in ordered:
node = node_by_id[node_id] node = node_by_id[node_id]
if node.type.startswith("source.") or node.type == "combine.join": if node.type.startswith("source.") or node.type in {"combine.join", "combine.union"}:
continue continue
if node.type == "filter": if node.type == "filter":
if selected: if selected:
@@ -482,7 +606,16 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
[_node_sql_error(node.id, "sql.node_not_representable", f"{node.type!r} cannot be rendered as SQL.")] [_node_sql_error(node.id, "sql.node_not_representable", f"{node.type!r} cannot be rendered as SQL.")]
) )
if union_expression is not None:
query = exp.select(*select_expressions).from_(
union_expression.subquery("_unioned")
)
elif left_source_name:
query = exp.select(*select_expressions).from_(exp.to_table(left_source_name)) query = exp.select(*select_expressions).from_(exp.to_table(left_source_name))
else:
raise SqlCompilationError(
[_sql_error("sql.source_required", "SQL rendering needs a source.")]
)
if join_node is not None and right_source_name and right_alias: if join_node is not None and right_source_name and right_alias:
join_conditions = [ join_conditions = [
exp.EQ( exp.EQ(
@@ -513,7 +646,124 @@ def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
return query.sql(dialect="duckdb", pretty=True), diagnostics return query.sql(dialect="duckdb", pretty=True), diagnostics
def _reject_unsupported_query_shape(query: exp.Select) -> None: def _select_query(
statement: exp.Select | exp.Union,
) -> tuple[exp.Select, exp.Union | None]:
if isinstance(statement, exp.Select):
from_clause = statement.args.get("from_")
if (
from_clause is not None
and isinstance(from_clause.this, exp.Subquery)
and isinstance(from_clause.this.this, exp.Union)
):
return statement, from_clause.this.this
return statement, None
union_expression = statement.copy()
order = union_expression.args.get("order")
limit = union_expression.args.get("limit")
union_expression.set("order", None)
union_expression.set("limit", None)
query = exp.select("*").from_(union_expression.subquery("_unioned"))
if order is not None:
query.set("order", order.copy())
if limit is not None:
query.set("limit", limit.copy())
return query, union_expression
def _union_tables(union_expression: exp.Union) -> tuple[list[exp.Table], str]:
branches: list[exp.Select] = []
modes: set[bool] = set()
def collect(expression: exp.Expression) -> None:
if isinstance(expression, exp.Union):
if expression.args.get("by_name") is not True:
raise SqlCompilationError(
[
_sql_error(
"sql.union_by_name",
"Use UNION BY NAME so SQL matches Dataflow column-name append semantics.",
)
]
)
if expression.args.get("order") or expression.args.get("limit"):
raise SqlCompilationError(
[
_sql_error(
"sql.union_branch_order",
"Apply ORDER BY or LIMIT after the complete UNION BY NAME.",
)
]
)
modes.add(bool(expression.args.get("distinct")))
collect(expression.this)
collect(expression.expression)
return
if not isinstance(expression, exp.Select):
raise SqlCompilationError(
[_sql_error("sql.union_branch", "UNION BY NAME branches must be SELECT statements.")]
)
branches.append(expression)
collect(union_expression)
if len(modes) != 1:
raise SqlCompilationError(
[
_sql_error(
"sql.union_mode",
"A Dataflow append node cannot mix UNION and UNION ALL.",
)
]
)
tables: list[exp.Table] = []
for branch in branches:
_reject_unsupported_query_shape(branch)
if (
branch.args.get("joins")
or branch.args.get("where")
or branch.args.get("group")
or branch.args.get("order")
or branch.args.get("limit")
or branch.args.get("distinct")
):
raise SqlCompilationError(
[
_sql_error(
"sql.union_branch_shape",
"UNION BY NAME inputs must be direct SELECT * source reads.",
)
]
)
if len(branch.expressions) != 1 or not isinstance(branch.expressions[0], exp.Star):
raise SqlCompilationError(
[
_sql_error(
"sql.union_branch_projection",
"UNION BY NAME inputs must select all source columns.",
)
]
)
from_clause = branch.args.get("from_")
table = from_clause.this if from_clause is not None else None
if not isinstance(table, exp.Table):
raise SqlCompilationError(
[_sql_error("sql.union_source", "Each UNION BY NAME input needs a logical source.")]
)
if table.catalog or table.db:
raise SqlCompilationError(
[_sql_error("sql.qualified_source", "Use logical source names without a catalog or schema.")]
)
tables.append(table)
return tables, "distinct" if modes == {True} else "all"
def _reject_unsupported_query_shape(
query: exp.Select,
*,
allowed_subquery: exp.Subquery | None = None,
) -> None:
unsupported_args = { unsupported_args = {
"with_": "WITH queries", "with_": "WITH queries",
"having": "HAVING", "having": "HAVING",
@@ -527,7 +777,10 @@ def _reject_unsupported_query_shape(query: exp.Select) -> None:
raise SqlCompilationError( raise SqlCompilationError(
[_sql_error("sql.unsupported_clause", f"{label} are not supported by the first Dataflow dialect.")] [_sql_error("sql.unsupported_clause", f"{label} are not supported by the first Dataflow dialect.")]
) )
if any(True for _ in query.find_all(exp.Subquery)): if any(
subquery is not allowed_subquery
for subquery in query.find_all(exp.Subquery)
):
raise SqlCompilationError([_sql_error("sql.subquery", "Subqueries are not supported by the first Dataflow dialect.")]) raise SqlCompilationError([_sql_error("sql.subquery", "Subqueries are not supported by the first Dataflow dialect.")])

View File

@@ -288,6 +288,27 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
result = execute_preview(graph, row_limit=100) result = execute_preview(graph, row_limit=100)
self.assertEqual([{"id": 1}, {"id": 2}, {"id": 3}], result.rows) self.assertEqual([{"id": 1}, {"id": 2}, {"id": 3}], result.rows)
rendered, diagnostics = render_sql(graph)
self.assertEqual([], diagnostics)
self.assertIn("UNION BY NAME", rendered)
roundtrip, normalized, diagnostics = compile_sql(
rendered,
source_nodes=[first, second],
)
self.assertEqual(rendered, normalized)
self.assertEqual([], diagnostics)
self.assertEqual(
["source.inline", "source.inline", "combine.union", "output"],
[node.type for node in roundtrip.nodes],
)
self.assertEqual(result.rows, execute_preview(roundtrip, row_limit=100).rows)
def test_legacy_composite_edge_ids_remain_readable(self) -> None:
edge_id = f"edge-{'a' * 100}-{'b' * 100}"
edge = GraphEdge(id=edge_id, source="source", target="output")
self.assertEqual(edge_id, edge.id)
def test_connector_source_uses_bounded_resolver_and_records_lineage(self) -> None: def test_connector_source_uses_bounded_resolver_and_records_lineage(self) -> None:
source = GraphNode( source = GraphNode(

View File

@@ -11,6 +11,22 @@ const checks = [
[page.includes("Apply SQL"), "SQL workbench"], [page.includes("Apply SQL"), "SQL workbench"],
[page.includes("<NodeInspector"), "node inspector"], [page.includes("<NodeInspector"), "node inspector"],
[page.includes("previewDataflowPipeline"), "preview action"], [page.includes("previewDataflowPipeline"), "preview action"],
[
page.includes('inactiveLabel="Manual"')
&& page.includes('activeLabel="Auto"')
&& page.includes("void runPreview(selectedNodeId, true)"),
"selection-driven automatic and manual preview modes"
],
[
!page.includes('aria-label="Preview stage"')
&& page.includes('className="dataflow-preview-stage"'),
"selected-node preview stage without dropdown"
],
[
page.includes('resetKey={error} floating')
&& !css.includes(".dataflow-alerts"),
"central floating dismissible alerts"
],
[page.includes("useUnsavedDraftGuard"), "unsaved-change guard"], [page.includes("useUnsavedDraftGuard"), "unsaved-change guard"],
[canvas.includes("application/x-govoplan-dataflow-node"), "palette drop handling"], [canvas.includes("application/x-govoplan-dataflow-node"), "palette drop handling"],
[canvas.includes("isValidConnection"), "edge validation"], [canvas.includes("isValidConnection"), "edge validation"],
@@ -20,6 +36,11 @@ const checks = [
[canvas.includes("onReconnect={onReconnect}"), "edge reconnection"], [canvas.includes("onReconnect={onReconnect}"), "edge reconnection"],
[canvas.includes("onReconnectEnd="), "edge deletion on dropped reconnection"], [canvas.includes("onReconnectEnd="), "edge deletion on dropped reconnection"],
[canvas.includes("selectedEdgeId"), "persistent edge selection"], [canvas.includes("selectedEdgeId"), "persistent edge selection"],
[
canvas.includes("newGraphEdgeId()")
&& interactions.includes("return `edge-${crypto.randomUUID()}`"),
"bounded opaque edge identifiers"
],
[ [
canvas.includes("closestProximityConnection") canvas.includes("closestProximityConnection")
&& interactions.includes("definitionConnectionError"), && interactions.includes("definitionConnectionError"),

View File

@@ -26,6 +26,7 @@ import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
import { import {
closestProximityConnection, closestProximityConnection,
graphEdgesFromFlow, graphEdgesFromFlow,
newGraphEdgeId,
proximityPreviewEdge proximityPreviewEdge
} from "./graphInteractions"; } from "./graphInteractions";
import { FALLBACK_NODE_LIBRARY, newNode } from "./model"; import { FALLBACK_NODE_LIBRARY, newNode } from "./model";
@@ -164,7 +165,7 @@ export default function DataflowCanvas({
const next = addEdge( const next = addEdge(
{ {
...connection, ...connection,
id: `edge-${connection.source}-${connection.target}-${crypto.randomUUID()}`, id: newGraphEdgeId(),
type: "smoothstep" type: "smoothstep"
}, },
edges edges
@@ -280,7 +281,7 @@ export default function DataflowCanvas({
const nextEdges = addEdge( const nextEdges = addEdge(
{ {
...connection, ...connection,
id: `edge-${crypto.randomUUID()}`, id: newGraphEdgeId(),
type: "smoothstep", type: "smoothstep",
className: "dataflow-edge" className: "dataflow-edge"
}, },

View File

@@ -2,6 +2,7 @@ import {
useCallback, useCallback,
useEffect, useEffect,
useMemo, useMemo,
useRef,
useState, useState,
type DragEvent type DragEvent
} from "react"; } from "react";
@@ -92,6 +93,7 @@ type ResultTab = "preview" | "diagnostics";
type SnapshotFormat = "json" | "csv"; type SnapshotFormat = "json" | "csv";
const NODE_CATEGORY_ORDER: NodeCategory[] = ["load", "combine", "filter", "transform", "output"]; const NODE_CATEGORY_ORDER: NodeCategory[] = ["load", "combine", "filter", "transform", "output"];
const AUTO_PREVIEW_STORAGE_KEY = "govoplan.dataflow.auto-preview";
export default function DataflowPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo }) { export default function DataflowPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo }) {
const { requestNavigation, requestDiscard } = useUnsavedChanges(); const { requestNavigation, requestDiscard } = useUnsavedChanges();
@@ -108,6 +110,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
const [diagnostics, setDiagnostics] = useState<DataflowDiagnostic[]>([]); const [diagnostics, setDiagnostics] = useState<DataflowDiagnostic[]>([]);
const [nodeDiagnostics, setNodeDiagnostics] = useState<NodePreviewDiagnostic[]>([]); const [nodeDiagnostics, setNodeDiagnostics] = useState<NodePreviewDiagnostic[]>([]);
const [preview, setPreview] = useState<PipelinePreview | null>(null); const [preview, setPreview] = useState<PipelinePreview | null>(null);
const [autoPreview, setAutoPreview] = useState(readAutoPreviewPreference);
const [resultOpen, setResultOpen] = useState(false); const [resultOpen, setResultOpen] = useState(false);
const [resultTab, setResultTab] = useState<ResultTab>("preview"); const [resultTab, setResultTab] = useState<ResultTab>("preview");
const [deleteOpen, setDeleteOpen] = useState(false); const [deleteOpen, setDeleteOpen] = useState(false);
@@ -120,6 +123,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
const [sources, setSources] = useState<TabularSource[]>([]); const [sources, setSources] = useState<TabularSource[]>([]);
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false); const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
const [sourceCatalogueWritable, setSourceCatalogueWritable] = useState(false); const [sourceCatalogueWritable, setSourceCatalogueWritable] = useState(false);
const previewRequestId = useRef(0);
const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft); const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft);
const canWrite = hasScope(auth, "dataflow:pipeline:write") || hasScope(auth, "dataflow:pipeline:admin"); const canWrite = hasScope(auth, "dataflow:pipeline:write") || hasScope(auth, "dataflow:pipeline:admin");
@@ -345,10 +349,12 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
}; };
const updateGraph = (graph: PipelineDraft["graph"]) => { const updateGraph = (graph: PipelineDraft["graph"]) => {
previewRequestId.current += 1;
updateDraft({ graph }); updateDraft({ graph });
setDiagnostics([]); setDiagnostics([]);
setNodeDiagnostics([]); setNodeDiagnostics([]);
setPreview(null); setPreview(null);
setWorking(false);
}; };
const validate = async () => { const validate = async () => {
@@ -426,13 +432,18 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
await applySql(true); await applySql(true);
}; };
const runPreview = async (previewNodeId?: string) => { const runPreview = useCallback(async (
previewNodeId?: string,
quiet = false
) => {
if (!draft || !canPreview) return; if (!draft || !canPreview) return;
const targetNodeId = previewNodeId const targetNodeId = previewNodeId
?? draft.graph.nodes.find((node) => node.type === "output")?.id; ?? draft.graph.nodes.find((node) => node.type === "output")?.id;
const requestId = previewRequestId.current + 1;
previewRequestId.current = requestId;
setWorking(true); setWorking(true);
setError(""); setError("");
setSuccess(""); if (!quiet) setSuccess("");
try { try {
const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision
? { ? {
@@ -448,6 +459,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
preview_node_id: targetNodeId, preview_node_id: targetNodeId,
row_limit: 100 row_limit: 100
}); });
if (requestId !== previewRequestId.current) return;
setPreview(response); setPreview(response);
setDiagnostics(response.diagnostics); setDiagnostics(response.diagnostics);
setNodeDiagnostics(response.node_diagnostics); setNodeDiagnostics(response.node_diagnostics);
@@ -457,7 +469,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
? "preview" ? "preview"
: "diagnostics" : "diagnostics"
); );
if (response.status === "succeeded" && response.node_preview) { if (!quiet && response.status === "succeeded" && response.node_preview) {
const node = draft.graph.nodes.find( const node = draft.graph.nodes.find(
(candidate) => candidate.id === response.node_preview?.node_id (candidate) => candidate.id === response.node_preview?.node_id
); );
@@ -467,10 +479,38 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
); );
} }
} catch (previewError) { } catch (previewError) {
if (requestId !== previewRequestId.current) return;
setError(apiErrorMessage(previewError)); setError(apiErrorMessage(previewError));
} finally { } finally {
setWorking(false); if (requestId === previewRequestId.current) setWorking(false);
} }
}, [canPreview, dirty, draft, settings]);
useEffect(() => {
if (
!autoPreview
|| !selectedNodeId
|| !canPreview
|| draft?.editorMode !== "graph"
) {
return;
}
const timeout = window.setTimeout(() => {
void runPreview(selectedNodeId, true);
}, 350);
return () => window.clearTimeout(timeout);
}, [autoPreview, canPreview, draft?.editorMode, runPreview, selectedNodeId]);
const selectNode = useCallback((nodeId: string | null) => {
setSelectedNodeId(nodeId);
if (nodeId && nodeId !== preview?.node_preview?.node_id) {
setPreview(null);
}
}, [preview?.node_preview?.node_id]);
const changeAutoPreview = (enabled: boolean) => {
setAutoPreview(enabled);
writeAutoPreviewPreference(enabled);
}; };
const removePipeline = async () => { const removePipeline = async () => {
@@ -614,14 +654,23 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
</Button> </Button>
<Button <Button
variant="primary" variant="primary"
onClick={() => void runPreview()} onClick={() => void runPreview(selectedNodeId ?? undefined)}
disabled={working || !canPreview} disabled={working || !canPreview}
disabledReason={ disabledReason={
draft.governance?.actions.run?.reason ?? undefined draft.governance?.actions.run?.reason ?? undefined
} }
> >
<Play size={16} /> Preview {preview ? <RefreshCw size={16} /> : <Play size={16} />}
{preview ? "Refresh" : "Preview"}
</Button> </Button>
<ToggleSwitch
label="Preview updates"
inactiveLabel="Manual"
activeLabel="Auto"
checked={autoPreview}
onChange={changeAutoPreview}
disabled={!canPreview}
/>
<Button <Button
onClick={() => setRunOpen(true)} onClick={() => setRunOpen(true)}
disabled={working || !canRun || dirty || !canStartSavedRun || !draft.currentRevision} disabled={working || !canRun || dirty || !canStartSavedRun || !draft.currentRevision}
@@ -683,11 +732,11 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
</Button> </Button>
</div> </div>
</div> </div>
{(error || success) ? ( {error ? (
<div className="dataflow-alerts"> <DismissibleAlert tone="danger" resetKey={error} floating>{error}</DismissibleAlert>
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null} ) : null}
{success ? <DismissibleAlert tone="success" resetKey={success}>{success}</DismissibleAlert> : null} {success ? (
</div> <DismissibleAlert tone="success" resetKey={success} floating>{success}</DismissibleAlert>
) : null} ) : null}
<div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}> <div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}>
{draft.editorMode === "graph" ? ( {draft.editorMode === "graph" ? (
@@ -742,7 +791,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
selectedNodeId={selectedNodeId} selectedNodeId={selectedNodeId}
readOnly={!canEdit} readOnly={!canEdit}
onGraphChange={updateGraph} onGraphChange={updateGraph}
onSelectNode={setSelectedNodeId} onSelectNode={selectNode}
/> />
</ReactFlowProvider> </ReactFlowProvider>
) : ( ) : (
@@ -793,10 +842,9 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
onTabChange={setResultTab} onTabChange={setResultTab}
preview={preview} preview={preview}
nodes={draft.graph.nodes} nodes={draft.graph.nodes}
working={working}
diagnostics={diagnostics} diagnostics={diagnostics}
nodeDiagnostics={nodeDiagnostics} nodeDiagnostics={nodeDiagnostics}
onPreviewNodeChange={(nodeId) => void runPreview(nodeId)} selectedNodeId={selectedNodeId}
onClose={() => setResultOpen(false)} onClose={() => setResultOpen(false)}
/> />
) : null} ) : null}
@@ -1872,24 +1920,23 @@ function ResultPanel({
onTabChange, onTabChange,
preview, preview,
nodes, nodes,
working,
diagnostics, diagnostics,
nodeDiagnostics, nodeDiagnostics,
onPreviewNodeChange, selectedNodeId,
onClose onClose
}: { }: {
tab: ResultTab; tab: ResultTab;
onTabChange: (tab: ResultTab) => void; onTabChange: (tab: ResultTab) => void;
preview: PipelinePreview | null; preview: PipelinePreview | null;
nodes: PipelineGraphNode[]; nodes: PipelineGraphNode[];
working: boolean;
diagnostics: DataflowDiagnostic[]; diagnostics: DataflowDiagnostic[];
nodeDiagnostics: NodePreviewDiagnostic[]; nodeDiagnostics: NodePreviewDiagnostic[];
onPreviewNodeChange: (nodeId: string) => void; selectedNodeId: string | null;
onClose: () => void; onClose: () => void;
}) { }) {
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? ""; const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
const previewNodeId = preview?.node_preview?.node_id ?? outputNodeId; const previewNodeId = preview?.node_preview?.node_id ?? selectedNodeId ?? outputNodeId;
const previewNode = nodes.find((node) => node.id === previewNodeId);
const previewRows = preview?.node_preview?.total_rows ?? preview?.total_rows; const previewRows = preview?.node_preview?.total_rows ?? preview?.total_rows;
return ( return (
<section className="dataflow-results" aria-label="Pipeline results"> <section className="dataflow-results" aria-label="Pipeline results">
@@ -1904,19 +1951,11 @@ function ResultPanel({
value={tab} value={tab}
onChange={onTabChange} onChange={onTabChange}
/> />
{tab === "preview" && nodes.length ? ( {tab === "preview" && previewNode ? (
<select <span className="dataflow-preview-stage">
aria-label="Preview stage" {previewNode.label}
value={previewNodeId} {previewNode.type === "output" ? " (final output)" : ""}
onChange={(event) => onPreviewNodeChange(event.target.value)} </span>
disabled={working}
>
{nodes.map((node) => (
<option key={node.id} value={node.id}>
{node.label}{node.type === "output" ? " (final output)" : ""}
</option>
))}
</select>
) : null} ) : null}
</div> </div>
<Button variant="ghost" onClick={onClose}>Close</Button> <Button variant="ghost" onClick={onClose}>Close</Button>
@@ -2016,6 +2055,26 @@ function uniqueNodeExists(draft: PipelineDraft, type: string): boolean {
return false; return false;
} }
function readAutoPreviewPreference(): boolean {
if (typeof window === "undefined") return true;
try {
return window.localStorage.getItem(AUTO_PREVIEW_STORAGE_KEY) !== "manual";
} catch {
return true;
}
}
function writeAutoPreviewPreference(enabled: boolean): void {
try {
window.localStorage.setItem(
AUTO_PREVIEW_STORAGE_KEY,
enabled ? "auto" : "manual"
);
} catch {
// The preference is non-essential when browser storage is unavailable.
}
}
function startPaletteDrag(event: DragEvent<HTMLButtonElement>, type: string) { function startPaletteDrag(event: DragEvent<HTMLButtonElement>, type: string) {
event.dataTransfer.setData("application/x-govoplan-dataflow-node", type); event.dataTransfer.setData("application/x-govoplan-dataflow-node", type);
event.dataTransfer.effectAllowed = "copy"; event.dataTransfer.effectAllowed = "copy";

View File

@@ -9,6 +9,10 @@ import type { DataflowFlowNode } from "./DataflowNode";
const PROXIMITY_GAP = 72; const PROXIMITY_GAP = 72;
const PROXIMITY_EDGE_ID = "dataflow-proximity-preview"; const PROXIMITY_EDGE_ID = "dataflow-proximity-preview";
export function newGraphEdgeId(): string {
return `edge-${crypto.randomUUID()}`;
}
export function graphEdgesFromFlow(edges: Edge[]): PipelineGraph["edges"] { export function graphEdgesFromFlow(edges: Edge[]): PipelineGraph["edges"] {
return edges return edges
.filter((edge) => edge.id !== PROXIMITY_EDGE_ID) .filter((edge) => edge.id !== PROXIMITY_EDGE_ID)

View File

@@ -343,20 +343,9 @@
gap: 5px; gap: 5px;
} }
.dataflow-alerts { .dataflow-command-bar .toggle-switch-row {
position: absolute; min-height: 34px;
z-index: 12; white-space: nowrap;
top: 66px;
right: 12px;
display: grid;
width: min(500px, calc(100% - 24px));
gap: 6px;
pointer-events: none;
}
.dataflow-alerts .alert {
pointer-events: auto;
box-shadow: var(--shadow-popover);
} }
.dataflow-editor { .dataflow-editor {
@@ -923,9 +912,14 @@
flex-wrap: wrap; flex-wrap: wrap;
} }
.dataflow-results-view-controls select { .dataflow-preview-stage {
width: clamp(150px, 22vw, 280px); max-width: min(36vw, 320px);
min-height: 30px; overflow: hidden;
color: var(--text-strong);
font-size: 12px;
font-weight: 650;
text-overflow: ellipsis;
white-space: nowrap;
} }
.dataflow-preview-table-wrap, .dataflow-preview-table-wrap,