Add intermediate previews and graph reconnection
This commit is contained in:
@@ -34,7 +34,8 @@ nodes by purpose:
|
|||||||
|
|
||||||
Join nodes have explicit left and right ports. Append nodes accept two or more
|
Join nodes have explicit left and right ports. Append nodes accept two or more
|
||||||
inputs. Derived columns use a constrained operation catalogue rather than
|
inputs. Derived columns use a constrained operation catalogue rather than
|
||||||
arbitrary code.
|
arbitrary code. The graph editor validates manual, proximity-created, and
|
||||||
|
reconnected edges against the same port, multiplicity, and cycle rules.
|
||||||
|
|
||||||
Governed inputs are resolved through the versioned Core capability
|
Governed inputs are resolved through the versioned Core capability
|
||||||
`datasources.catalogue`; Dataflow imports neither Datasources nor Connectors and
|
`datasources.catalogue`; Dataflow imports neither Datasources nor Connectors and
|
||||||
@@ -53,7 +54,9 @@ execution.
|
|||||||
Preview reads at most 250 rows per source and enforces time, intermediate-row,
|
Preview reads at most 250 rows per source and enforces time, intermediate-row,
|
||||||
result-byte, graph-node, and response-row bounds. Saved previews record the
|
result-byte, graph-node, and response-row bounds. Saved previews record the
|
||||||
pipeline revision, executor version, source fingerprints, node diagnostics,
|
pipeline revision, executor version, source fingerprints, node diagnostics,
|
||||||
and output summary, but not source or result rows.
|
and output summary, but not source or result rows. A preview may return one
|
||||||
|
explicitly requested intermediate node state under the same response bound;
|
||||||
|
that row payload remains ephemeral and is not added to run evidence.
|
||||||
|
|
||||||
Saved revisions can also be started through the versioned
|
Saved revisions can also be started through the versioned
|
||||||
`dataflow.runLifecycle` capability or the Run dialog. The first runner is
|
`dataflow.runLifecycle` capability or the Run dialog. The first runner is
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from govoplan_dataflow.backend.schemas import (
|
|||||||
DataflowDiagnostic,
|
DataflowDiagnostic,
|
||||||
GraphNode,
|
GraphNode,
|
||||||
NodePreviewDiagnostic,
|
NodePreviewDiagnostic,
|
||||||
|
NodePreviewResult,
|
||||||
PipelineGraph,
|
PipelineGraph,
|
||||||
PreviewColumn,
|
PreviewColumn,
|
||||||
)
|
)
|
||||||
@@ -35,6 +36,7 @@ class PipelineExecutionError(RuntimeError):
|
|||||||
source_fingerprints: tuple[dict[str, Any], ...] = (),
|
source_fingerprints: tuple[dict[str, Any], ...] = (),
|
||||||
input_row_count: int = 0,
|
input_row_count: int = 0,
|
||||||
diagnostics: tuple[DataflowDiagnostic, ...] = (),
|
diagnostics: tuple[DataflowDiagnostic, ...] = (),
|
||||||
|
node_preview: NodePreviewResult | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
self.node_id = node_id
|
self.node_id = node_id
|
||||||
@@ -42,6 +44,7 @@ class PipelineExecutionError(RuntimeError):
|
|||||||
self.source_fingerprints = source_fingerprints
|
self.source_fingerprints = source_fingerprints
|
||||||
self.input_row_count = input_row_count
|
self.input_row_count = input_row_count
|
||||||
self.diagnostics = diagnostics
|
self.diagnostics = diagnostics
|
||||||
|
self.node_preview = node_preview
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -52,6 +55,7 @@ class PipelineExecutionResult:
|
|||||||
columns: list[PreviewColumn]
|
columns: list[PreviewColumn]
|
||||||
diagnostics: list[DataflowDiagnostic]
|
diagnostics: list[DataflowDiagnostic]
|
||||||
node_diagnostics: list[NodePreviewDiagnostic]
|
node_diagnostics: list[NodePreviewDiagnostic]
|
||||||
|
node_preview: NodePreviewResult | None
|
||||||
source_fingerprints: list[dict[str, Any]]
|
source_fingerprints: list[dict[str, Any]]
|
||||||
input_row_count: int
|
input_row_count: int
|
||||||
|
|
||||||
@@ -74,6 +78,7 @@ def execute_preview(
|
|||||||
*,
|
*,
|
||||||
row_limit: int,
|
row_limit: int,
|
||||||
source_resolver: SourceResolver | None = None,
|
source_resolver: SourceResolver | None = None,
|
||||||
|
preview_node_id: str | None = None,
|
||||||
) -> PipelineExecutionResult:
|
) -> PipelineExecutionResult:
|
||||||
validation = validate_graph(graph)
|
validation = validate_graph(graph)
|
||||||
errors = [item for item in validation if item.severity == "error"]
|
errors = [item for item in validation if item.severity == "error"]
|
||||||
@@ -81,6 +86,11 @@ def execute_preview(
|
|||||||
raise PipelineExecutionError(errors[0].message, node_id=errors[0].node_id)
|
raise PipelineExecutionError(errors[0].message, node_id=errors[0].node_id)
|
||||||
|
|
||||||
node_by_id = {node.id: node for node in graph.nodes}
|
node_by_id = {node.id: node for node in graph.nodes}
|
||||||
|
if preview_node_id is not None and preview_node_id not in node_by_id:
|
||||||
|
raise PipelineExecutionError(
|
||||||
|
"The requested preview node is not part of this pipeline.",
|
||||||
|
node_id=preview_node_id,
|
||||||
|
)
|
||||||
inputs = graph_inputs_by_port(graph)
|
inputs = graph_inputs_by_port(graph)
|
||||||
ordered, cyclic = topological_order(graph)
|
ordered, cyclic = topological_order(graph)
|
||||||
if cyclic:
|
if cyclic:
|
||||||
@@ -120,6 +130,7 @@ def execute_preview(
|
|||||||
source_fingerprints=tuple(source_fingerprints),
|
source_fingerprints=tuple(source_fingerprints),
|
||||||
input_row_count=input_row_count,
|
input_row_count=input_row_count,
|
||||||
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
||||||
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
if node.type == "source.inline":
|
if node.type == "source.inline":
|
||||||
@@ -230,6 +241,7 @@ def execute_preview(
|
|||||||
source_fingerprints=tuple(source_fingerprints),
|
source_fingerprints=tuple(source_fingerprints),
|
||||||
input_row_count=input_row_count,
|
input_row_count=input_row_count,
|
||||||
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
||||||
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
outputs[node.id] = output_rows
|
outputs[node.id] = output_rows
|
||||||
@@ -255,11 +267,30 @@ def execute_preview(
|
|||||||
columns=infer_columns(all_rows),
|
columns=infer_columns(all_rows),
|
||||||
diagnostics=[item for item in validation if item.severity != "error"],
|
diagnostics=[item for item in validation if item.severity != "error"],
|
||||||
node_diagnostics=node_diagnostics,
|
node_diagnostics=node_diagnostics,
|
||||||
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
||||||
source_fingerprints=source_fingerprints,
|
source_fingerprints=source_fingerprints,
|
||||||
input_row_count=input_row_count,
|
input_row_count=input_row_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _node_preview(
|
||||||
|
outputs: dict[str, list[dict[str, Any]]],
|
||||||
|
node_id: str | None,
|
||||||
|
row_limit: int,
|
||||||
|
) -> NodePreviewResult | None:
|
||||||
|
if node_id is None or node_id not in outputs:
|
||||||
|
return None
|
||||||
|
all_rows = outputs[node_id]
|
||||||
|
rows = [dict(row) for row in all_rows[:row_limit]]
|
||||||
|
return NodePreviewResult(
|
||||||
|
node_id=node_id,
|
||||||
|
columns=infer_columns(all_rows),
|
||||||
|
rows=rows,
|
||||||
|
total_rows=len(all_rows),
|
||||||
|
truncated=len(rows) < len(all_rows),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def infer_columns(rows: list[dict[str, Any]]) -> list[PreviewColumn]:
|
def infer_columns(rows: list[dict[str, Any]]) -> list[PreviewColumn]:
|
||||||
names: list[str] = []
|
names: list[str] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
|||||||
@@ -1096,6 +1096,11 @@ def api_preview_pipeline(
|
|||||||
"run_id": response.run_id,
|
"run_id": response.run_id,
|
||||||
"status": response.status,
|
"status": response.status,
|
||||||
"output_row_count": response.total_rows,
|
"output_row_count": response.total_rows,
|
||||||
|
"preview_node_id": (
|
||||||
|
response.node_preview.node_id
|
||||||
|
if response.node_preview is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|||||||
@@ -184,6 +184,12 @@ class PipelinePreviewRequest(BaseModel):
|
|||||||
graph: PipelineGraph | None = None
|
graph: PipelineGraph | None = None
|
||||||
sql_text: str | None = Field(default=None, max_length=100_000)
|
sql_text: str | None = Field(default=None, max_length=100_000)
|
||||||
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
|
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
|
||||||
|
preview_node_id: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
min_length=1,
|
||||||
|
max_length=100,
|
||||||
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
||||||
|
)
|
||||||
row_limit: int = Field(default=100, ge=1, le=500)
|
row_limit: int = Field(default=100, ge=1, le=500)
|
||||||
|
|
||||||
|
|
||||||
@@ -203,6 +209,14 @@ class NodePreviewDiagnostic(BaseModel):
|
|||||||
messages: list[str] = Field(default_factory=list)
|
messages: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class NodePreviewResult(BaseModel):
|
||||||
|
node_id: str
|
||||||
|
columns: list[PreviewColumn]
|
||||||
|
rows: list[dict[str, Any]]
|
||||||
|
total_rows: int
|
||||||
|
truncated: bool
|
||||||
|
|
||||||
|
|
||||||
class PipelinePreviewResponse(BaseModel):
|
class PipelinePreviewResponse(BaseModel):
|
||||||
run_id: str | None
|
run_id: str | None
|
||||||
pipeline_id: str | None
|
pipeline_id: str | None
|
||||||
@@ -214,6 +228,7 @@ class PipelinePreviewResponse(BaseModel):
|
|||||||
truncated: bool
|
truncated: bool
|
||||||
diagnostics: list[DataflowDiagnostic]
|
diagnostics: list[DataflowDiagnostic]
|
||||||
node_diagnostics: list[NodePreviewDiagnostic]
|
node_diagnostics: list[NodePreviewDiagnostic]
|
||||||
|
node_preview: NodePreviewResult | None = None
|
||||||
source_fingerprints: list[dict[str, Any]]
|
source_fingerprints: list[dict[str, Any]]
|
||||||
input_row_count: int
|
input_row_count: int
|
||||||
definition_hash: str
|
definition_hash: str
|
||||||
|
|||||||
@@ -567,6 +567,7 @@ def preview_pipeline(
|
|||||||
truncated=False,
|
truncated=False,
|
||||||
diagnostics=validated.diagnostics,
|
diagnostics=validated.diagnostics,
|
||||||
node_diagnostics=[],
|
node_diagnostics=[],
|
||||||
|
node_preview=None,
|
||||||
source_fingerprints=[],
|
source_fingerprints=[],
|
||||||
input_row_count=0,
|
input_row_count=0,
|
||||||
definition_hash="",
|
definition_hash="",
|
||||||
@@ -622,6 +623,7 @@ def preview_pipeline(
|
|||||||
graph,
|
graph,
|
||||||
row_limit=payload.row_limit,
|
row_limit=payload.row_limit,
|
||||||
source_resolver=resolve_source,
|
source_resolver=resolve_source,
|
||||||
|
preview_node_id=payload.preview_node_id,
|
||||||
)
|
)
|
||||||
status = "succeeded"
|
status = "succeeded"
|
||||||
error = None
|
error = None
|
||||||
@@ -631,6 +633,7 @@ def preview_pipeline(
|
|||||||
total_rows = result.total_rows
|
total_rows = result.total_rows
|
||||||
truncated = result.truncated
|
truncated = result.truncated
|
||||||
node_diagnostics = result.node_diagnostics
|
node_diagnostics = result.node_diagnostics
|
||||||
|
node_preview = result.node_preview
|
||||||
source_fingerprints = result.source_fingerprints
|
source_fingerprints = result.source_fingerprints
|
||||||
input_row_count = result.input_row_count
|
input_row_count = result.input_row_count
|
||||||
except PipelineExecutionError as exc:
|
except PipelineExecutionError as exc:
|
||||||
@@ -650,6 +653,7 @@ def preview_pipeline(
|
|||||||
total_rows = 0
|
total_rows = 0
|
||||||
truncated = False
|
truncated = False
|
||||||
node_diagnostics = list(exc.node_diagnostics)
|
node_diagnostics = list(exc.node_diagnostics)
|
||||||
|
node_preview = exc.node_preview
|
||||||
source_fingerprints = list(exc.source_fingerprints)
|
source_fingerprints = list(exc.source_fingerprints)
|
||||||
input_row_count = exc.input_row_count
|
input_row_count = exc.input_row_count
|
||||||
|
|
||||||
@@ -686,6 +690,7 @@ def preview_pipeline(
|
|||||||
truncated=truncated,
|
truncated=truncated,
|
||||||
diagnostics=diagnostics,
|
diagnostics=diagnostics,
|
||||||
node_diagnostics=node_diagnostics,
|
node_diagnostics=node_diagnostics,
|
||||||
|
node_preview=node_preview,
|
||||||
source_fingerprints=source_fingerprints,
|
source_fingerprints=source_fingerprints,
|
||||||
input_row_count=input_row_count,
|
input_row_count=input_row_count,
|
||||||
definition_hash=graph_hash,
|
definition_hash=graph_hash,
|
||||||
|
|||||||
@@ -446,6 +446,43 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
|||||||
self.assertEqual(3, result.total_rows)
|
self.assertEqual(3, result.total_rows)
|
||||||
self.assertTrue(result.truncated)
|
self.assertTrue(result.truncated)
|
||||||
|
|
||||||
|
def test_preview_can_return_a_bounded_intermediate_node_result(self) -> None:
|
||||||
|
source = inline_source()
|
||||||
|
filter_node = GraphNode(
|
||||||
|
id="filter",
|
||||||
|
type="filter",
|
||||||
|
label="Open cases",
|
||||||
|
position=GraphPosition(x=240, y=160),
|
||||||
|
config={"column": "status", "operator": "eq", "value": "open"},
|
||||||
|
)
|
||||||
|
output = GraphNode(
|
||||||
|
id="output",
|
||||||
|
type="output",
|
||||||
|
label="Output",
|
||||||
|
position=GraphPosition(x=440, y=160),
|
||||||
|
config={},
|
||||||
|
)
|
||||||
|
graph = PipelineGraph(
|
||||||
|
nodes=[source, filter_node, output],
|
||||||
|
edges=[
|
||||||
|
GraphEdge(id="source-filter", source="source", target="filter"),
|
||||||
|
GraphEdge(id="filter-output", source="filter", target="output"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = execute_preview(
|
||||||
|
graph,
|
||||||
|
row_limit=1,
|
||||||
|
preview_node_id="source",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, result.total_rows)
|
||||||
|
self.assertIsNotNone(result.node_preview)
|
||||||
|
self.assertEqual("source", result.node_preview.node_id)
|
||||||
|
self.assertEqual(3, result.node_preview.total_rows)
|
||||||
|
self.assertEqual(1, len(result.node_preview.rows))
|
||||||
|
self.assertTrue(result.node_preview.truncated)
|
||||||
|
|
||||||
def test_empty_aggregate_and_null_sort_are_deterministic(self) -> None:
|
def test_empty_aggregate_and_null_sort_are_deterministic(self) -> None:
|
||||||
graph, _, _ = compile_sql(
|
graph, _, _ = compile_sql(
|
||||||
"""
|
"""
|
||||||
@@ -524,7 +561,7 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with self.assertRaises(PipelineExecutionError) as raised:
|
with self.assertRaises(PipelineExecutionError) as raised:
|
||||||
execute_preview(graph, row_limit=100)
|
execute_preview(graph, row_limit=100, preview_node_id="source")
|
||||||
|
|
||||||
self.assertEqual("filter-1", raised.exception.node_id)
|
self.assertEqual("filter-1", raised.exception.node_id)
|
||||||
self.assertIn("Cannot apply", str(raised.exception))
|
self.assertIn("Cannot apply", str(raised.exception))
|
||||||
@@ -536,6 +573,12 @@ class DataflowGraphAndSqlTests(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
self.assertEqual(1, len(raised.exception.source_fingerprints))
|
self.assertEqual(1, len(raised.exception.source_fingerprints))
|
||||||
|
self.assertIsNotNone(raised.exception.node_preview)
|
||||||
|
self.assertEqual("source", raised.exception.node_preview.node_id)
|
||||||
|
self.assertEqual(
|
||||||
|
[{"amount": "not-a-number"}],
|
||||||
|
raised.exception.node_preview.rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -277,7 +277,11 @@ class DataflowServiceTests(unittest.TestCase):
|
|||||||
self.session,
|
self.session,
|
||||||
tenant_id="tenant-1",
|
tenant_id="tenant-1",
|
||||||
actor_id="user-1",
|
actor_id="user-1",
|
||||||
payload=PipelinePreviewRequest(pipeline_id=pipeline.id, row_limit=1),
|
payload=PipelinePreviewRequest(
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
preview_node_id="source",
|
||||||
|
row_limit=1,
|
||||||
|
),
|
||||||
principal=principal(),
|
principal=principal(),
|
||||||
)
|
)
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
@@ -287,6 +291,11 @@ class DataflowServiceTests(unittest.TestCase):
|
|||||||
self.assertEqual([{"id": 2, "amount": 15}], response.rows)
|
self.assertEqual([{"id": 2, "amount": 15}], response.rows)
|
||||||
self.assertEqual(2, response.total_rows)
|
self.assertEqual(2, response.total_rows)
|
||||||
self.assertTrue(response.truncated)
|
self.assertTrue(response.truncated)
|
||||||
|
self.assertIsNotNone(response.node_preview)
|
||||||
|
self.assertEqual("source", response.node_preview.node_id)
|
||||||
|
self.assertEqual([{"id": 1, "amount": 5}], response.node_preview.rows)
|
||||||
|
self.assertEqual(3, response.node_preview.total_rows)
|
||||||
|
self.assertTrue(response.node_preview.truncated)
|
||||||
self.assertEqual(2, run.output_row_count)
|
self.assertEqual(2, run.output_row_count)
|
||||||
self.assertEqual(3, run.input_row_count)
|
self.assertEqual(3, run.input_row_count)
|
||||||
self.assertEqual(1, len(run.source_fingerprints))
|
self.assertEqual(1, len(run.source_fingerprints))
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
|
|||||||
|
|
||||||
const page = readFileSync(new URL("../src/features/dataflow/DataflowPage.tsx", import.meta.url), "utf8");
|
const page = readFileSync(new URL("../src/features/dataflow/DataflowPage.tsx", import.meta.url), "utf8");
|
||||||
const canvas = readFileSync(new URL("../src/features/dataflow/DataflowCanvas.tsx", import.meta.url), "utf8");
|
const canvas = readFileSync(new URL("../src/features/dataflow/DataflowCanvas.tsx", import.meta.url), "utf8");
|
||||||
|
const interactions = readFileSync(new URL("../src/features/dataflow/graphInteractions.ts", import.meta.url), "utf8");
|
||||||
const css = readFileSync(new URL("../src/styles/dataflow.css", import.meta.url), "utf8");
|
const css = readFileSync(new URL("../src/styles/dataflow.css", import.meta.url), "utf8");
|
||||||
const moduleEntry = readFileSync(new URL("../src/module.ts", import.meta.url), "utf8");
|
const moduleEntry = readFileSync(new URL("../src/module.ts", import.meta.url), "utf8");
|
||||||
|
|
||||||
@@ -16,6 +17,15 @@ const checks = [
|
|||||||
[canvas.includes("connectionRadius={32}"), "visible connection drop radius"],
|
[canvas.includes("connectionRadius={32}"), "visible connection drop radius"],
|
||||||
[canvas.includes("initialWidth: 180"), "known initial node width"],
|
[canvas.includes("initialWidth: 180"), "known initial node width"],
|
||||||
[canvas.includes('change.type !== "dimensions"'), "measurement-only change filtering"],
|
[canvas.includes('change.type !== "dimensions"'), "measurement-only change filtering"],
|
||||||
|
[canvas.includes("onReconnect={onReconnect}"), "edge reconnection"],
|
||||||
|
[canvas.includes("onReconnectEnd="), "edge deletion on dropped reconnection"],
|
||||||
|
[canvas.includes("selectedEdgeId"), "persistent edge selection"],
|
||||||
|
[
|
||||||
|
canvas.includes("closestProximityConnection")
|
||||||
|
&& interactions.includes("definitionConnectionError"),
|
||||||
|
"validated proximity connection"
|
||||||
|
],
|
||||||
|
[css.includes(".dataflow-edge-proximity"), "proximity connection indicator"],
|
||||||
[
|
[
|
||||||
moduleEntry.indexOf("@xyflow/react/dist/style.css") <
|
moduleEntry.indexOf("@xyflow/react/dist/style.css") <
|
||||||
moduleEntry.indexOf("./styles/dataflow.css"),
|
moduleEntry.indexOf("./styles/dataflow.css"),
|
||||||
|
|||||||
@@ -125,6 +125,14 @@ export type NodePreviewDiagnostic = {
|
|||||||
messages: string[];
|
messages: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type NodePreviewResult = {
|
||||||
|
node_id: string;
|
||||||
|
columns: PreviewColumn[];
|
||||||
|
rows: Record<string, unknown>[];
|
||||||
|
total_rows: number;
|
||||||
|
truncated: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type PipelinePreview = {
|
export type PipelinePreview = {
|
||||||
run_id?: string | null;
|
run_id?: string | null;
|
||||||
pipeline_id?: string | null;
|
pipeline_id?: string | null;
|
||||||
@@ -136,6 +144,7 @@ export type PipelinePreview = {
|
|||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
diagnostics: DataflowDiagnostic[];
|
diagnostics: DataflowDiagnostic[];
|
||||||
node_diagnostics: NodePreviewDiagnostic[];
|
node_diagnostics: NodePreviewDiagnostic[];
|
||||||
|
node_preview?: NodePreviewResult | null;
|
||||||
source_fingerprints: Record<string, unknown>[];
|
source_fingerprints: Record<string, unknown>[];
|
||||||
input_row_count: number;
|
input_row_count: number;
|
||||||
definition_hash: string;
|
definition_hash: string;
|
||||||
@@ -451,6 +460,7 @@ export function previewDataflowPipeline(
|
|||||||
graph?: PipelineGraph;
|
graph?: PipelineGraph;
|
||||||
sql_text?: string;
|
sql_text?: string;
|
||||||
source_nodes?: PipelineGraphNode[];
|
source_nodes?: PipelineGraphNode[];
|
||||||
|
preview_node_id?: string;
|
||||||
row_limit?: number;
|
row_limit?: number;
|
||||||
}
|
}
|
||||||
): Promise<PipelinePreview> {
|
): Promise<PipelinePreview> {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { useMemo, useState, type DragEvent } from "react";
|
import { useMemo, useRef, useState, type DragEvent } from "react";
|
||||||
import {
|
import {
|
||||||
addEdge,
|
addEdge,
|
||||||
applyEdgeChanges,
|
applyEdgeChanges,
|
||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
Controls,
|
Controls,
|
||||||
MiniMap,
|
MiniMap,
|
||||||
ReactFlow,
|
ReactFlow,
|
||||||
|
reconnectEdge,
|
||||||
type Connection,
|
type Connection,
|
||||||
type Edge,
|
type Edge,
|
||||||
type ReactFlowInstance
|
type ReactFlowInstance
|
||||||
@@ -22,6 +23,11 @@ import type {
|
|||||||
PipelineGraphNode
|
PipelineGraphNode
|
||||||
} from "../../api/dataflow";
|
} from "../../api/dataflow";
|
||||||
import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
|
import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
|
||||||
|
import {
|
||||||
|
closestProximityConnection,
|
||||||
|
graphEdgesFromFlow,
|
||||||
|
proximityPreviewEdge
|
||||||
|
} from "./graphInteractions";
|
||||||
import { FALLBACK_NODE_LIBRARY, newNode } from "./model";
|
import { FALLBACK_NODE_LIBRARY, newNode } from "./model";
|
||||||
|
|
||||||
const nodeTypes = { dataflow: DataflowNode };
|
const nodeTypes = { dataflow: DataflowNode };
|
||||||
@@ -48,6 +54,10 @@ export default function DataflowCanvas({
|
|||||||
onSelectNode
|
onSelectNode
|
||||||
}: DataflowCanvasProps) {
|
}: DataflowCanvasProps) {
|
||||||
const [instance, setInstance] = useState<ReactFlowInstance<DataflowFlowNode, Edge> | null>(null);
|
const [instance, setInstance] = useState<ReactFlowInstance<DataflowFlowNode, Edge> | null>(null);
|
||||||
|
const [proximityEdge, setProximityEdge] = useState<Edge | null>(null);
|
||||||
|
const [selectedEdgeId, setSelectedEdgeId] = useState<string | null>(null);
|
||||||
|
const reconnectSuccessful = useRef(true);
|
||||||
|
const reconnectingEdgeId = useRef<string | null>(null);
|
||||||
const errorNodeIds = useMemo(
|
const errorNodeIds = useMemo(
|
||||||
() => new Set(diagnostics.filter((item) => item.severity === "error" && item.node_id).map((item) => item.node_id)),
|
() => new Set(diagnostics.filter((item) => item.severity === "error" && item.node_id).map((item) => item.node_id)),
|
||||||
[diagnostics]
|
[diagnostics]
|
||||||
@@ -92,9 +102,14 @@ export default function DataflowCanvas({
|
|||||||
sourceHandle: edge.source_port ?? "output",
|
sourceHandle: edge.source_port ?? "output",
|
||||||
targetHandle: edge.target_port ?? "input",
|
targetHandle: edge.target_port ?? "input",
|
||||||
type: "smoothstep",
|
type: "smoothstep",
|
||||||
className: "dataflow-edge"
|
className: "dataflow-edge",
|
||||||
|
selected: edge.id === selectedEdgeId
|
||||||
})),
|
})),
|
||||||
[graph.edges]
|
[graph.edges, selectedEdgeId]
|
||||||
|
);
|
||||||
|
const displayedEdges = useMemo(
|
||||||
|
() => proximityEdge ? [...edges, proximityEdge] : edges,
|
||||||
|
[edges, proximityEdge]
|
||||||
);
|
);
|
||||||
|
|
||||||
const updateNodes = (nextNodes: DataflowFlowNode[]) => {
|
const updateNodes = (nextNodes: DataflowFlowNode[]) => {
|
||||||
@@ -116,20 +131,20 @@ export default function DataflowCanvas({
|
|||||||
const updateEdges = (nextEdges: Edge[]) => {
|
const updateEdges = (nextEdges: Edge[]) => {
|
||||||
onGraphChange({
|
onGraphChange({
|
||||||
...graph,
|
...graph,
|
||||||
edges: nextEdges.map((edge) => ({
|
edges: graphEdgesFromFlow(nextEdges)
|
||||||
id: edge.id,
|
|
||||||
source: edge.source,
|
|
||||||
target: edge.target,
|
|
||||||
source_port: edge.sourceHandle ?? "output",
|
|
||||||
target_port: edge.targetHandle ?? "input"
|
|
||||||
}))
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const isValidConnection = (connection: Connection | Edge): boolean => {
|
const connectionError = (
|
||||||
if (readOnly || !connection.source || !connection.target) return false;
|
connection: Connection | Edge,
|
||||||
return definitionConnectionError(
|
ignoredEdgeId: string | null = reconnectingEdgeId.current
|
||||||
graph,
|
): string | null => definitionConnectionError(
|
||||||
|
ignoredEdgeId
|
||||||
|
? {
|
||||||
|
...graph,
|
||||||
|
edges: graph.edges.filter((edge) => edge.id !== ignoredEdgeId)
|
||||||
|
}
|
||||||
|
: graph,
|
||||||
nodeLibrary,
|
nodeLibrary,
|
||||||
{
|
{
|
||||||
source: connection.source,
|
source: connection.source,
|
||||||
@@ -137,7 +152,11 @@ export default function DataflowCanvas({
|
|||||||
sourcePort: connection.sourceHandle,
|
sourcePort: connection.sourceHandle,
|
||||||
targetPort: connection.targetHandle
|
targetPort: connection.targetHandle
|
||||||
}
|
}
|
||||||
) === null;
|
);
|
||||||
|
|
||||||
|
const isValidConnection = (connection: Connection | Edge): boolean => {
|
||||||
|
if (readOnly || !connection.source || !connection.target) return false;
|
||||||
|
return connectionError(connection) === null;
|
||||||
};
|
};
|
||||||
|
|
||||||
const onConnect = (connection: Connection) => {
|
const onConnect = (connection: Connection) => {
|
||||||
@@ -153,6 +172,26 @@ export default function DataflowCanvas({
|
|||||||
updateEdges(next);
|
updateEdges(next);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const onReconnect = (oldEdge: Edge, connection: Connection) => {
|
||||||
|
if (readOnly || connectionError(connection, oldEdge.id)) return;
|
||||||
|
reconnectSuccessful.current = true;
|
||||||
|
updateEdges(reconnectEdge(
|
||||||
|
oldEdge,
|
||||||
|
connection,
|
||||||
|
edges,
|
||||||
|
{ shouldReplaceId: false }
|
||||||
|
));
|
||||||
|
};
|
||||||
|
|
||||||
|
const proximityConnection = (draggedNode: DataflowFlowNode) => (
|
||||||
|
closestProximityConnection(
|
||||||
|
draggedNode,
|
||||||
|
nodes,
|
||||||
|
graph,
|
||||||
|
nodeLibrary
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
const onDrop = (event: DragEvent<HTMLDivElement>) => {
|
const onDrop = (event: DragEvent<HTMLDivElement>) => {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
if (readOnly || !instance) return;
|
if (readOnly || !instance) return;
|
||||||
@@ -177,7 +216,7 @@ export default function DataflowCanvas({
|
|||||||
>
|
>
|
||||||
<ReactFlow<DataflowFlowNode, Edge>
|
<ReactFlow<DataflowFlowNode, Edge>
|
||||||
nodes={nodes}
|
nodes={nodes}
|
||||||
edges={edges}
|
edges={displayedEdges}
|
||||||
nodeTypes={nodeTypes}
|
nodeTypes={nodeTypes}
|
||||||
onInit={setInstance}
|
onInit={setInstance}
|
||||||
onNodesChange={(changes) => {
|
onNodesChange={(changes) => {
|
||||||
@@ -189,14 +228,90 @@ export default function DataflowCanvas({
|
|||||||
}}
|
}}
|
||||||
onEdgesChange={(changes) => {
|
onEdgesChange={(changes) => {
|
||||||
if (readOnly) return;
|
if (readOnly) return;
|
||||||
updateEdges(applyEdgeChanges(changes, edges));
|
const selectedChange = changes.find(
|
||||||
|
(change) => change.type === "select" && change.selected
|
||||||
|
);
|
||||||
|
if (selectedChange?.type === "select") {
|
||||||
|
setSelectedEdgeId(selectedChange.id);
|
||||||
|
} else if (changes.some(
|
||||||
|
(change) => (
|
||||||
|
change.type === "select"
|
||||||
|
&& change.id === selectedEdgeId
|
||||||
|
&& !change.selected
|
||||||
|
)
|
||||||
|
)) {
|
||||||
|
setSelectedEdgeId(null);
|
||||||
|
}
|
||||||
|
const graphChanges = changes.filter((change) => change.type !== "select");
|
||||||
|
if (graphChanges.length) {
|
||||||
|
if (graphChanges.some(
|
||||||
|
(change) => change.type === "remove" && change.id === selectedEdgeId
|
||||||
|
)) {
|
||||||
|
setSelectedEdgeId(null);
|
||||||
|
}
|
||||||
|
updateEdges(applyEdgeChanges(graphChanges, edges));
|
||||||
|
}
|
||||||
}}
|
}}
|
||||||
onConnect={onConnect}
|
onConnect={onConnect}
|
||||||
|
onReconnect={onReconnect}
|
||||||
|
onReconnectStart={(_event, edge) => {
|
||||||
|
reconnectSuccessful.current = false;
|
||||||
|
reconnectingEdgeId.current = edge.id;
|
||||||
|
setProximityEdge(null);
|
||||||
|
}}
|
||||||
|
onReconnectEnd={(_event, edge) => {
|
||||||
|
if (!reconnectSuccessful.current && !readOnly) {
|
||||||
|
updateEdges(edges.filter((candidate) => candidate.id !== edge.id));
|
||||||
|
setSelectedEdgeId(null);
|
||||||
|
}
|
||||||
|
reconnectSuccessful.current = true;
|
||||||
|
reconnectingEdgeId.current = null;
|
||||||
|
}}
|
||||||
|
onNodeDrag={(_event, node) => {
|
||||||
|
if (readOnly) return;
|
||||||
|
const connection = proximityConnection(node);
|
||||||
|
setProximityEdge(connection ? proximityPreviewEdge(connection) : null);
|
||||||
|
}}
|
||||||
|
onNodeDragStop={(_event, node) => {
|
||||||
|
setProximityEdge(null);
|
||||||
|
if (readOnly) return;
|
||||||
|
const connection = proximityConnection(node);
|
||||||
|
if (!connection) return;
|
||||||
|
const nextEdges = addEdge(
|
||||||
|
{
|
||||||
|
...connection,
|
||||||
|
id: `edge-${crypto.randomUUID()}`,
|
||||||
|
type: "smoothstep",
|
||||||
|
className: "dataflow-edge"
|
||||||
|
},
|
||||||
|
edges
|
||||||
|
);
|
||||||
|
onGraphChange({
|
||||||
|
...graph,
|
||||||
|
nodes: graph.nodes.map((graphNode) => (
|
||||||
|
graphNode.id === node.id
|
||||||
|
? { ...graphNode, position: node.position }
|
||||||
|
: graphNode
|
||||||
|
)),
|
||||||
|
edges: graphEdgesFromFlow(nextEdges)
|
||||||
|
});
|
||||||
|
}}
|
||||||
isValidConnection={isValidConnection}
|
isValidConnection={isValidConnection}
|
||||||
onNodeClick={(_event, node) => onSelectNode(node.id)}
|
onNodeClick={(_event, node) => {
|
||||||
onPaneClick={() => onSelectNode(null)}
|
setSelectedEdgeId(null);
|
||||||
|
onSelectNode(node.id);
|
||||||
|
}}
|
||||||
|
onEdgeClick={(_event, edge) => {
|
||||||
|
setSelectedEdgeId(edge.id);
|
||||||
|
onSelectNode(null);
|
||||||
|
}}
|
||||||
|
onPaneClick={() => {
|
||||||
|
setSelectedEdgeId(null);
|
||||||
|
onSelectNode(null);
|
||||||
|
}}
|
||||||
nodesDraggable={!readOnly}
|
nodesDraggable={!readOnly}
|
||||||
nodesConnectable={!readOnly}
|
nodesConnectable={!readOnly}
|
||||||
|
edgesReconnectable={!readOnly}
|
||||||
elementsSelectable
|
elementsSelectable
|
||||||
deleteKeyCode={readOnly ? null : ["Backspace", "Delete"]}
|
deleteKeyCode={readOnly ? null : ["Backspace", "Delete"]}
|
||||||
connectionLineType={ConnectionLineType.SmoothStep}
|
connectionLineType={ConnectionLineType.SmoothStep}
|
||||||
|
|||||||
@@ -348,6 +348,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
updateDraft({ graph });
|
updateDraft({ graph });
|
||||||
setDiagnostics([]);
|
setDiagnostics([]);
|
||||||
setNodeDiagnostics([]);
|
setNodeDiagnostics([]);
|
||||||
|
setPreview(null);
|
||||||
};
|
};
|
||||||
|
|
||||||
const validate = async () => {
|
const validate = async () => {
|
||||||
@@ -425,27 +426,45 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
await applySql(true);
|
await applySql(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const runPreview = async () => {
|
const runPreview = async (previewNodeId?: string) => {
|
||||||
if (!draft || !canRun) return;
|
if (!draft || !canPreview) return;
|
||||||
|
const targetNodeId = previewNodeId
|
||||||
|
?? draft.graph.nodes.find((node) => node.type === "output")?.id;
|
||||||
setWorking(true);
|
setWorking(true);
|
||||||
setError("");
|
setError("");
|
||||||
setSuccess("");
|
setSuccess("");
|
||||||
try {
|
try {
|
||||||
const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision
|
const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision
|
||||||
? { pipeline_id: draft.id, revision: draft.currentRevision, row_limit: 100 }
|
? {
|
||||||
|
pipeline_id: draft.id,
|
||||||
|
revision: draft.currentRevision,
|
||||||
|
preview_node_id: targetNodeId,
|
||||||
|
row_limit: 100
|
||||||
|
}
|
||||||
: {
|
: {
|
||||||
graph: draft.graph,
|
graph: draft.graph,
|
||||||
sql_text: draft.editorMode === "sql" ? draft.sqlText : undefined,
|
sql_text: draft.editorMode === "sql" ? draft.sqlText : undefined,
|
||||||
source_nodes: sourceNodes(draft.graph),
|
source_nodes: sourceNodes(draft.graph),
|
||||||
|
preview_node_id: targetNodeId,
|
||||||
row_limit: 100
|
row_limit: 100
|
||||||
});
|
});
|
||||||
setPreview(response);
|
setPreview(response);
|
||||||
setDiagnostics(response.diagnostics);
|
setDiagnostics(response.diagnostics);
|
||||||
setNodeDiagnostics(response.node_diagnostics);
|
setNodeDiagnostics(response.node_diagnostics);
|
||||||
setResultOpen(true);
|
setResultOpen(true);
|
||||||
setResultTab(response.status === "succeeded" ? "preview" : "diagnostics");
|
setResultTab(
|
||||||
if (response.status === "succeeded") {
|
response.status === "succeeded" || response.node_preview
|
||||||
setSuccess(`Preview produced ${response.total_rows} row${response.total_rows === 1 ? "" : "s"}.`);
|
? "preview"
|
||||||
|
: "diagnostics"
|
||||||
|
);
|
||||||
|
if (response.status === "succeeded" && response.node_preview) {
|
||||||
|
const node = draft.graph.nodes.find(
|
||||||
|
(candidate) => candidate.id === response.node_preview?.node_id
|
||||||
|
);
|
||||||
|
setSuccess(
|
||||||
|
`${node?.label ?? "Preview"} produced ${response.node_preview.total_rows} `
|
||||||
|
+ `row${response.node_preview.total_rows === 1 ? "" : "s"}.`
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch (previewError) {
|
} catch (previewError) {
|
||||||
setError(apiErrorMessage(previewError));
|
setError(apiErrorMessage(previewError));
|
||||||
@@ -754,7 +773,10 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
sources={sources}
|
sources={sources}
|
||||||
sourceCatalogueAvailable={sourceCatalogueAvailable}
|
sourceCatalogueAvailable={sourceCatalogueAvailable}
|
||||||
readOnly={!canEdit}
|
readOnly={!canEdit}
|
||||||
|
canPreview={canPreview}
|
||||||
|
previewing={working}
|
||||||
onChange={(node: PipelineGraphNode) => updateGraph(updateGraphNode(draft.graph, node))}
|
onChange={(node: PipelineGraphNode) => updateGraph(updateGraphNode(draft.graph, node))}
|
||||||
|
onPreview={(nodeId) => void runPreview(nodeId)}
|
||||||
onDelete={(nodeId) => {
|
onDelete={(nodeId) => {
|
||||||
updateGraph({
|
updateGraph({
|
||||||
...draft.graph,
|
...draft.graph,
|
||||||
@@ -770,8 +792,11 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
tab={resultTab}
|
tab={resultTab}
|
||||||
onTabChange={setResultTab}
|
onTabChange={setResultTab}
|
||||||
preview={preview}
|
preview={preview}
|
||||||
|
nodes={draft.graph.nodes}
|
||||||
|
working={working}
|
||||||
diagnostics={diagnostics}
|
diagnostics={diagnostics}
|
||||||
nodeDiagnostics={nodeDiagnostics}
|
nodeDiagnostics={nodeDiagnostics}
|
||||||
|
onPreviewNodeChange={(nodeId) => void runPreview(nodeId)}
|
||||||
onClose={() => setResultOpen(false)}
|
onClose={() => setResultOpen(false)}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -1846,29 +1871,54 @@ function ResultPanel({
|
|||||||
tab,
|
tab,
|
||||||
onTabChange,
|
onTabChange,
|
||||||
preview,
|
preview,
|
||||||
|
nodes,
|
||||||
|
working,
|
||||||
diagnostics,
|
diagnostics,
|
||||||
nodeDiagnostics,
|
nodeDiagnostics,
|
||||||
|
onPreviewNodeChange,
|
||||||
onClose
|
onClose
|
||||||
}: {
|
}: {
|
||||||
tab: ResultTab;
|
tab: ResultTab;
|
||||||
onTabChange: (tab: ResultTab) => void;
|
onTabChange: (tab: ResultTab) => void;
|
||||||
preview: PipelinePreview | null;
|
preview: PipelinePreview | null;
|
||||||
|
nodes: PipelineGraphNode[];
|
||||||
|
working: boolean;
|
||||||
diagnostics: DataflowDiagnostic[];
|
diagnostics: DataflowDiagnostic[];
|
||||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||||
|
onPreviewNodeChange: (nodeId: string) => void;
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
}) {
|
}) {
|
||||||
|
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
|
||||||
|
const previewNodeId = preview?.node_preview?.node_id ?? outputNodeId;
|
||||||
|
const previewRows = preview?.node_preview?.total_rows ?? preview?.total_rows;
|
||||||
return (
|
return (
|
||||||
<section className="dataflow-results" aria-label="Pipeline results">
|
<section className="dataflow-results" aria-label="Pipeline results">
|
||||||
<div className="dataflow-results-toolbar">
|
<div className="dataflow-results-toolbar">
|
||||||
|
<div className="dataflow-results-view-controls">
|
||||||
<SegmentedControl<ResultTab>
|
<SegmentedControl<ResultTab>
|
||||||
ariaLabel="Result view"
|
ariaLabel="Result view"
|
||||||
options={[
|
options={[
|
||||||
{ id: "preview", label: `Preview${preview ? ` (${preview.total_rows})` : ""}` },
|
{ id: "preview", label: `Preview${previewRows !== undefined ? ` (${previewRows})` : ""}` },
|
||||||
{ id: "diagnostics", label: `Diagnostics (${diagnostics.length + nodeDiagnostics.length})` }
|
{ id: "diagnostics", label: `Diagnostics (${diagnostics.length + nodeDiagnostics.length})` }
|
||||||
]}
|
]}
|
||||||
value={tab}
|
value={tab}
|
||||||
onChange={onTabChange}
|
onChange={onTabChange}
|
||||||
/>
|
/>
|
||||||
|
{tab === "preview" && nodes.length ? (
|
||||||
|
<select
|
||||||
|
aria-label="Preview stage"
|
||||||
|
value={previewNodeId}
|
||||||
|
onChange={(event) => onPreviewNodeChange(event.target.value)}
|
||||||
|
disabled={working}
|
||||||
|
>
|
||||||
|
{nodes.map((node) => (
|
||||||
|
<option key={node.id} value={node.id}>
|
||||||
|
{node.label}{node.type === "output" ? " (final output)" : ""}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
<Button variant="ghost" onClick={onClose}>Close</Button>
|
<Button variant="ghost" onClick={onClose}>Close</Button>
|
||||||
</div>
|
</div>
|
||||||
{tab === "preview" ? (
|
{tab === "preview" ? (
|
||||||
@@ -1882,13 +1932,16 @@ function ResultPanel({
|
|||||||
|
|
||||||
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
||||||
if (!preview) return <div className="dataflow-results-empty">No preview has been run.</div>;
|
if (!preview) return <div className="dataflow-results-empty">No preview has been run.</div>;
|
||||||
if (preview.status === "failed") return <div className="dataflow-results-empty">Preview failed.</div>;
|
if (preview.status === "failed" && !preview.node_preview) {
|
||||||
|
return <div className="dataflow-results-empty">Preview failed.</div>;
|
||||||
|
}
|
||||||
|
const result = preview.node_preview ?? preview;
|
||||||
return (
|
return (
|
||||||
<div className="dataflow-preview-table-wrap">
|
<div className="dataflow-preview-table-wrap">
|
||||||
<table className="dataflow-preview-table">
|
<table className="dataflow-preview-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
{preview.columns.map((column) => (
|
{result.columns.map((column) => (
|
||||||
<th key={column.name}>
|
<th key={column.name}>
|
||||||
<span>{column.name}</span>
|
<span>{column.name}</span>
|
||||||
<small>{column.type}{column.nullable ? " · nullable" : ""}</small>
|
<small>{column.type}{column.nullable ? " · nullable" : ""}</small>
|
||||||
@@ -1897,9 +1950,9 @@ function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{preview.rows.map((row, rowIndex) => (
|
{result.rows.map((row, rowIndex) => (
|
||||||
<tr key={rowIndex}>
|
<tr key={rowIndex}>
|
||||||
{preview.columns.map((column) => (
|
{result.columns.map((column) => (
|
||||||
<td key={column.name}>{formatCell(row[column.name])}</td>
|
<td key={column.name}>{formatCell(row[column.name])}</td>
|
||||||
))}
|
))}
|
||||||
</tr>
|
</tr>
|
||||||
@@ -1908,8 +1961,9 @@ function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
|||||||
</table>
|
</table>
|
||||||
<div className="dataflow-preview-summary">
|
<div className="dataflow-preview-summary">
|
||||||
<span>
|
<span>
|
||||||
Showing {preview.rows.length} of {preview.total_rows} output rows
|
Showing {result.rows.length} of {result.total_rows} rows
|
||||||
</span>
|
</span>
|
||||||
|
{preview.status === "failed" ? <span>Pipeline failed after this stage</span> : null}
|
||||||
<span>{preview.input_row_count} input rows</span>
|
<span>{preview.input_row_count} input rows</span>
|
||||||
<span>
|
<span>
|
||||||
{preview.source_fingerprints.length} source
|
{preview.source_fingerprints.length} source
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
import { useEffect, useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
import { Trash2 } from "lucide-react";
|
import { Play, Trash2 } from "lucide-react";
|
||||||
import { Button, DismissibleAlert, FormField } from "@govoplan/core-webui";
|
import {
|
||||||
|
DismissibleAlert,
|
||||||
|
FormField,
|
||||||
|
IconButton
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
import type {
|
import type {
|
||||||
NodeTypeDefinition,
|
NodeTypeDefinition,
|
||||||
PipelineGraphNode,
|
PipelineGraphNode,
|
||||||
@@ -13,8 +17,11 @@ type NodeInspectorProps = {
|
|||||||
sources: TabularSource[];
|
sources: TabularSource[];
|
||||||
sourceCatalogueAvailable: boolean;
|
sourceCatalogueAvailable: boolean;
|
||||||
readOnly: boolean;
|
readOnly: boolean;
|
||||||
|
canPreview: boolean;
|
||||||
|
previewing: boolean;
|
||||||
onChange: (node: PipelineGraphNode) => void;
|
onChange: (node: PipelineGraphNode) => void;
|
||||||
onDelete: (nodeId: string) => void;
|
onDelete: (nodeId: string) => void;
|
||||||
|
onPreview: (nodeId: string) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export default function NodeInspector({
|
export default function NodeInspector({
|
||||||
@@ -23,8 +30,11 @@ export default function NodeInspector({
|
|||||||
sources,
|
sources,
|
||||||
sourceCatalogueAvailable,
|
sourceCatalogueAvailable,
|
||||||
readOnly,
|
readOnly,
|
||||||
|
canPreview,
|
||||||
|
previewing,
|
||||||
onChange,
|
onChange,
|
||||||
onDelete
|
onDelete,
|
||||||
|
onPreview
|
||||||
}: NodeInspectorProps) {
|
}: NodeInspectorProps) {
|
||||||
const [rowsText, setRowsText] = useState("");
|
const [rowsText, setRowsText] = useState("");
|
||||||
const [aggregateText, setAggregateText] = useState("");
|
const [aggregateText, setAggregateText] = useState("");
|
||||||
@@ -94,16 +104,22 @@ export default function NodeInspector({
|
|||||||
<strong>Inspector</strong>
|
<strong>Inspector</strong>
|
||||||
<small>{definition?.label ?? node.type}</small>
|
<small>{definition?.label ?? node.type}</small>
|
||||||
</span>
|
</span>
|
||||||
<Button
|
<span className="dataflow-inspector-actions">
|
||||||
|
<IconButton
|
||||||
|
label="Preview this node"
|
||||||
|
icon={<Play size={16} />}
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
className="dataflow-inspector-delete"
|
onClick={() => onPreview(node.id)}
|
||||||
|
disabled={!canPreview || previewing}
|
||||||
|
/>
|
||||||
|
<IconButton
|
||||||
|
label="Delete node"
|
||||||
|
icon={<Trash2 size={16} />}
|
||||||
|
variant="danger"
|
||||||
onClick={() => onDelete(node.id)}
|
onClick={() => onDelete(node.id)}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
aria-label="Delete node"
|
/>
|
||||||
title="Delete node"
|
</span>
|
||||||
>
|
|
||||||
<Trash2 size={16} />
|
|
||||||
</Button>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="dataflow-inspector-fields">
|
<div className="dataflow-inspector-fields">
|
||||||
{localError ? (
|
{localError ? (
|
||||||
|
|||||||
119
webui/src/features/dataflow/graphInteractions.ts
Normal file
119
webui/src/features/dataflow/graphInteractions.ts
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
import { definitionConnectionError } from "@govoplan/core-webui/definition-graph";
|
||||||
|
import type { Connection, Edge } from "@xyflow/react";
|
||||||
|
import type {
|
||||||
|
NodeTypeDefinition,
|
||||||
|
PipelineGraph
|
||||||
|
} from "../../api/dataflow";
|
||||||
|
import type { DataflowFlowNode } from "./DataflowNode";
|
||||||
|
|
||||||
|
const PROXIMITY_GAP = 72;
|
||||||
|
const PROXIMITY_EDGE_ID = "dataflow-proximity-preview";
|
||||||
|
|
||||||
|
export function graphEdgesFromFlow(edges: Edge[]): PipelineGraph["edges"] {
|
||||||
|
return edges
|
||||||
|
.filter((edge) => edge.id !== PROXIMITY_EDGE_ID)
|
||||||
|
.map((edge) => ({
|
||||||
|
id: edge.id,
|
||||||
|
source: edge.source,
|
||||||
|
target: edge.target,
|
||||||
|
source_port: edge.sourceHandle ?? "output",
|
||||||
|
target_port: edge.targetHandle ?? "input"
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function proximityPreviewEdge(connection: Connection): Edge {
|
||||||
|
return {
|
||||||
|
...connection,
|
||||||
|
id: PROXIMITY_EDGE_ID,
|
||||||
|
type: "smoothstep",
|
||||||
|
className: "dataflow-edge dataflow-edge-proximity",
|
||||||
|
animated: true,
|
||||||
|
selectable: false,
|
||||||
|
reconnectable: false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestProximityConnection(
|
||||||
|
draggedNode: DataflowFlowNode,
|
||||||
|
flowNodes: DataflowFlowNode[],
|
||||||
|
graph: PipelineGraph,
|
||||||
|
nodeLibrary: NodeTypeDefinition[]
|
||||||
|
): Connection | null {
|
||||||
|
const currentNodes = flowNodes.map((node) => (
|
||||||
|
node.id === draggedNode.id ? draggedNode : node
|
||||||
|
));
|
||||||
|
const neighbors = currentNodes
|
||||||
|
.filter((node) => node.id !== draggedNode.id)
|
||||||
|
.map((node) => ({
|
||||||
|
node,
|
||||||
|
distance: nodeGap(draggedNode, node)
|
||||||
|
}))
|
||||||
|
.filter((candidate) => candidate.distance <= PROXIMITY_GAP)
|
||||||
|
.sort((left, right) => left.distance - right.distance);
|
||||||
|
|
||||||
|
for (const { node: neighbor } of neighbors) {
|
||||||
|
const draggedCenter = nodeCenter(draggedNode);
|
||||||
|
const neighborCenter = nodeCenter(neighbor);
|
||||||
|
const orientations: Array<[DataflowFlowNode, DataflowFlowNode]> = (
|
||||||
|
draggedCenter.x <= neighborCenter.x
|
||||||
|
? [[draggedNode, neighbor], [neighbor, draggedNode]]
|
||||||
|
: [[neighbor, draggedNode], [draggedNode, neighbor]]
|
||||||
|
);
|
||||||
|
for (const [source, target] of orientations) {
|
||||||
|
for (const sourcePort of source.data.definition.output_ports) {
|
||||||
|
for (const targetPort of target.data.definition.input_ports) {
|
||||||
|
const connection: Connection = {
|
||||||
|
source: source.id,
|
||||||
|
target: target.id,
|
||||||
|
sourceHandle: sourcePort.id,
|
||||||
|
targetHandle: targetPort.id
|
||||||
|
};
|
||||||
|
if (definitionConnectionError(
|
||||||
|
graph,
|
||||||
|
nodeLibrary,
|
||||||
|
{
|
||||||
|
source: connection.source,
|
||||||
|
target: connection.target,
|
||||||
|
sourcePort: connection.sourceHandle,
|
||||||
|
targetPort: connection.targetHandle
|
||||||
|
}
|
||||||
|
) === null) {
|
||||||
|
return connection;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeGap(left: DataflowFlowNode, right: DataflowFlowNode): number {
|
||||||
|
const leftCenter = nodeCenter(left);
|
||||||
|
const rightCenter = nodeCenter(right);
|
||||||
|
const horizontal = Math.max(
|
||||||
|
0,
|
||||||
|
Math.abs(leftCenter.x - rightCenter.x)
|
||||||
|
- (nodeWidth(left) + nodeWidth(right)) / 2
|
||||||
|
);
|
||||||
|
const vertical = Math.max(
|
||||||
|
0,
|
||||||
|
Math.abs(leftCenter.y - rightCenter.y)
|
||||||
|
- (nodeHeight(left) + nodeHeight(right)) / 2
|
||||||
|
);
|
||||||
|
return Math.hypot(horizontal, vertical);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeCenter(node: DataflowFlowNode): { x: number; y: number } {
|
||||||
|
return {
|
||||||
|
x: node.position.x + nodeWidth(node) / 2,
|
||||||
|
y: node.position.y + nodeHeight(node) / 2
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeWidth(node: DataflowFlowNode): number {
|
||||||
|
return node.measured?.width ?? node.initialWidth ?? 180;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeHeight(node: DataflowFlowNode): number {
|
||||||
|
return node.measured?.height ?? node.initialHeight ?? 54;
|
||||||
|
}
|
||||||
@@ -537,6 +537,12 @@
|
|||||||
stroke-width: 3;
|
stroke-width: 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dataflow-canvas .react-flow__edge.dataflow-edge-proximity .react-flow__edge-path {
|
||||||
|
stroke: var(--accent);
|
||||||
|
stroke-width: 3;
|
||||||
|
stroke-dasharray: 7 6;
|
||||||
|
}
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__connection-path {
|
.dataflow-canvas .react-flow__connection-path {
|
||||||
stroke: var(--accent);
|
stroke: var(--accent);
|
||||||
stroke-width: 3;
|
stroke-width: 3;
|
||||||
@@ -898,12 +904,30 @@
|
|||||||
.dataflow-results-toolbar {
|
.dataflow-results-toolbar {
|
||||||
min-height: 42px;
|
min-height: 42px;
|
||||||
padding: 5px 9px;
|
padding: 5px 9px;
|
||||||
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-results-toolbar .btn {
|
.dataflow-results-toolbar .btn {
|
||||||
min-height: 30px;
|
min-height: 30px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dataflow-results-view-controls,
|
||||||
|
.dataflow-inspector-actions {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 7px;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-results-view-controls {
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-results-view-controls select {
|
||||||
|
width: clamp(150px, 22vw, 280px);
|
||||||
|
min-height: 30px;
|
||||||
|
}
|
||||||
|
|
||||||
.dataflow-preview-table-wrap,
|
.dataflow-preview-table-wrap,
|
||||||
.dataflow-diagnostics-list {
|
.dataflow-diagnostics-list {
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
|
|||||||
Reference in New Issue
Block a user