Initialize governed Dataflow module
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
import { useMemo, useState, type DragEvent } from "react";
|
||||
import {
|
||||
addEdge,
|
||||
applyEdgeChanges,
|
||||
applyNodeChanges,
|
||||
Background,
|
||||
BackgroundVariant,
|
||||
ConnectionLineType,
|
||||
Controls,
|
||||
MiniMap,
|
||||
ReactFlow,
|
||||
type Connection,
|
||||
type Edge,
|
||||
type ReactFlowInstance
|
||||
} from "@xyflow/react";
|
||||
import type {
|
||||
DataflowDiagnostic,
|
||||
NodePreviewDiagnostic,
|
||||
PipelineGraph,
|
||||
PipelineGraphNode
|
||||
} from "../../api/dataflow";
|
||||
import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
|
||||
import { newNode } from "./model";
|
||||
|
||||
const nodeTypes = { dataflow: DataflowNode };
|
||||
|
||||
type DataflowCanvasProps = {
|
||||
graph: PipelineGraph;
|
||||
diagnostics: DataflowDiagnostic[];
|
||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||
selectedNodeId: string | null;
|
||||
readOnly: boolean;
|
||||
onGraphChange: (graph: PipelineGraph) => void;
|
||||
onSelectNode: (nodeId: string | null) => void;
|
||||
};
|
||||
|
||||
export default function DataflowCanvas({
|
||||
graph,
|
||||
diagnostics,
|
||||
nodeDiagnostics,
|
||||
selectedNodeId,
|
||||
readOnly,
|
||||
onGraphChange,
|
||||
onSelectNode
|
||||
}: DataflowCanvasProps) {
|
||||
const [instance, setInstance] = useState<ReactFlowInstance<DataflowFlowNode, Edge> | null>(null);
|
||||
const errorNodeIds = useMemo(
|
||||
() => new Set(diagnostics.filter((item) => item.severity === "error" && item.node_id).map((item) => item.node_id)),
|
||||
[diagnostics]
|
||||
);
|
||||
const rowCounts = useMemo(
|
||||
() => new Map(nodeDiagnostics.map((item) => [item.node_id, item.output_rows])),
|
||||
[nodeDiagnostics]
|
||||
);
|
||||
const nodes = useMemo<DataflowFlowNode[]>(
|
||||
() => graph.nodes.map((node) => ({
|
||||
id: node.id,
|
||||
type: "dataflow",
|
||||
position: node.position,
|
||||
selected: node.id === selectedNodeId,
|
||||
data: {
|
||||
label: node.label,
|
||||
transformType: node.type,
|
||||
config: node.config,
|
||||
hasError: errorNodeIds.has(node.id),
|
||||
outputRows: rowCounts.get(node.id)
|
||||
}
|
||||
})),
|
||||
[errorNodeIds, graph.nodes, rowCounts, selectedNodeId]
|
||||
);
|
||||
const edges = useMemo<Edge[]>(
|
||||
() => graph.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
type: "smoothstep",
|
||||
className: "dataflow-edge"
|
||||
})),
|
||||
[graph.edges]
|
||||
);
|
||||
|
||||
const updateNodes = (nextNodes: DataflowFlowNode[]) => {
|
||||
const ids = new Set(nextNodes.map((node) => node.id));
|
||||
const nextGraphNodes = nextNodes.map((flowNode) => {
|
||||
const current = graph.nodes.find((node) => node.id === flowNode.id);
|
||||
if (!current) {
|
||||
throw new Error(`Unknown Dataflow node: ${flowNode.id}`);
|
||||
}
|
||||
return { ...current, position: flowNode.position };
|
||||
});
|
||||
onGraphChange({
|
||||
...graph,
|
||||
nodes: nextGraphNodes,
|
||||
edges: graph.edges.filter((edge) => ids.has(edge.source) && ids.has(edge.target))
|
||||
});
|
||||
};
|
||||
|
||||
const updateEdges = (nextEdges: Edge[]) => {
|
||||
onGraphChange({
|
||||
...graph,
|
||||
edges: nextEdges.map((edge) => ({
|
||||
id: edge.id,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
source_port: edge.sourceHandle ?? "output",
|
||||
target_port: edge.targetHandle ?? "input"
|
||||
}))
|
||||
});
|
||||
};
|
||||
|
||||
const isValidConnection = (connection: Connection | Edge): boolean => {
|
||||
if (readOnly || !connection.source || !connection.target || connection.source === connection.target) return false;
|
||||
const source = graph.nodes.find((node) => node.id === connection.source);
|
||||
const target = graph.nodes.find((node) => node.id === connection.target);
|
||||
if (!source || !target || source.type === "output" || target.type.startsWith("source.")) return false;
|
||||
const targetAlreadyConnected = graph.edges.some(
|
||||
(edge) => edge.target === connection.target && edge.source !== connection.source
|
||||
);
|
||||
return !targetAlreadyConnected;
|
||||
};
|
||||
|
||||
const onConnect = (connection: Connection) => {
|
||||
if (!isValidConnection(connection)) return;
|
||||
const next = addEdge(
|
||||
{
|
||||
...connection,
|
||||
id: `edge-${connection.source}-${connection.target}-${crypto.randomUUID()}`,
|
||||
type: "smoothstep"
|
||||
},
|
||||
edges
|
||||
);
|
||||
updateEdges(next);
|
||||
};
|
||||
|
||||
const onDrop = (event: DragEvent<HTMLDivElement>) => {
|
||||
event.preventDefault();
|
||||
if (readOnly || !instance) return;
|
||||
const type = event.dataTransfer.getData("application/x-govoplan-dataflow-node");
|
||||
if (!type) return;
|
||||
const position = instance.screenToFlowPosition({ x: event.clientX, y: event.clientY });
|
||||
const node = newNode(type, position);
|
||||
onGraphChange({ ...graph, nodes: [...graph.nodes, node] });
|
||||
onSelectNode(node.id);
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
className="dataflow-canvas"
|
||||
onDragOver={(event) => {
|
||||
if (event.dataTransfer.types.includes("application/x-govoplan-dataflow-node")) {
|
||||
event.preventDefault();
|
||||
event.dataTransfer.dropEffect = "copy";
|
||||
}
|
||||
}}
|
||||
onDrop={onDrop}
|
||||
>
|
||||
<ReactFlow<DataflowFlowNode, Edge>
|
||||
nodes={nodes}
|
||||
edges={edges}
|
||||
nodeTypes={nodeTypes}
|
||||
onInit={setInstance}
|
||||
onNodesChange={(changes) => {
|
||||
if (readOnly) return;
|
||||
updateNodes(applyNodeChanges(changes, nodes));
|
||||
}}
|
||||
onEdgesChange={(changes) => {
|
||||
if (readOnly) return;
|
||||
updateEdges(applyEdgeChanges(changes, edges));
|
||||
}}
|
||||
onConnect={onConnect}
|
||||
isValidConnection={isValidConnection}
|
||||
onNodeClick={(_event, node) => onSelectNode(node.id)}
|
||||
onPaneClick={() => onSelectNode(null)}
|
||||
nodesDraggable={!readOnly}
|
||||
nodesConnectable={!readOnly}
|
||||
elementsSelectable
|
||||
deleteKeyCode={readOnly ? null : ["Backspace", "Delete"]}
|
||||
connectionLineType={ConnectionLineType.SmoothStep}
|
||||
connectionLineStyle={{ stroke: "var(--accent)", strokeWidth: 3 }}
|
||||
connectionRadius={32}
|
||||
fitView
|
||||
fitViewOptions={{ padding: 0.22, maxZoom: 1.25 }}
|
||||
minZoom={0.25}
|
||||
maxZoom={1.8}
|
||||
>
|
||||
<Background variant={BackgroundVariant.Dots} gap={20} size={1.3} />
|
||||
<MiniMap
|
||||
pannable
|
||||
zoomable
|
||||
nodeStrokeWidth={2}
|
||||
nodeColor={(node) => node.data.hasError ? "var(--danger)" : "var(--accent)"}
|
||||
/>
|
||||
<Controls showInteractive={false} />
|
||||
</ReactFlow>
|
||||
{!graph.nodes.length ? (
|
||||
<div className="dataflow-canvas-empty">Drop a source here</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function updateGraphNode(
|
||||
graph: PipelineGraph,
|
||||
updatedNode: PipelineGraphNode
|
||||
): PipelineGraph {
|
||||
return {
|
||||
...graph,
|
||||
nodes: graph.nodes.map((node) => node.id === updatedNode.id ? updatedNode : node)
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import type { ComponentType } from "react";
|
||||
import {
|
||||
Braces,
|
||||
Columns3,
|
||||
Database,
|
||||
Filter,
|
||||
ListEnd,
|
||||
Sigma,
|
||||
SortAsc,
|
||||
Unplug
|
||||
} from "lucide-react";
|
||||
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
|
||||
|
||||
export type DataflowNodeData = {
|
||||
label: string;
|
||||
transformType: string;
|
||||
config: Record<string, unknown>;
|
||||
hasError?: boolean;
|
||||
outputRows?: number;
|
||||
};
|
||||
|
||||
export type DataflowFlowNode = Node<DataflowNodeData, "dataflow">;
|
||||
|
||||
const iconByType: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
|
||||
"source.inline": Braces,
|
||||
"source.reference": Database,
|
||||
filter: Filter,
|
||||
select: Columns3,
|
||||
aggregate: Sigma,
|
||||
sort: SortAsc,
|
||||
limit: ListEnd,
|
||||
output: Unplug
|
||||
};
|
||||
|
||||
export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowNode>) {
|
||||
const Icon = iconByType[data.transformType] ?? Braces;
|
||||
const isSource = data.transformType.startsWith("source.");
|
||||
const isOutput = data.transformType === "output";
|
||||
|
||||
return (
|
||||
<div
|
||||
className={[
|
||||
"dataflow-node",
|
||||
`dataflow-node-${data.transformType.replace(".", "-")}`,
|
||||
selected ? "is-selected" : "",
|
||||
data.hasError ? "has-error" : ""
|
||||
].filter(Boolean).join(" ")}
|
||||
>
|
||||
{!isSource ? (
|
||||
<Handle
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
className="dataflow-node-handle dataflow-node-handle-input"
|
||||
/>
|
||||
) : null}
|
||||
<span className="dataflow-node-icon" aria-hidden="true">
|
||||
<Icon size={17} strokeWidth={1.8} />
|
||||
</span>
|
||||
<span className="dataflow-node-copy">
|
||||
<strong>{data.label}</strong>
|
||||
<small>{transformLabel(data.transformType)}</small>
|
||||
</span>
|
||||
{typeof data.outputRows === "number" ? (
|
||||
<span className="dataflow-node-count">{data.outputRows}</span>
|
||||
) : null}
|
||||
{!isOutput ? (
|
||||
<Handle
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
className="dataflow-node-handle dataflow-node-handle-output"
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function transformLabel(type: string): string {
|
||||
if (type === "source.inline") return "Inline source";
|
||||
if (type === "source.reference") return "Connector source";
|
||||
if (type === "filter") return "Filter";
|
||||
if (type === "select") return "Projection";
|
||||
if (type === "aggregate") return "Aggregation";
|
||||
if (type === "sort") return "Sort";
|
||||
if (type === "limit") return "Limit";
|
||||
if (type === "output") return "Output";
|
||||
return type;
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
type ComponentType,
|
||||
type DragEvent
|
||||
} from "react";
|
||||
import {
|
||||
Braces,
|
||||
CheckCircle2,
|
||||
Code2,
|
||||
Columns3,
|
||||
Database,
|
||||
Filter,
|
||||
ListEnd,
|
||||
Network,
|
||||
Play,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
RotateCcw,
|
||||
Save,
|
||||
Sigma,
|
||||
SortAsc,
|
||||
Trash2,
|
||||
TriangleAlert
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Button,
|
||||
ConfirmDialog,
|
||||
DismissibleAlert,
|
||||
IconButton,
|
||||
LoadingFrame,
|
||||
SegmentedControl,
|
||||
StatusBadge,
|
||||
hasScope,
|
||||
isApiError,
|
||||
useUnsavedChanges,
|
||||
useUnsavedDraftGuard,
|
||||
type ApiSettings,
|
||||
type AuthInfo
|
||||
} from "@govoplan/core-webui";
|
||||
import { ReactFlowProvider } from "@xyflow/react";
|
||||
import {
|
||||
compileDataflowSql,
|
||||
createDataflowPipeline,
|
||||
deleteDataflowPipeline,
|
||||
listDataflowPipelines,
|
||||
previewDataflowPipeline,
|
||||
renderDataflowSql,
|
||||
updateDataflowPipeline,
|
||||
validateDataflowPipeline,
|
||||
type DataflowDiagnostic,
|
||||
type EditorMode,
|
||||
type NodePreviewDiagnostic,
|
||||
type Pipeline,
|
||||
type PipelineGraphNode,
|
||||
type PipelinePreview
|
||||
} from "../../api/dataflow";
|
||||
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
||||
import NodeInspector from "./NodeInspector";
|
||||
import {
|
||||
NODE_LABELS,
|
||||
PALETTE_NODE_TYPES,
|
||||
draftFingerprint,
|
||||
draftFromPipeline,
|
||||
newNode,
|
||||
pipelinePayload,
|
||||
sampleDraft,
|
||||
sourceNodes,
|
||||
type PipelineDraft
|
||||
} from "./model";
|
||||
|
||||
type ResultTab = "preview" | "diagnostics";
|
||||
|
||||
const paletteIcons: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
|
||||
"source.inline": Braces,
|
||||
filter: Filter,
|
||||
select: Columns3,
|
||||
aggregate: Sigma,
|
||||
sort: SortAsc,
|
||||
limit: ListEnd,
|
||||
output: Network
|
||||
};
|
||||
|
||||
export default function DataflowPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo }) {
|
||||
const { requestNavigation, requestDiscard } = useUnsavedChanges();
|
||||
const [pipelines, setPipelines] = useState<Pipeline[]>([]);
|
||||
const [draft, setDraft] = useState<PipelineDraft | null>(null);
|
||||
const [savedDraft, setSavedDraft] = useState<PipelineDraft | null>(null);
|
||||
const [selectedNodeId, setSelectedNodeId] = useState<string | null>(null);
|
||||
const [search, setSearch] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [working, setWorking] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
const [success, setSuccess] = useState("");
|
||||
const [diagnostics, setDiagnostics] = useState<DataflowDiagnostic[]>([]);
|
||||
const [nodeDiagnostics, setNodeDiagnostics] = useState<NodePreviewDiagnostic[]>([]);
|
||||
const [preview, setPreview] = useState<PipelinePreview | null>(null);
|
||||
const [resultOpen, setResultOpen] = useState(false);
|
||||
const [resultTab, setResultTab] = useState<ResultTab>("preview");
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
|
||||
const canWrite = hasScope(auth, "dataflow:pipeline:write") || hasScope(auth, "dataflow:pipeline:admin");
|
||||
const canRun = hasScope(auth, "dataflow:pipeline:run") || hasScope(auth, "dataflow:pipeline:admin");
|
||||
const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft);
|
||||
const selectedNode = useMemo(
|
||||
() => draft?.graph.nodes.find((node) => node.id === selectedNodeId) ?? null,
|
||||
[draft, selectedNodeId]
|
||||
);
|
||||
const filteredPipelines = useMemo(() => {
|
||||
const query = search.trim().toLocaleLowerCase();
|
||||
if (!query) return pipelines;
|
||||
return pipelines.filter((pipeline) =>
|
||||
`${pipeline.name} ${pipeline.description ?? ""} ${pipeline.status}`.toLocaleLowerCase().includes(query)
|
||||
);
|
||||
}, [pipelines, search]);
|
||||
|
||||
const loadPipelines = useCallback(async (preferredId?: string | null) => {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
const items = await listDataflowPipelines(settings);
|
||||
setPipelines(items);
|
||||
const selected = items.find((item) => item.id === (preferredId ?? draft?.id)) ?? items[0];
|
||||
if (selected) {
|
||||
const next = draftFromPipeline(selected);
|
||||
setDraft(next);
|
||||
setSavedDraft(structuredClone(next));
|
||||
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
|
||||
} else if (!draft?.id) {
|
||||
setDraft(null);
|
||||
setSavedDraft(null);
|
||||
setSelectedNodeId(null);
|
||||
}
|
||||
} catch (loadError) {
|
||||
setError(apiErrorMessage(loadError));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [draft?.id, settings]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadPipelines();
|
||||
}, []);
|
||||
|
||||
const discardDraft = useCallback(() => {
|
||||
if (savedDraft) {
|
||||
const reset = structuredClone(savedDraft);
|
||||
setDraft(reset);
|
||||
setSelectedNodeId(reset.graph.nodes[0]?.id ?? null);
|
||||
} else {
|
||||
setDraft(null);
|
||||
setSelectedNodeId(null);
|
||||
}
|
||||
setDiagnostics([]);
|
||||
setNodeDiagnostics([]);
|
||||
setPreview(null);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
}, [savedDraft]);
|
||||
|
||||
const saveDraft = useCallback(async (): Promise<boolean> => {
|
||||
if (!draft || !canWrite || !draft.name.trim()) {
|
||||
setError(!draft?.name.trim() ? "Pipeline name is required." : "You cannot save this pipeline.");
|
||||
return false;
|
||||
}
|
||||
setSaving(true);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
try {
|
||||
const payload = pipelinePayload(draft);
|
||||
const saved = draft.id && draft.currentRevision
|
||||
? await updateDataflowPipeline(settings, draft.id, {
|
||||
...payload,
|
||||
expected_revision: draft.currentRevision
|
||||
})
|
||||
: await createDataflowPipeline(settings, payload);
|
||||
const next = draftFromPipeline(saved);
|
||||
setDraft(next);
|
||||
setSavedDraft(structuredClone(next));
|
||||
setPipelines((current) => [saved, ...current.filter((item) => item.id !== saved.id)]);
|
||||
setSelectedNodeId((current) => current && next.graph.nodes.some((node) => node.id === current)
|
||||
? current
|
||||
: next.graph.nodes[0]?.id ?? null);
|
||||
setDiagnostics([]);
|
||||
setSuccess(`Saved revision ${saved.current_revision}.`);
|
||||
return true;
|
||||
} catch (saveError) {
|
||||
setError(apiErrorMessage(saveError));
|
||||
return false;
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
}, [canWrite, draft, settings]);
|
||||
|
||||
useUnsavedDraftGuard({
|
||||
dirty,
|
||||
title: "Unsaved pipeline",
|
||||
message: "Save or discard the pipeline changes before leaving this workspace.",
|
||||
onSave: saveDraft,
|
||||
onDiscard: discardDraft
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const beforeUnload = (event: BeforeUnloadEvent) => {
|
||||
if (!dirty) return;
|
||||
event.preventDefault();
|
||||
event.returnValue = "";
|
||||
};
|
||||
window.addEventListener("beforeunload", beforeUnload);
|
||||
return () => window.removeEventListener("beforeunload", beforeUnload);
|
||||
}, [dirty]);
|
||||
|
||||
const selectPipeline = (pipeline: Pipeline) => {
|
||||
requestNavigation(() => {
|
||||
const next = draftFromPipeline(pipeline);
|
||||
setDraft(next);
|
||||
setSavedDraft(structuredClone(next));
|
||||
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
|
||||
setDiagnostics([]);
|
||||
setNodeDiagnostics([]);
|
||||
setPreview(null);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
});
|
||||
};
|
||||
|
||||
const createNew = () => {
|
||||
requestNavigation(() => {
|
||||
const next = sampleDraft();
|
||||
setDraft(next);
|
||||
setSavedDraft(null);
|
||||
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
|
||||
setDiagnostics([]);
|
||||
setNodeDiagnostics([]);
|
||||
setPreview(null);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
});
|
||||
};
|
||||
|
||||
const updateDraft = (patch: Partial<PipelineDraft>) => {
|
||||
setDraft((current) => current ? { ...current, ...patch } : current);
|
||||
setSuccess("");
|
||||
};
|
||||
|
||||
const updateGraph = (graph: PipelineDraft["graph"]) => {
|
||||
updateDraft({ graph });
|
||||
setDiagnostics([]);
|
||||
setNodeDiagnostics([]);
|
||||
};
|
||||
|
||||
const validate = async () => {
|
||||
if (!draft) return;
|
||||
setWorking(true);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
try {
|
||||
const response = await validateDataflowPipeline(settings, draft.editorMode === "sql"
|
||||
? { graph: draft.graph, sql_text: draft.sqlText, source_nodes: sourceNodes(draft.graph) }
|
||||
: { graph: draft.graph });
|
||||
setDiagnostics(response.diagnostics);
|
||||
if (response.valid) setSuccess("Pipeline definition is valid.");
|
||||
setResultOpen(true);
|
||||
setResultTab("diagnostics");
|
||||
} catch (validationError) {
|
||||
setError(apiErrorMessage(validationError));
|
||||
} finally {
|
||||
setWorking(false);
|
||||
}
|
||||
};
|
||||
|
||||
const applySql = async (switchToGraph = false): Promise<boolean> => {
|
||||
if (!draft) return false;
|
||||
setWorking(true);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
try {
|
||||
const response = await compileDataflowSql(settings, {
|
||||
graph: draft.graph,
|
||||
sql_text: draft.sqlText,
|
||||
source_nodes: sourceNodes(draft.graph)
|
||||
});
|
||||
setDiagnostics(response.diagnostics);
|
||||
setResultOpen(true);
|
||||
setResultTab("diagnostics");
|
||||
if (!response.valid || !response.graph) return false;
|
||||
updateDraft({
|
||||
graph: response.graph,
|
||||
sqlText: response.sql_text ?? draft.sqlText,
|
||||
editorMode: switchToGraph ? "graph" : "sql"
|
||||
});
|
||||
setSelectedNodeId(response.graph.nodes[0]?.id ?? null);
|
||||
setSuccess("SQL compiled into the pipeline graph.");
|
||||
return true;
|
||||
} catch (compileError) {
|
||||
setError(apiErrorMessage(compileError));
|
||||
return false;
|
||||
} finally {
|
||||
setWorking(false);
|
||||
}
|
||||
};
|
||||
|
||||
const changeEditorMode = async (mode: EditorMode) => {
|
||||
if (!draft || mode === draft.editorMode) return;
|
||||
if (mode === "sql") {
|
||||
setWorking(true);
|
||||
setError("");
|
||||
try {
|
||||
const response = await renderDataflowSql(settings, draft.graph);
|
||||
setDiagnostics(response.diagnostics);
|
||||
if (!response.valid || !response.sql_text) {
|
||||
setResultOpen(true);
|
||||
setResultTab("diagnostics");
|
||||
return;
|
||||
}
|
||||
updateDraft({ editorMode: "sql", sqlText: response.sql_text });
|
||||
} catch (renderError) {
|
||||
setError(apiErrorMessage(renderError));
|
||||
} finally {
|
||||
setWorking(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
await applySql(true);
|
||||
};
|
||||
|
||||
const runPreview = async () => {
|
||||
if (!draft || !canRun) return;
|
||||
setWorking(true);
|
||||
setError("");
|
||||
setSuccess("");
|
||||
try {
|
||||
const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision
|
||||
? { pipeline_id: draft.id, revision: draft.currentRevision, row_limit: 100 }
|
||||
: {
|
||||
graph: draft.graph,
|
||||
sql_text: draft.editorMode === "sql" ? draft.sqlText : undefined,
|
||||
source_nodes: sourceNodes(draft.graph),
|
||||
row_limit: 100
|
||||
});
|
||||
setPreview(response);
|
||||
setDiagnostics(response.diagnostics);
|
||||
setNodeDiagnostics(response.node_diagnostics);
|
||||
setResultOpen(true);
|
||||
setResultTab(response.status === "succeeded" ? "preview" : "diagnostics");
|
||||
if (response.status === "succeeded") {
|
||||
setSuccess(`Preview produced ${response.total_rows} row${response.total_rows === 1 ? "" : "s"}.`);
|
||||
}
|
||||
} catch (previewError) {
|
||||
setError(apiErrorMessage(previewError));
|
||||
} finally {
|
||||
setWorking(false);
|
||||
}
|
||||
};
|
||||
|
||||
const removePipeline = async () => {
|
||||
if (!draft?.id) return;
|
||||
setSaving(true);
|
||||
setError("");
|
||||
try {
|
||||
await deleteDataflowPipeline(settings, draft.id);
|
||||
const remaining = pipelines.filter((item) => item.id !== draft.id);
|
||||
setPipelines(remaining);
|
||||
const next = remaining[0] ? draftFromPipeline(remaining[0]) : null;
|
||||
setDraft(next);
|
||||
setSavedDraft(next ? structuredClone(next) : null);
|
||||
setSelectedNodeId(next?.graph.nodes[0]?.id ?? null);
|
||||
setDeleteOpen(false);
|
||||
setPreview(null);
|
||||
setDiagnostics([]);
|
||||
} catch (deleteError) {
|
||||
setError(apiErrorMessage(deleteError));
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const addNode = (type: string) => {
|
||||
if (!draft || !canWrite || uniqueNodeExists(draft, type)) return;
|
||||
const node = newNode(type, {
|
||||
x: 120 + draft.graph.nodes.length * 70,
|
||||
y: 100 + (draft.graph.nodes.length % 4) * 80
|
||||
});
|
||||
updateGraph({ ...draft.graph, nodes: [...draft.graph.nodes, node] });
|
||||
setSelectedNodeId(node.id);
|
||||
};
|
||||
|
||||
return (
|
||||
<main className="dataflow-page">
|
||||
<div className="dataflow-shell">
|
||||
<aside className="dataflow-pipeline-panel" aria-label="Data pipelines">
|
||||
<div className="dataflow-panel-toolbar">
|
||||
<strong>Pipelines</strong>
|
||||
<span className="dataflow-toolbar-actions">
|
||||
<IconButton
|
||||
label="Refresh pipelines"
|
||||
icon={<RefreshCw size={16} />}
|
||||
variant="ghost"
|
||||
onClick={() => requestNavigation(() => void loadPipelines(draft?.id))}
|
||||
disabled={loading}
|
||||
/>
|
||||
<IconButton
|
||||
label="New pipeline"
|
||||
icon={<Plus size={17} />}
|
||||
variant="primary"
|
||||
onClick={createNew}
|
||||
disabled={!canWrite}
|
||||
/>
|
||||
</span>
|
||||
</div>
|
||||
<div className="dataflow-pipeline-search">
|
||||
<input
|
||||
type="search"
|
||||
value={search}
|
||||
onChange={(event) => setSearch(event.target.value)}
|
||||
placeholder="Search pipelines"
|
||||
aria-label="Search pipelines"
|
||||
/>
|
||||
</div>
|
||||
<LoadingFrame loading={loading} label="Loading pipelines" className="dataflow-pipeline-list-frame">
|
||||
<div className="dataflow-pipeline-list">
|
||||
{filteredPipelines.map((pipeline) => (
|
||||
<button
|
||||
key={pipeline.id}
|
||||
type="button"
|
||||
className={pipeline.id === draft?.id ? "is-selected" : ""}
|
||||
onClick={() => selectPipeline(pipeline)}
|
||||
>
|
||||
<span>
|
||||
<strong>{pipeline.name}</strong>
|
||||
<small>Revision {pipeline.current_revision}</small>
|
||||
</span>
|
||||
<StatusBadge status={pipeline.status} />
|
||||
</button>
|
||||
))}
|
||||
{!loading && !filteredPipelines.length ? (
|
||||
<div className="dataflow-pipeline-empty">
|
||||
{search ? "No matching pipelines" : "No pipelines yet"}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</LoadingFrame>
|
||||
</aside>
|
||||
|
||||
<section className="dataflow-workspace">
|
||||
{draft ? (
|
||||
<>
|
||||
<div className="dataflow-workspace-toolbar">
|
||||
<div className="dataflow-identity-fields">
|
||||
<input
|
||||
className="dataflow-name-input"
|
||||
value={draft.name}
|
||||
onChange={(event) => updateDraft({ name: event.target.value })}
|
||||
aria-label="Pipeline name"
|
||||
disabled={!canWrite}
|
||||
/>
|
||||
<input
|
||||
className="dataflow-description-input"
|
||||
value={draft.description}
|
||||
onChange={(event) => updateDraft({ description: event.target.value })}
|
||||
placeholder="Description"
|
||||
aria-label="Pipeline description"
|
||||
disabled={!canWrite}
|
||||
/>
|
||||
<select
|
||||
value={draft.status}
|
||||
onChange={(event) => updateDraft({ status: event.target.value as PipelineDraft["status"] })}
|
||||
aria-label="Pipeline status"
|
||||
disabled={!canWrite}
|
||||
>
|
||||
<option value="draft">Draft</option>
|
||||
<option value="active">Active</option>
|
||||
<option value="archived">Archived</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="dataflow-command-bar">
|
||||
<SegmentedControl<EditorMode>
|
||||
ariaLabel="Pipeline editor mode"
|
||||
options={[
|
||||
{ id: "graph", label: <><Network size={15} /> Graph</> },
|
||||
{ id: "sql", label: <><Code2 size={15} /> SQL</> }
|
||||
]}
|
||||
value={draft.editorMode}
|
||||
onChange={(mode) => void changeEditorMode(mode)}
|
||||
disabled={working}
|
||||
/>
|
||||
<Button onClick={() => void validate()} disabled={working}>
|
||||
<CheckCircle2 size={16} /> Validate
|
||||
</Button>
|
||||
<Button variant="primary" onClick={() => void runPreview()} disabled={working || !canRun}>
|
||||
<Play size={16} /> Preview
|
||||
</Button>
|
||||
<IconButton
|
||||
label="Discard changes"
|
||||
icon={<RotateCcw size={16} />}
|
||||
variant="ghost"
|
||||
onClick={() => requestDiscard(() => undefined)}
|
||||
disabled={!dirty || saving}
|
||||
/>
|
||||
{draft.id ? (
|
||||
<IconButton
|
||||
label="Delete pipeline"
|
||||
icon={<Trash2 size={16} />}
|
||||
variant="danger"
|
||||
onClick={() => setDeleteOpen(true)}
|
||||
disabled={!canWrite || saving}
|
||||
/>
|
||||
) : null}
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={() => void saveDraft()}
|
||||
disabled={saving || working || !dirty || !canWrite}
|
||||
>
|
||||
<Save size={16} /> {saving ? "Saving..." : "Save"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
{(error || success) ? (
|
||||
<div className="dataflow-alerts">
|
||||
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null}
|
||||
{success ? <DismissibleAlert tone="success" resetKey={success}>{success}</DismissibleAlert> : null}
|
||||
</div>
|
||||
) : null}
|
||||
<div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}>
|
||||
{draft.editorMode === "graph" ? (
|
||||
<aside className="dataflow-palette" aria-label="Transform palette">
|
||||
<div className="dataflow-panel-heading">
|
||||
<strong>Transforms</strong>
|
||||
</div>
|
||||
<div className="dataflow-palette-items">
|
||||
{PALETTE_NODE_TYPES.map((type) => {
|
||||
const Icon = paletteIcons[type] ?? Database;
|
||||
const disabled = !canWrite || uniqueNodeExists(draft, type);
|
||||
return (
|
||||
<button
|
||||
key={type}
|
||||
type="button"
|
||||
draggable={!disabled}
|
||||
disabled={disabled}
|
||||
onDragStart={(event) => startPaletteDrag(event, type)}
|
||||
onClick={() => addNode(type)}
|
||||
>
|
||||
<Icon size={16} />
|
||||
<span>{NODE_LABELS[type]}</span>
|
||||
<Plus size={14} className="dataflow-palette-add" />
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</aside>
|
||||
) : null}
|
||||
<section className="dataflow-editor-surface">
|
||||
{draft.editorMode === "graph" ? (
|
||||
<ReactFlowProvider>
|
||||
<DataflowCanvas
|
||||
graph={draft.graph}
|
||||
diagnostics={diagnostics}
|
||||
nodeDiagnostics={nodeDiagnostics}
|
||||
selectedNodeId={selectedNodeId}
|
||||
readOnly={!canWrite}
|
||||
onGraphChange={updateGraph}
|
||||
onSelectNode={setSelectedNodeId}
|
||||
/>
|
||||
</ReactFlowProvider>
|
||||
) : (
|
||||
<div className="dataflow-sql-workbench">
|
||||
<div className="dataflow-sql-toolbar">
|
||||
<span>Constrained Dataflow SQL</span>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={() => void applySql(false)}
|
||||
disabled={working || !canWrite}
|
||||
>
|
||||
<Code2 size={16} /> Apply SQL
|
||||
</Button>
|
||||
</div>
|
||||
<textarea
|
||||
value={draft.sqlText}
|
||||
onChange={(event) => updateDraft({ sqlText: event.target.value })}
|
||||
spellCheck={false}
|
||||
disabled={!canWrite}
|
||||
aria-label="Dataflow SQL"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
<NodeInspector
|
||||
node={selectedNode}
|
||||
readOnly={!canWrite}
|
||||
onChange={(node: PipelineGraphNode) => updateGraph(updateGraphNode(draft.graph, node))}
|
||||
onDelete={(nodeId) => {
|
||||
updateGraph({
|
||||
...draft.graph,
|
||||
nodes: draft.graph.nodes.filter((node) => node.id !== nodeId),
|
||||
edges: draft.graph.edges.filter((edge) => edge.source !== nodeId && edge.target !== nodeId)
|
||||
});
|
||||
setSelectedNodeId(null);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{resultOpen ? (
|
||||
<ResultPanel
|
||||
tab={resultTab}
|
||||
onTabChange={setResultTab}
|
||||
preview={preview}
|
||||
diagnostics={diagnostics}
|
||||
nodeDiagnostics={nodeDiagnostics}
|
||||
onClose={() => setResultOpen(false)}
|
||||
/>
|
||||
) : null}
|
||||
</>
|
||||
) : (
|
||||
<div className="dataflow-workspace-empty">
|
||||
<Network size={30} />
|
||||
<strong>No pipeline selected</strong>
|
||||
<Button variant="primary" onClick={createNew} disabled={!canWrite}>
|
||||
<Plus size={16} /> New pipeline
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{working ? <div className="dataflow-working-indicator" role="status">Working...</div> : null}
|
||||
</section>
|
||||
</div>
|
||||
<ConfirmDialog
|
||||
open={deleteOpen}
|
||||
title="Delete pipeline"
|
||||
message={`Delete ${draft?.name ?? "this pipeline"} and all of its saved revisions and run evidence?`}
|
||||
confirmLabel="Delete"
|
||||
tone="danger"
|
||||
busy={saving}
|
||||
onCancel={() => setDeleteOpen(false)}
|
||||
onConfirm={() => void removePipeline()}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
function ResultPanel({
|
||||
tab,
|
||||
onTabChange,
|
||||
preview,
|
||||
diagnostics,
|
||||
nodeDiagnostics,
|
||||
onClose
|
||||
}: {
|
||||
tab: ResultTab;
|
||||
onTabChange: (tab: ResultTab) => void;
|
||||
preview: PipelinePreview | null;
|
||||
diagnostics: DataflowDiagnostic[];
|
||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||
onClose: () => void;
|
||||
}) {
|
||||
return (
|
||||
<section className="dataflow-results" aria-label="Pipeline results">
|
||||
<div className="dataflow-results-toolbar">
|
||||
<SegmentedControl<ResultTab>
|
||||
ariaLabel="Result view"
|
||||
options={[
|
||||
{ id: "preview", label: `Preview${preview ? ` (${preview.total_rows})` : ""}` },
|
||||
{ id: "diagnostics", label: `Diagnostics (${diagnostics.length + nodeDiagnostics.length})` }
|
||||
]}
|
||||
value={tab}
|
||||
onChange={onTabChange}
|
||||
/>
|
||||
<Button variant="ghost" onClick={onClose}>Close</Button>
|
||||
</div>
|
||||
{tab === "preview" ? (
|
||||
<PreviewTable preview={preview} />
|
||||
) : (
|
||||
<DiagnosticsPanel diagnostics={diagnostics} nodeDiagnostics={nodeDiagnostics} />
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
||||
if (!preview) return <div className="dataflow-results-empty">No preview has been run.</div>;
|
||||
if (preview.status === "failed") return <div className="dataflow-results-empty">Preview failed.</div>;
|
||||
return (
|
||||
<div className="dataflow-preview-table-wrap">
|
||||
<table className="dataflow-preview-table">
|
||||
<thead>
|
||||
<tr>
|
||||
{preview.columns.map((column) => (
|
||||
<th key={column.name}>
|
||||
<span>{column.name}</span>
|
||||
<small>{column.type}{column.nullable ? " · nullable" : ""}</small>
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{preview.rows.map((row, rowIndex) => (
|
||||
<tr key={rowIndex}>
|
||||
{preview.columns.map((column) => (
|
||||
<td key={column.name}>{formatCell(row[column.name])}</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{preview.truncated ? (
|
||||
<div className="dataflow-preview-truncated">Showing {preview.rows.length} of {preview.total_rows} rows</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DiagnosticsPanel({
|
||||
diagnostics,
|
||||
nodeDiagnostics
|
||||
}: {
|
||||
diagnostics: DataflowDiagnostic[];
|
||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||
}) {
|
||||
if (!diagnostics.length && !nodeDiagnostics.length) {
|
||||
return <div className="dataflow-results-empty">No diagnostics.</div>;
|
||||
}
|
||||
return (
|
||||
<div className="dataflow-diagnostics-list">
|
||||
{diagnostics.map((item, index) => (
|
||||
<div key={`${item.code}-${item.node_id ?? "pipeline"}-${index}`} className={`is-${item.severity}`}>
|
||||
{item.severity === "error" ? <TriangleAlert size={15} /> : <CheckCircle2 size={15} />}
|
||||
<span>
|
||||
<strong>{item.code}</strong>
|
||||
<small>{item.message}</small>
|
||||
</span>
|
||||
{item.node_id ? <code>{item.node_id}</code> : null}
|
||||
</div>
|
||||
))}
|
||||
{nodeDiagnostics.map((item) => (
|
||||
<div key={item.node_id} className={`is-${item.status}`}>
|
||||
<CheckCircle2 size={15} />
|
||||
<span>
|
||||
<strong>{item.node_id}</strong>
|
||||
<small>{item.input_rows} in · {item.output_rows} out · {item.duration_ms.toFixed(2)} ms</small>
|
||||
</span>
|
||||
<code>{item.columns.length} columns</code>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function uniqueNodeExists(draft: PipelineDraft, type: string): boolean {
|
||||
if (type === "output") return draft.graph.nodes.some((node) => node.type === "output");
|
||||
if (type.startsWith("source.")) return draft.graph.nodes.some((node) => node.type.startsWith("source."));
|
||||
return false;
|
||||
}
|
||||
|
||||
function startPaletteDrag(event: DragEvent<HTMLButtonElement>, type: string) {
|
||||
event.dataTransfer.setData("application/x-govoplan-dataflow-node", type);
|
||||
event.dataTransfer.effectAllowed = "copy";
|
||||
}
|
||||
|
||||
function formatCell(value: unknown): string {
|
||||
if (value === null || value === undefined) return "—";
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function apiErrorMessage(error: unknown): string {
|
||||
if (!isApiError(error)) return error instanceof Error ? error.message : "The request failed.";
|
||||
try {
|
||||
const parsed = JSON.parse(error.body) as { detail?: string | { message?: string } };
|
||||
if (typeof parsed.detail === "string") return parsed.detail;
|
||||
if (parsed.detail?.message) return parsed.detail.message;
|
||||
} catch {
|
||||
// Fall through to the API error message.
|
||||
}
|
||||
return error.message;
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Trash2 } from "lucide-react";
|
||||
import { Button, DismissibleAlert, FormField } from "@govoplan/core-webui";
|
||||
import type { PipelineGraphNode } from "../../api/dataflow";
|
||||
|
||||
type NodeInspectorProps = {
|
||||
node: PipelineGraphNode | null;
|
||||
readOnly: boolean;
|
||||
onChange: (node: PipelineGraphNode) => void;
|
||||
onDelete: (nodeId: string) => void;
|
||||
};
|
||||
|
||||
export default function NodeInspector({ node, readOnly, onChange, onDelete }: NodeInspectorProps) {
|
||||
const [rowsText, setRowsText] = useState("");
|
||||
const [aggregateText, setAggregateText] = useState("");
|
||||
const [sortText, setSortText] = useState("");
|
||||
const [localError, setLocalError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
setRowsText(node ? JSON.stringify(node.config.rows ?? [], null, 2) : "");
|
||||
setAggregateText(node ? aggregatesToText(node.config.aggregates) : "");
|
||||
setSortText(node ? sortFieldsToText(node.config.fields) : "");
|
||||
setLocalError("");
|
||||
}, [node?.id]);
|
||||
|
||||
if (!node) {
|
||||
return (
|
||||
<aside className="dataflow-inspector" aria-label="Node inspector">
|
||||
<div className="dataflow-panel-heading">
|
||||
<strong>Inspector</strong>
|
||||
</div>
|
||||
<div className="dataflow-inspector-empty">No node selected</div>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
const updateConfig = (patch: Record<string, unknown>) => {
|
||||
onChange({ ...node, config: { ...node.config, ...patch } });
|
||||
};
|
||||
|
||||
const commitRows = () => {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(rowsText);
|
||||
if (!Array.isArray(parsed) || parsed.some((item) => !isRecord(item))) {
|
||||
throw new Error("Rows must be a JSON array of objects.");
|
||||
}
|
||||
setLocalError("");
|
||||
updateConfig({ rows: parsed });
|
||||
} catch (error) {
|
||||
setLocalError(error instanceof Error ? error.message : "Rows could not be parsed.");
|
||||
}
|
||||
};
|
||||
|
||||
const commitAggregates = () => {
|
||||
try {
|
||||
const parsed = aggregatesFromText(aggregateText);
|
||||
setLocalError("");
|
||||
updateConfig({ aggregates: parsed });
|
||||
} catch (error) {
|
||||
setLocalError(error instanceof Error ? error.message : "Aggregates could not be parsed.");
|
||||
}
|
||||
};
|
||||
|
||||
const commitSort = () => {
|
||||
try {
|
||||
const parsed = sortFieldsFromText(sortText);
|
||||
setLocalError("");
|
||||
updateConfig({ fields: parsed });
|
||||
} catch (error) {
|
||||
setLocalError(error instanceof Error ? error.message : "Sort fields could not be parsed.");
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<aside className="dataflow-inspector" aria-label="Node inspector">
|
||||
<div className="dataflow-panel-heading">
|
||||
<span>
|
||||
<strong>Inspector</strong>
|
||||
<small>{node.type}</small>
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
className="dataflow-inspector-delete"
|
||||
onClick={() => onDelete(node.id)}
|
||||
disabled={readOnly}
|
||||
aria-label="Delete node"
|
||||
title="Delete node"
|
||||
>
|
||||
<Trash2 size={16} />
|
||||
</Button>
|
||||
</div>
|
||||
<div className="dataflow-inspector-fields">
|
||||
{localError ? (
|
||||
<DismissibleAlert tone="danger" resetKey={localError}>
|
||||
{localError}
|
||||
</DismissibleAlert>
|
||||
) : null}
|
||||
<FormField label="Name">
|
||||
<input
|
||||
value={node.label}
|
||||
onChange={(event) => onChange({ ...node, label: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
{node.type.startsWith("source.") ? (
|
||||
<FormField label="Logical source name">
|
||||
<input
|
||||
value={textValue(node.config.source_name)}
|
||||
onChange={(event) => updateConfig({ source_name: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "source.inline" ? (
|
||||
<FormField label="Rows">
|
||||
<textarea
|
||||
className="dataflow-json-editor"
|
||||
value={rowsText}
|
||||
onChange={(event) => setRowsText(event.target.value)}
|
||||
onBlur={commitRows}
|
||||
spellCheck={false}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "filter" ? (
|
||||
<>
|
||||
<FormField label="Column">
|
||||
<input
|
||||
value={textValue(node.config.column)}
|
||||
onChange={(event) => updateConfig({ column: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Operator">
|
||||
<select
|
||||
value={textValue(node.config.operator) || "eq"}
|
||||
onChange={(event) => updateConfig({ operator: event.target.value })}
|
||||
disabled={readOnly}
|
||||
>
|
||||
<option value="eq">equals</option>
|
||||
<option value="ne">does not equal</option>
|
||||
<option value="gt">greater than</option>
|
||||
<option value="gte">greater than or equal</option>
|
||||
<option value="lt">less than</option>
|
||||
<option value="lte">less than or equal</option>
|
||||
<option value="contains">contains</option>
|
||||
<option value="is_null">is null</option>
|
||||
<option value="not_null">is not null</option>
|
||||
</select>
|
||||
</FormField>
|
||||
{!["is_null", "not_null"].includes(textValue(node.config.operator)) ? (
|
||||
<FormField label="Value">
|
||||
<input
|
||||
value={displayScalar(node.config.value)}
|
||||
onChange={(event) => updateConfig({ value: parseScalar(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "select" ? (
|
||||
<FormField label="Columns">
|
||||
<input
|
||||
value={selectFieldsToText(node.config.fields)}
|
||||
onChange={(event) => updateConfig({ fields: selectFieldsFromText(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "aggregate" ? (
|
||||
<>
|
||||
<FormField label="Group by">
|
||||
<input
|
||||
value={stringList(node.config.group_by).join(", ")}
|
||||
onChange={(event) => updateConfig({ group_by: commaList(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Aggregates">
|
||||
<textarea
|
||||
className="dataflow-expression-editor"
|
||||
value={aggregateText}
|
||||
onChange={(event) => setAggregateText(event.target.value)}
|
||||
onBlur={commitAggregates}
|
||||
spellCheck={false}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "sort" ? (
|
||||
<FormField label="Sort fields">
|
||||
<textarea
|
||||
className="dataflow-expression-editor"
|
||||
value={sortText}
|
||||
onChange={(event) => setSortText(event.target.value)}
|
||||
onBlur={commitSort}
|
||||
spellCheck={false}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "limit" ? (
|
||||
<FormField label="Maximum rows">
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={100000}
|
||||
value={numberValue(node.config.count, 100)}
|
||||
onChange={(event) => updateConfig({ count: Number(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
</div>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function textValue(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function numberValue(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
||||
}
|
||||
|
||||
function displayScalar(value: unknown): string {
|
||||
if (value === null) return "null";
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return "";
|
||||
}
|
||||
|
||||
function parseScalar(value: string): string | number | boolean | null {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed === "null") return null;
|
||||
if (trimmed === "true") return true;
|
||||
if (trimmed === "false") return false;
|
||||
if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
||||
return value;
|
||||
}
|
||||
|
||||
function stringList(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function commaList(value: string): string[] {
|
||||
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function selectFieldsToText(value: unknown): string {
|
||||
if (!Array.isArray(value)) return "";
|
||||
return value.map((item) => {
|
||||
if (typeof item === "string") return item;
|
||||
if (!isRecord(item)) return "";
|
||||
const column = textValue(item.column);
|
||||
const alias = textValue(item.alias);
|
||||
return alias && alias !== column ? `${column} as ${alias}` : column;
|
||||
}).filter(Boolean).join(", ");
|
||||
}
|
||||
|
||||
function selectFieldsFromText(value: string): Array<{ column: string; alias: string }> {
|
||||
return commaList(value).map((item) => {
|
||||
const match = item.match(/^(.+?)\s+as\s+(.+)$/i);
|
||||
const column = (match?.[1] ?? item).trim();
|
||||
return { column, alias: (match?.[2] ?? column).trim() };
|
||||
});
|
||||
}
|
||||
|
||||
function aggregatesToText(value: unknown): string {
|
||||
if (!Array.isArray(value)) return "";
|
||||
return value.map((item) => {
|
||||
if (!isRecord(item)) return "";
|
||||
return `${textValue(item.function)}(${textValue(item.column) || "*"}) as ${textValue(item.alias)}`;
|
||||
}).filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function aggregatesFromText(value: string): Array<Record<string, string>> {
|
||||
const lines = value.split("\n").map((line) => line.trim()).filter(Boolean);
|
||||
if (!lines.length) throw new Error("Add at least one aggregate.");
|
||||
return lines.map((line) => {
|
||||
const match = line.match(/^(count|sum|avg|min|max)\(([^)]+)\)\s+as\s+([A-Za-z_][A-Za-z0-9_]*)$/i);
|
||||
if (!match) throw new Error(`Invalid aggregate: ${line}`);
|
||||
return {
|
||||
function: match[1].toLowerCase(),
|
||||
column: match[2].trim(),
|
||||
alias: match[3]
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function sortFieldsToText(value: unknown): string {
|
||||
if (!Array.isArray(value)) return "";
|
||||
return value.map((item) => {
|
||||
if (!isRecord(item)) return "";
|
||||
return `${textValue(item.column)} ${textValue(item.direction) || "asc"}`;
|
||||
}).filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function sortFieldsFromText(value: string): Array<Record<string, string>> {
|
||||
const lines = value.split("\n").map((line) => line.trim()).filter(Boolean);
|
||||
if (!lines.length) throw new Error("Add at least one sort field.");
|
||||
return lines.map((line) => {
|
||||
const match = line.match(/^(.+?)(?:\s+(asc|desc))?$/i);
|
||||
const column = match?.[1]?.trim() ?? "";
|
||||
if (!column) throw new Error(`Invalid sort field: ${line}`);
|
||||
return { column, direction: (match?.[2] ?? "asc").toLowerCase() };
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import type {
|
||||
EditorMode,
|
||||
Pipeline,
|
||||
PipelineGraph,
|
||||
PipelineGraphNode,
|
||||
PipelinePayload,
|
||||
PipelineStatus
|
||||
} from "../../api/dataflow";
|
||||
|
||||
export type PipelineDraft = {
|
||||
id: string | null;
|
||||
currentRevision: number | null;
|
||||
name: string;
|
||||
description: string;
|
||||
status: PipelineStatus;
|
||||
graph: PipelineGraph;
|
||||
sqlText: string;
|
||||
editorMode: EditorMode;
|
||||
};
|
||||
|
||||
export const NODE_LABELS: Record<string, string> = {
|
||||
"source.inline": "Inline source",
|
||||
"source.reference": "Connector source",
|
||||
filter: "Filter rows",
|
||||
select: "Select columns",
|
||||
aggregate: "Aggregate",
|
||||
sort: "Sort rows",
|
||||
limit: "Limit rows",
|
||||
output: "Output"
|
||||
};
|
||||
|
||||
export const PALETTE_NODE_TYPES = [
|
||||
"source.inline",
|
||||
"filter",
|
||||
"select",
|
||||
"aggregate",
|
||||
"sort",
|
||||
"limit",
|
||||
"output"
|
||||
] as const;
|
||||
|
||||
export function draftFromPipeline(pipeline: Pipeline): PipelineDraft {
|
||||
return {
|
||||
id: pipeline.id,
|
||||
currentRevision: pipeline.current_revision,
|
||||
name: pipeline.name,
|
||||
description: pipeline.description ?? "",
|
||||
status: pipeline.status,
|
||||
graph: structuredClone(pipeline.revision.graph),
|
||||
sqlText: pipeline.revision.sql_text ?? "",
|
||||
editorMode: pipeline.revision.editor_mode
|
||||
};
|
||||
}
|
||||
|
||||
export function sampleDraft(): PipelineDraft {
|
||||
return {
|
||||
id: null,
|
||||
currentRevision: null,
|
||||
name: "Monthly case overview",
|
||||
description: "Filter open cases and aggregate their value by district.",
|
||||
status: "draft",
|
||||
editorMode: "graph",
|
||||
sqlText: "",
|
||||
graph: {
|
||||
schema_version: 1,
|
||||
nodes: [
|
||||
{
|
||||
id: "source",
|
||||
type: "source.inline",
|
||||
label: "Monthly cases",
|
||||
position: { x: 60, y: 170 },
|
||||
config: {
|
||||
source_name: "monthly_cases",
|
||||
rows: [
|
||||
{ case_id: "A-1001", district: "North", status: "open", amount: 1200 },
|
||||
{ case_id: "A-1002", district: "South", status: "closed", amount: 800 },
|
||||
{ case_id: "A-1003", district: "North", status: "open", amount: 450 },
|
||||
{ case_id: "A-1004", district: "West", status: "open", amount: 2100 },
|
||||
{ case_id: "A-1005", district: "South", status: "open", amount: 650 }
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "filter-open",
|
||||
type: "filter",
|
||||
label: "Only open cases",
|
||||
position: { x: 300, y: 170 },
|
||||
config: { column: "status", operator: "eq", value: "open" }
|
||||
},
|
||||
{
|
||||
id: "aggregate-district",
|
||||
type: "aggregate",
|
||||
label: "Totals by district",
|
||||
position: { x: 540, y: 170 },
|
||||
config: {
|
||||
group_by: ["district"],
|
||||
aggregates: [
|
||||
{ function: "count", column: "*", alias: "cases" },
|
||||
{ function: "sum", column: "amount", alias: "total_amount" }
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "sort-total",
|
||||
type: "sort",
|
||||
label: "Largest total first",
|
||||
position: { x: 780, y: 170 },
|
||||
config: { fields: [{ column: "total_amount", direction: "desc" }] }
|
||||
},
|
||||
{
|
||||
id: "output",
|
||||
type: "output",
|
||||
label: "Preview output",
|
||||
position: { x: 1020, y: 170 },
|
||||
config: {}
|
||||
}
|
||||
],
|
||||
edges: [
|
||||
{ id: "source-filter", source: "source", target: "filter-open" },
|
||||
{ id: "filter-aggregate", source: "filter-open", target: "aggregate-district" },
|
||||
{ id: "aggregate-sort", source: "aggregate-district", target: "sort-total" },
|
||||
{ id: "sort-output", source: "sort-total", target: "output" }
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function pipelinePayload(draft: PipelineDraft): PipelinePayload {
|
||||
return {
|
||||
name: draft.name.trim(),
|
||||
description: draft.description.trim() || null,
|
||||
status: draft.status,
|
||||
graph: draft.graph,
|
||||
sql_text: draft.sqlText.trim() || null,
|
||||
editor_mode: draft.editorMode
|
||||
};
|
||||
}
|
||||
|
||||
export function sourceNodes(graph: PipelineGraph): PipelineGraphNode[] {
|
||||
return graph.nodes.filter((node) => node.type.startsWith("source."));
|
||||
}
|
||||
|
||||
export function draftFingerprint(draft: PipelineDraft | null): string {
|
||||
if (!draft) return "";
|
||||
return JSON.stringify({
|
||||
name: draft.name,
|
||||
description: draft.description,
|
||||
status: draft.status,
|
||||
graph: draft.graph,
|
||||
sqlText: draft.sqlText,
|
||||
editorMode: draft.editorMode
|
||||
});
|
||||
}
|
||||
|
||||
export function newNode(type: string, position: { x: number; y: number }): PipelineGraphNode {
|
||||
const id = `${type.replace(".", "-")}-${crypto.randomUUID()}`;
|
||||
const config: Record<string, unknown> = defaultNodeConfig(type);
|
||||
return {
|
||||
id,
|
||||
type,
|
||||
label: NODE_LABELS[type] ?? type,
|
||||
position,
|
||||
config
|
||||
};
|
||||
}
|
||||
|
||||
function defaultNodeConfig(type: string): Record<string, unknown> {
|
||||
if (type === "source.inline") return { source_name: "new_source", rows: [] };
|
||||
if (type === "filter") return { column: "", operator: "eq", value: "" };
|
||||
if (type === "select") return { fields: [{ column: "", alias: "" }] };
|
||||
if (type === "aggregate") {
|
||||
return { group_by: [], aggregates: [{ function: "count", column: "*", alias: "row_count" }] };
|
||||
}
|
||||
if (type === "sort") return { fields: [{ column: "", direction: "asc" }] };
|
||||
if (type === "limit") return { count: 100 };
|
||||
return {};
|
||||
}
|
||||
Reference in New Issue
Block a user