Expand governed Dataflow editor and node library
This commit is contained in:
@@ -2,6 +2,7 @@ import { apiFetch, type ApiSettings } from "@govoplan/core-webui";
|
||||
|
||||
export type PipelineStatus = "draft" | "active" | "archived";
|
||||
export type EditorMode = "graph" | "sql";
|
||||
export type NodeCategory = "load" | "combine" | "filter" | "transform" | "output";
|
||||
|
||||
export type GraphPosition = { x: number; y: number };
|
||||
export type PipelineGraphNode = {
|
||||
@@ -101,10 +102,109 @@ export type PipelinePreview = {
|
||||
truncated: boolean;
|
||||
diagnostics: DataflowDiagnostic[];
|
||||
node_diagnostics: NodePreviewDiagnostic[];
|
||||
source_fingerprints: Record<string, unknown>[];
|
||||
input_row_count: number;
|
||||
definition_hash: string;
|
||||
executor_version: string;
|
||||
};
|
||||
|
||||
export type NodePortDefinition = {
|
||||
id: string;
|
||||
label: string;
|
||||
required: boolean;
|
||||
multiple: boolean;
|
||||
minimum_connections: number;
|
||||
};
|
||||
|
||||
export type NodeConfigFieldDefinition = {
|
||||
id: string;
|
||||
label: string;
|
||||
kind: string;
|
||||
required: boolean;
|
||||
description?: string | null;
|
||||
options: Array<[string, string]>;
|
||||
};
|
||||
|
||||
export type NodeTypeDefinition = {
|
||||
type: string;
|
||||
category: NodeCategory;
|
||||
category_label: string;
|
||||
label: string;
|
||||
description: string;
|
||||
icon: string;
|
||||
input_ports: NodePortDefinition[];
|
||||
output_ports: NodePortDefinition[];
|
||||
config_fields: NodeConfigFieldDefinition[];
|
||||
default_config: Record<string, unknown>;
|
||||
sql_support: "full" | "partial" | "none";
|
||||
};
|
||||
|
||||
export type TabularSourceColumn = {
|
||||
name: string;
|
||||
data_type: string;
|
||||
nullable: boolean;
|
||||
};
|
||||
|
||||
export type TabularSource = {
|
||||
ref: string;
|
||||
provider: string;
|
||||
source_name: string;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
columns: TabularSourceColumn[];
|
||||
schema_version: string;
|
||||
fingerprint: string;
|
||||
row_count?: number | null;
|
||||
byte_count?: number | null;
|
||||
updated_at?: string | null;
|
||||
capabilities: string[];
|
||||
};
|
||||
|
||||
export type TabularSourceCatalogue = {
|
||||
available: boolean;
|
||||
writable: boolean;
|
||||
sources: TabularSource[];
|
||||
};
|
||||
|
||||
export async function listDataflowNodeTypes(settings: ApiSettings): Promise<NodeTypeDefinition[]> {
|
||||
const response = await apiFetch<{ nodes: NodeTypeDefinition[] }>(
|
||||
settings,
|
||||
"/api/v1/dataflow/node-types"
|
||||
);
|
||||
return response.nodes;
|
||||
}
|
||||
|
||||
export function listDataflowSources(
|
||||
settings: ApiSettings,
|
||||
query = ""
|
||||
): Promise<TabularSourceCatalogue> {
|
||||
const suffix = query.trim() ? `?query=${encodeURIComponent(query.trim())}` : "";
|
||||
return apiFetch<TabularSourceCatalogue>(settings, `/api/v1/dataflow/sources${suffix}`);
|
||||
}
|
||||
|
||||
export function createDataflowSourceSnapshot(
|
||||
settings: ApiSettings,
|
||||
payload: {
|
||||
name: string;
|
||||
source_name: string;
|
||||
description?: string | null;
|
||||
} & (
|
||||
{
|
||||
format: "json";
|
||||
rows: Record<string, unknown>[];
|
||||
} | {
|
||||
format: "csv";
|
||||
csv_text: string;
|
||||
delimiter: string;
|
||||
}
|
||||
)
|
||||
): Promise<TabularSource> {
|
||||
return apiFetch<TabularSource>(settings, "/api/v1/dataflow/sources/snapshots", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
}
|
||||
|
||||
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
|
||||
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
|
||||
return response.pipelines;
|
||||
|
||||
@@ -15,12 +15,13 @@ import {
|
||||
} from "@xyflow/react";
|
||||
import type {
|
||||
DataflowDiagnostic,
|
||||
NodeTypeDefinition,
|
||||
NodePreviewDiagnostic,
|
||||
PipelineGraph,
|
||||
PipelineGraphNode
|
||||
} from "../../api/dataflow";
|
||||
import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
|
||||
import { newNode } from "./model";
|
||||
import { FALLBACK_NODE_LIBRARY, newNode } from "./model";
|
||||
|
||||
const nodeTypes = { dataflow: DataflowNode };
|
||||
|
||||
@@ -28,6 +29,7 @@ type DataflowCanvasProps = {
|
||||
graph: PipelineGraph;
|
||||
diagnostics: DataflowDiagnostic[];
|
||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||
nodeLibrary: NodeTypeDefinition[];
|
||||
selectedNodeId: string | null;
|
||||
readOnly: boolean;
|
||||
onGraphChange: (graph: PipelineGraph) => void;
|
||||
@@ -38,6 +40,7 @@ export default function DataflowCanvas({
|
||||
graph,
|
||||
diagnostics,
|
||||
nodeDiagnostics,
|
||||
nodeLibrary,
|
||||
selectedNodeId,
|
||||
readOnly,
|
||||
onGraphChange,
|
||||
@@ -52,27 +55,39 @@ export default function DataflowCanvas({
|
||||
() => new Map(nodeDiagnostics.map((item) => [item.node_id, item.output_rows])),
|
||||
[nodeDiagnostics]
|
||||
);
|
||||
const definitions = useMemo(
|
||||
() => new Map(nodeLibrary.map((definition) => [definition.type, definition])),
|
||||
[nodeLibrary]
|
||||
);
|
||||
const nodes = useMemo<DataflowFlowNode[]>(
|
||||
() => graph.nodes.map((node) => ({
|
||||
id: node.id,
|
||||
type: "dataflow",
|
||||
position: node.position,
|
||||
selected: node.id === selectedNodeId,
|
||||
data: {
|
||||
label: node.label,
|
||||
transformType: node.type,
|
||||
config: node.config,
|
||||
hasError: errorNodeIds.has(node.id),
|
||||
outputRows: rowCounts.get(node.id)
|
||||
}
|
||||
})),
|
||||
[errorNodeIds, graph.nodes, rowCounts, selectedNodeId]
|
||||
() => graph.nodes.map((node) => {
|
||||
const definition = definitions.get(node.type)
|
||||
?? FALLBACK_NODE_LIBRARY.find((item) => item.type === node.type)
|
||||
?? FALLBACK_NODE_LIBRARY[0];
|
||||
return {
|
||||
id: node.id,
|
||||
type: "dataflow",
|
||||
position: node.position,
|
||||
selected: node.id === selectedNodeId,
|
||||
data: {
|
||||
label: node.label,
|
||||
transformType: node.type,
|
||||
definition,
|
||||
config: node.config,
|
||||
hasError: errorNodeIds.has(node.id),
|
||||
outputRows: rowCounts.get(node.id)
|
||||
}
|
||||
};
|
||||
}),
|
||||
[definitions, errorNodeIds, graph.nodes, rowCounts, selectedNodeId]
|
||||
);
|
||||
const edges = useMemo<Edge[]>(
|
||||
() => graph.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
sourceHandle: edge.source_port ?? "output",
|
||||
targetHandle: edge.target_port ?? "input",
|
||||
type: "smoothstep",
|
||||
className: "dataflow-edge"
|
||||
})),
|
||||
@@ -113,10 +128,30 @@ export default function DataflowCanvas({
|
||||
const source = graph.nodes.find((node) => node.id === connection.source);
|
||||
const target = graph.nodes.find((node) => node.id === connection.target);
|
||||
if (!source || !target || source.type === "output" || target.type.startsWith("source.")) return false;
|
||||
const targetAlreadyConnected = graph.edges.some(
|
||||
(edge) => edge.target === connection.target && edge.source !== connection.source
|
||||
const sourcePort = connection.sourceHandle ?? "output";
|
||||
const targetPort = connection.targetHandle ?? "input";
|
||||
const sourceDefinition = definitions.get(source.type);
|
||||
const targetDefinition = definitions.get(target.type);
|
||||
if (
|
||||
!sourceDefinition?.output_ports.some((port) => port.id === sourcePort)
|
||||
|| !targetDefinition?.input_ports.some((port) => port.id === targetPort)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const duplicate = graph.edges.some(
|
||||
(edge) =>
|
||||
edge.source === connection.source
|
||||
&& edge.target === connection.target
|
||||
&& (edge.source_port ?? "output") === sourcePort
|
||||
&& (edge.target_port ?? "input") === targetPort
|
||||
);
|
||||
return !targetAlreadyConnected;
|
||||
if (duplicate) return false;
|
||||
const port = targetDefinition.input_ports.find((item) => item.id === targetPort);
|
||||
const targetPortConnections = graph.edges.filter(
|
||||
(edge) => edge.target === connection.target && (edge.target_port ?? "input") === targetPort
|
||||
);
|
||||
if (!port?.multiple && targetPortConnections.length) return false;
|
||||
return !wouldCreateCycle(graph, connection.source, connection.target);
|
||||
};
|
||||
|
||||
const onConnect = (connection: Connection) => {
|
||||
@@ -138,7 +173,7 @@ export default function DataflowCanvas({
|
||||
const type = event.dataTransfer.getData("application/x-govoplan-dataflow-node");
|
||||
if (!type) return;
|
||||
const position = instance.screenToFlowPosition({ x: event.clientX, y: event.clientY });
|
||||
const node = newNode(type, position);
|
||||
const node = newNode(type, position, nodeLibrary);
|
||||
onGraphChange({ ...graph, nodes: [...graph.nodes, node] });
|
||||
onSelectNode(node.id);
|
||||
};
|
||||
@@ -199,6 +234,23 @@ export default function DataflowCanvas({
|
||||
);
|
||||
}
|
||||
|
||||
function wouldCreateCycle(graph: PipelineGraph, source: string, target: string): boolean {
|
||||
const outgoing = new Map<string, string[]>();
|
||||
graph.edges.forEach((edge) => {
|
||||
outgoing.set(edge.source, [...(outgoing.get(edge.source) ?? []), edge.target]);
|
||||
});
|
||||
const pending = [target];
|
||||
const visited = new Set<string>();
|
||||
while (pending.length) {
|
||||
const nodeId = pending.pop();
|
||||
if (!nodeId || visited.has(nodeId)) continue;
|
||||
if (nodeId === source) return true;
|
||||
visited.add(nodeId);
|
||||
pending.push(...(outgoing.get(nodeId) ?? []));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function updateGraphNode(
|
||||
graph: PipelineGraph,
|
||||
updatedNode: PipelineGraphNode
|
||||
|
||||
@@ -1,19 +1,11 @@
|
||||
import type { ComponentType } from "react";
|
||||
import {
|
||||
Braces,
|
||||
Columns3,
|
||||
Database,
|
||||
Filter,
|
||||
ListEnd,
|
||||
Sigma,
|
||||
SortAsc,
|
||||
Unplug
|
||||
} from "lucide-react";
|
||||
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
|
||||
import type { NodeTypeDefinition } from "../../api/dataflow";
|
||||
import { dataflowNodeIcon } from "./nodeIcons";
|
||||
|
||||
export type DataflowNodeData = {
|
||||
label: string;
|
||||
transformType: string;
|
||||
definition: NodeTypeDefinition;
|
||||
config: Record<string, unknown>;
|
||||
hasError?: boolean;
|
||||
outputRows?: number;
|
||||
@@ -21,21 +13,8 @@ export type DataflowNodeData = {
|
||||
|
||||
export type DataflowFlowNode = Node<DataflowNodeData, "dataflow">;
|
||||
|
||||
const iconByType: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
|
||||
"source.inline": Braces,
|
||||
"source.reference": Database,
|
||||
filter: Filter,
|
||||
select: Columns3,
|
||||
aggregate: Sigma,
|
||||
sort: SortAsc,
|
||||
limit: ListEnd,
|
||||
output: Unplug
|
||||
};
|
||||
|
||||
export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowNode>) {
|
||||
const Icon = iconByType[data.transformType] ?? Braces;
|
||||
const isSource = data.transformType.startsWith("source.");
|
||||
const isOutput = data.transformType === "output";
|
||||
const Icon = dataflowNodeIcon(data.definition.icon);
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -46,42 +25,51 @@ export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowN
|
||||
data.hasError ? "has-error" : ""
|
||||
].filter(Boolean).join(" ")}
|
||||
>
|
||||
{!isSource ? (
|
||||
{data.definition.input_ports.map((port, index) => (
|
||||
<Handle
|
||||
key={port.id}
|
||||
id={port.id}
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
className="dataflow-node-handle dataflow-node-handle-input"
|
||||
style={{ top: portPosition(index, data.definition.input_ports.length) }}
|
||||
/>
|
||||
) : null}
|
||||
))}
|
||||
{data.definition.input_ports.length > 1
|
||||
? data.definition.input_ports.map((port, index) => (
|
||||
<span
|
||||
key={port.id}
|
||||
className="dataflow-node-port-label"
|
||||
style={{ top: portPosition(index, data.definition.input_ports.length) }}
|
||||
>
|
||||
{port.label}
|
||||
</span>
|
||||
))
|
||||
: null}
|
||||
<span className="dataflow-node-icon" aria-hidden="true">
|
||||
<Icon size={17} strokeWidth={1.8} />
|
||||
</span>
|
||||
<span className="dataflow-node-copy">
|
||||
<strong>{data.label}</strong>
|
||||
<small>{transformLabel(data.transformType)}</small>
|
||||
<small>{data.definition.label}</small>
|
||||
</span>
|
||||
{typeof data.outputRows === "number" ? (
|
||||
<span className="dataflow-node-count">{data.outputRows}</span>
|
||||
) : null}
|
||||
{!isOutput ? (
|
||||
{data.definition.output_ports.map((port, index) => (
|
||||
<Handle
|
||||
key={port.id}
|
||||
id={port.id}
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
className="dataflow-node-handle dataflow-node-handle-output"
|
||||
style={{ top: portPosition(index, data.definition.output_ports.length) }}
|
||||
/>
|
||||
) : null}
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function transformLabel(type: string): string {
|
||||
if (type === "source.inline") return "Inline source";
|
||||
if (type === "source.reference") return "Connector source";
|
||||
if (type === "filter") return "Filter";
|
||||
if (type === "select") return "Projection";
|
||||
if (type === "aggregate") return "Aggregation";
|
||||
if (type === "sort") return "Sort";
|
||||
if (type === "limit") return "Limit";
|
||||
if (type === "output") return "Output";
|
||||
return type;
|
||||
function portPosition(index: number, count: number): string {
|
||||
return `${((index + 1) / (count + 1)) * 100}%`;
|
||||
}
|
||||
|
||||
@@ -3,32 +3,27 @@ import {
|
||||
useEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
type ComponentType,
|
||||
type DragEvent
|
||||
} from "react";
|
||||
import {
|
||||
Braces,
|
||||
CheckCircle2,
|
||||
Code2,
|
||||
Columns3,
|
||||
Database,
|
||||
Filter,
|
||||
ListEnd,
|
||||
Network,
|
||||
Play,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
RotateCcw,
|
||||
Save,
|
||||
Sigma,
|
||||
SortAsc,
|
||||
Trash2,
|
||||
TriangleAlert
|
||||
TriangleAlert,
|
||||
Upload
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Button,
|
||||
ConfirmDialog,
|
||||
Dialog,
|
||||
DismissibleAlert,
|
||||
FormField,
|
||||
IconButton,
|
||||
LoadingFrame,
|
||||
SegmentedControl,
|
||||
@@ -44,24 +39,29 @@ import { ReactFlowProvider } from "@xyflow/react";
|
||||
import {
|
||||
compileDataflowSql,
|
||||
createDataflowPipeline,
|
||||
createDataflowSourceSnapshot,
|
||||
deleteDataflowPipeline,
|
||||
listDataflowNodeTypes,
|
||||
listDataflowPipelines,
|
||||
listDataflowSources,
|
||||
previewDataflowPipeline,
|
||||
renderDataflowSql,
|
||||
updateDataflowPipeline,
|
||||
validateDataflowPipeline,
|
||||
type DataflowDiagnostic,
|
||||
type EditorMode,
|
||||
type NodeCategory,
|
||||
type NodeTypeDefinition,
|
||||
type NodePreviewDiagnostic,
|
||||
type Pipeline,
|
||||
type PipelineGraphNode,
|
||||
type PipelinePreview
|
||||
type PipelinePreview,
|
||||
type TabularSource
|
||||
} from "../../api/dataflow";
|
||||
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
||||
import NodeInspector from "./NodeInspector";
|
||||
import {
|
||||
NODE_LABELS,
|
||||
PALETTE_NODE_TYPES,
|
||||
FALLBACK_NODE_LIBRARY,
|
||||
draftFingerprint,
|
||||
draftFromPipeline,
|
||||
newNode,
|
||||
@@ -70,18 +70,12 @@ import {
|
||||
sourceNodes,
|
||||
type PipelineDraft
|
||||
} from "./model";
|
||||
import { dataflowNodeIcon } from "./nodeIcons";
|
||||
|
||||
type ResultTab = "preview" | "diagnostics";
|
||||
type SnapshotFormat = "json" | "csv";
|
||||
|
||||
const paletteIcons: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
|
||||
"source.inline": Braces,
|
||||
filter: Filter,
|
||||
select: Columns3,
|
||||
aggregate: Sigma,
|
||||
sort: SortAsc,
|
||||
limit: ListEnd,
|
||||
output: Network
|
||||
};
|
||||
const NODE_CATEGORY_ORDER: NodeCategory[] = ["load", "combine", "filter", "transform", "output"];
|
||||
|
||||
export default function DataflowPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo }) {
|
||||
const { requestNavigation, requestDiscard } = useUnsavedChanges();
|
||||
@@ -101,9 +95,20 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
const [resultOpen, setResultOpen] = useState(false);
|
||||
const [resultTab, setResultTab] = useState<ResultTab>("preview");
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
const [snapshotOpen, setSnapshotOpen] = useState(false);
|
||||
const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
|
||||
const [sources, setSources] = useState<TabularSource[]>([]);
|
||||
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
|
||||
const [sourceCatalogueWritable, setSourceCatalogueWritable] = useState(false);
|
||||
|
||||
const canWrite = hasScope(auth, "dataflow:pipeline:write") || hasScope(auth, "dataflow:pipeline:admin");
|
||||
const canRun = hasScope(auth, "dataflow:pipeline:run") || hasScope(auth, "dataflow:pipeline:admin");
|
||||
const canImportSources = canWrite
|
||||
&& sourceCatalogueWritable
|
||||
&& (
|
||||
hasScope(auth, "connectors:source:write")
|
||||
|| hasScope(auth, "connectors:source:admin")
|
||||
);
|
||||
const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft);
|
||||
const selectedNode = useMemo(
|
||||
() => draft?.graph.nodes.find((node) => node.id === selectedNodeId) ?? null,
|
||||
@@ -116,6 +121,14 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
`${pipeline.name} ${pipeline.description ?? ""} ${pipeline.status}`.toLocaleLowerCase().includes(query)
|
||||
);
|
||||
}, [pipelines, search]);
|
||||
const paletteGroups = useMemo(
|
||||
() => NODE_CATEGORY_ORDER.map((category) => ({
|
||||
category,
|
||||
label: nodeLibrary.find((item) => item.category === category)?.category_label ?? category,
|
||||
nodes: nodeLibrary.filter((item) => item.category === category)
|
||||
})).filter((group) => group.nodes.length),
|
||||
[nodeLibrary]
|
||||
);
|
||||
|
||||
const loadPipelines = useCallback(async (preferredId?: string | null) => {
|
||||
setLoading(true);
|
||||
@@ -145,6 +158,33 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
void loadPipelines();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void listDataflowNodeTypes(settings)
|
||||
.then((items) => {
|
||||
if (!cancelled && items.length) setNodeLibrary(items);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setNodeLibrary(FALLBACK_NODE_LIBRARY);
|
||||
});
|
||||
void listDataflowSources(settings)
|
||||
.then((catalogue) => {
|
||||
if (cancelled) return;
|
||||
setSources(catalogue.sources);
|
||||
setSourceCatalogueAvailable(catalogue.available);
|
||||
setSourceCatalogueWritable(catalogue.writable);
|
||||
})
|
||||
.catch(() => {
|
||||
if (cancelled) return;
|
||||
setSources([]);
|
||||
setSourceCatalogueAvailable(false);
|
||||
setSourceCatalogueWritable(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [settings]);
|
||||
|
||||
const discardDraft = useCallback(() => {
|
||||
if (savedDraft) {
|
||||
const reset = structuredClone(savedDraft);
|
||||
@@ -383,7 +423,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
const node = newNode(type, {
|
||||
x: 120 + draft.graph.nodes.length * 70,
|
||||
y: 100 + (draft.graph.nodes.length % 4) * 80
|
||||
});
|
||||
}, nodeLibrary);
|
||||
updateGraph({ ...draft.graph, nodes: [...draft.graph.nodes, node] });
|
||||
setSelectedNodeId(node.id);
|
||||
};
|
||||
@@ -528,27 +568,41 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
{draft.editorMode === "graph" ? (
|
||||
<aside className="dataflow-palette" aria-label="Transform palette">
|
||||
<div className="dataflow-panel-heading">
|
||||
<strong>Transforms</strong>
|
||||
<strong>Nodes</strong>
|
||||
{canImportSources ? (
|
||||
<IconButton
|
||||
label="Import tabular snapshot"
|
||||
icon={<Upload size={15} />}
|
||||
variant="ghost"
|
||||
onClick={() => setSnapshotOpen(true)}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="dataflow-palette-items">
|
||||
{PALETTE_NODE_TYPES.map((type) => {
|
||||
const Icon = paletteIcons[type] ?? Database;
|
||||
const disabled = !canWrite || uniqueNodeExists(draft, type);
|
||||
return (
|
||||
<button
|
||||
key={type}
|
||||
type="button"
|
||||
draggable={!disabled}
|
||||
disabled={disabled}
|
||||
onDragStart={(event) => startPaletteDrag(event, type)}
|
||||
onClick={() => addNode(type)}
|
||||
>
|
||||
<Icon size={16} />
|
||||
<span>{NODE_LABELS[type]}</span>
|
||||
<Plus size={14} className="dataflow-palette-add" />
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
{paletteGroups.map((group) => (
|
||||
<section key={group.category} className="dataflow-palette-group">
|
||||
<h3>{group.label}</h3>
|
||||
{group.nodes.map((definition) => {
|
||||
const Icon = dataflowNodeIcon(definition.icon);
|
||||
const disabled = !canWrite || uniqueNodeExists(draft, definition.type);
|
||||
return (
|
||||
<button
|
||||
key={definition.type}
|
||||
type="button"
|
||||
title={definition.description}
|
||||
draggable={!disabled}
|
||||
disabled={disabled}
|
||||
onDragStart={(event) => startPaletteDrag(event, definition.type)}
|
||||
onClick={() => addNode(definition.type)}
|
||||
>
|
||||
<Icon size={16} />
|
||||
<span>{definition.label}</span>
|
||||
<Plus size={14} className="dataflow-palette-add" />
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</section>
|
||||
))}
|
||||
</div>
|
||||
</aside>
|
||||
) : null}
|
||||
@@ -559,6 +613,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
graph={draft.graph}
|
||||
diagnostics={diagnostics}
|
||||
nodeDiagnostics={nodeDiagnostics}
|
||||
nodeLibrary={nodeLibrary}
|
||||
selectedNodeId={selectedNodeId}
|
||||
readOnly={!canWrite}
|
||||
onGraphChange={updateGraph}
|
||||
@@ -589,6 +644,9 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
</section>
|
||||
<NodeInspector
|
||||
node={selectedNode}
|
||||
nodeLibrary={nodeLibrary}
|
||||
sources={sources}
|
||||
sourceCatalogueAvailable={sourceCatalogueAvailable}
|
||||
readOnly={!canWrite}
|
||||
onChange={(node: PipelineGraphNode) => updateGraph(updateGraphNode(draft.graph, node))}
|
||||
onDelete={(nodeId) => {
|
||||
@@ -634,10 +692,218 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
||||
onCancel={() => setDeleteOpen(false)}
|
||||
onConfirm={() => void removePipeline()}
|
||||
/>
|
||||
<SourceSnapshotDialog
|
||||
open={snapshotOpen}
|
||||
settings={settings}
|
||||
onClose={() => setSnapshotOpen(false)}
|
||||
onCreated={(source) => {
|
||||
setSources((current) => [
|
||||
source,
|
||||
...current.filter((item) => item.ref !== source.ref)
|
||||
]);
|
||||
setSourceCatalogueAvailable(true);
|
||||
setSnapshotOpen(false);
|
||||
if (!draft) return;
|
||||
const node = newNode(
|
||||
"source.reference",
|
||||
{
|
||||
x: 100 + draft.graph.nodes.length * 60,
|
||||
y: 90 + (draft.graph.nodes.length % 4) * 90
|
||||
},
|
||||
nodeLibrary
|
||||
);
|
||||
node.label = source.name;
|
||||
node.config = {
|
||||
...node.config,
|
||||
source_ref: source.ref,
|
||||
source_name: source.source_name,
|
||||
expected_fingerprint: source.fingerprint,
|
||||
source_columns: source.columns
|
||||
};
|
||||
updateGraph({ ...draft.graph, nodes: [...draft.graph.nodes, node] });
|
||||
setSelectedNodeId(node.id);
|
||||
}}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
function SourceSnapshotDialog({
|
||||
open,
|
||||
settings,
|
||||
onClose,
|
||||
onCreated
|
||||
}: {
|
||||
open: boolean;
|
||||
settings: ApiSettings;
|
||||
onClose: () => void;
|
||||
onCreated: (source: TabularSource) => void;
|
||||
}) {
|
||||
const [name, setName] = useState("");
|
||||
const [sourceName, setSourceName] = useState("");
|
||||
const [description, setDescription] = useState("");
|
||||
const [format, setFormat] = useState<SnapshotFormat>("json");
|
||||
const [rowsText, setRowsText] = useState("[]");
|
||||
const [csvText, setCsvText] = useState("");
|
||||
const [delimiter, setDelimiter] = useState(",");
|
||||
const [fileInputKey, setFileInputKey] = useState(0);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
setName("");
|
||||
setSourceName("");
|
||||
setDescription("");
|
||||
setFormat("json");
|
||||
setRowsText("[]");
|
||||
setCsvText("");
|
||||
setDelimiter(",");
|
||||
setFileInputKey((current) => current + 1);
|
||||
setError("");
|
||||
}, [open]);
|
||||
|
||||
const create = async () => {
|
||||
setError("");
|
||||
let rows: Record<string, unknown>[] = [];
|
||||
if (format === "json") {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(rowsText);
|
||||
if (!Array.isArray(parsed) || parsed.some((row) => !isRecord(row))) {
|
||||
throw new Error("Rows must be a JSON array of objects.");
|
||||
}
|
||||
rows = parsed;
|
||||
} catch (parseError) {
|
||||
setError(parseError instanceof Error ? parseError.message : "Rows could not be parsed.");
|
||||
return;
|
||||
}
|
||||
} else if (!csvText.trim()) {
|
||||
setError("Choose a CSV file or paste CSV data.");
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
const source = await createDataflowSourceSnapshot(settings, {
|
||||
name: name.trim(),
|
||||
source_name: sourceName.trim(),
|
||||
description: description.trim() || null,
|
||||
...(format === "json"
|
||||
? { format, rows }
|
||||
: { format, csv_text: csvText, delimiter })
|
||||
});
|
||||
onCreated(source);
|
||||
} catch (createError) {
|
||||
setError(apiErrorMessage(createError));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const loadCsvFile = async (file: File | undefined) => {
|
||||
if (!file) return;
|
||||
setError("");
|
||||
try {
|
||||
setCsvText(await file.text());
|
||||
const baseName = file.name.replace(/\.[^.]+$/, "");
|
||||
setName((current) => current || baseName);
|
||||
setSourceName((current) => current || sourceNameFromFile(baseName));
|
||||
} catch {
|
||||
setError("The selected CSV file could not be read.");
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
title="Import tabular snapshot"
|
||||
className="dataflow-source-dialog"
|
||||
onClose={() => {
|
||||
if (!busy) onClose();
|
||||
}}
|
||||
footer={(
|
||||
<>
|
||||
<Button onClick={onClose} disabled={busy}>Cancel</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={() => void create()}
|
||||
disabled={busy || !name.trim() || !sourceName.trim()}
|
||||
>
|
||||
{busy ? "Importing..." : "Import"}
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
>
|
||||
<div className="dataflow-source-dialog-fields">
|
||||
{error ? (
|
||||
<DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert>
|
||||
) : null}
|
||||
<div className="dataflow-source-dialog-format">
|
||||
<SegmentedControl<SnapshotFormat>
|
||||
ariaLabel="Snapshot format"
|
||||
options={[
|
||||
{ id: "json", label: "JSON" },
|
||||
{ id: "csv", label: "CSV" }
|
||||
]}
|
||||
value={format}
|
||||
onChange={setFormat}
|
||||
/>
|
||||
</div>
|
||||
<FormField label="Name">
|
||||
<input value={name} onChange={(event) => setName(event.target.value)} />
|
||||
</FormField>
|
||||
<FormField label="Logical source name">
|
||||
<input
|
||||
value={sourceName}
|
||||
onChange={(event) => setSourceName(event.target.value)}
|
||||
pattern="[A-Za-z_][A-Za-z0-9_]*"
|
||||
placeholder="monthly_input"
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Description">
|
||||
<input value={description} onChange={(event) => setDescription(event.target.value)} />
|
||||
</FormField>
|
||||
{format === "csv" ? (
|
||||
<>
|
||||
<FormField label="CSV file">
|
||||
<input
|
||||
key={fileInputKey}
|
||||
type="file"
|
||||
accept=".csv,.tsv,text/csv,text/tab-separated-values,text/plain"
|
||||
onChange={(event) => void loadCsvFile(event.target.files?.[0])}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Delimiter">
|
||||
<select value={delimiter} onChange={(event) => setDelimiter(event.target.value)}>
|
||||
<option value=",">Comma</option>
|
||||
<option value=";">Semicolon</option>
|
||||
<option value={"\t"}>Tab</option>
|
||||
<option value="|">Pipe</option>
|
||||
</select>
|
||||
</FormField>
|
||||
<FormField label="CSV data">
|
||||
<textarea
|
||||
className="dataflow-json-editor"
|
||||
value={csvText}
|
||||
onChange={(event) => setCsvText(event.target.value)}
|
||||
spellCheck={false}
|
||||
/>
|
||||
</FormField>
|
||||
</>
|
||||
) : (
|
||||
<FormField label="Rows">
|
||||
<textarea
|
||||
className="dataflow-json-editor"
|
||||
value={rowsText}
|
||||
onChange={(event) => setRowsText(event.target.value)}
|
||||
spellCheck={false}
|
||||
/>
|
||||
</FormField>
|
||||
)}
|
||||
</div>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
function ResultPanel({
|
||||
tab,
|
||||
onTabChange,
|
||||
@@ -702,9 +968,17 @@ function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{preview.truncated ? (
|
||||
<div className="dataflow-preview-truncated">Showing {preview.rows.length} of {preview.total_rows} rows</div>
|
||||
) : null}
|
||||
<div className="dataflow-preview-summary">
|
||||
<span>
|
||||
Showing {preview.rows.length} of {preview.total_rows} output rows
|
||||
</span>
|
||||
<span>{preview.input_row_count} input rows</span>
|
||||
<span>
|
||||
{preview.source_fingerprints.length} source
|
||||
{preview.source_fingerprints.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<span>{preview.executor_version}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -747,7 +1021,6 @@ function DiagnosticsPanel({
|
||||
|
||||
function uniqueNodeExists(draft: PipelineDraft, type: string): boolean {
|
||||
if (type === "output") return draft.graph.nodes.some((node) => node.type === "output");
|
||||
if (type.startsWith("source.")) return draft.graph.nodes.some((node) => node.type.startsWith("source."));
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -762,6 +1035,22 @@ function formatCell(value: unknown): string {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function sourceNameFromFile(value: string): string {
|
||||
const normalized = value
|
||||
.normalize("NFKD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/[^A-Za-z0-9_]+/g, "_")
|
||||
.replace(/^_+|_+$/g, "")
|
||||
.toLocaleLowerCase()
|
||||
.slice(0, 120);
|
||||
if (!normalized) return "imported_source";
|
||||
return (/^[A-Za-z_]/.test(normalized) ? normalized : `source_${normalized}`).slice(0, 120);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function apiErrorMessage(error: unknown): string {
|
||||
if (!isApiError(error)) return error instanceof Error ? error.message : "The request failed.";
|
||||
try {
|
||||
|
||||
@@ -1,16 +1,31 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Trash2 } from "lucide-react";
|
||||
import { Button, DismissibleAlert, FormField } from "@govoplan/core-webui";
|
||||
import type { PipelineGraphNode } from "../../api/dataflow";
|
||||
import type {
|
||||
NodeTypeDefinition,
|
||||
PipelineGraphNode,
|
||||
TabularSource
|
||||
} from "../../api/dataflow";
|
||||
|
||||
type NodeInspectorProps = {
|
||||
node: PipelineGraphNode | null;
|
||||
nodeLibrary: NodeTypeDefinition[];
|
||||
sources: TabularSource[];
|
||||
sourceCatalogueAvailable: boolean;
|
||||
readOnly: boolean;
|
||||
onChange: (node: PipelineGraphNode) => void;
|
||||
onDelete: (nodeId: string) => void;
|
||||
};
|
||||
|
||||
export default function NodeInspector({ node, readOnly, onChange, onDelete }: NodeInspectorProps) {
|
||||
export default function NodeInspector({
|
||||
node,
|
||||
nodeLibrary,
|
||||
sources,
|
||||
sourceCatalogueAvailable,
|
||||
readOnly,
|
||||
onChange,
|
||||
onDelete
|
||||
}: NodeInspectorProps) {
|
||||
const [rowsText, setRowsText] = useState("");
|
||||
const [aggregateText, setAggregateText] = useState("");
|
||||
const [sortText, setSortText] = useState("");
|
||||
@@ -34,6 +49,7 @@ export default function NodeInspector({ node, readOnly, onChange, onDelete }: No
|
||||
);
|
||||
}
|
||||
|
||||
const definition = nodeLibrary.find((item) => item.type === node.type);
|
||||
const updateConfig = (patch: Record<string, unknown>) => {
|
||||
onChange({ ...node, config: { ...node.config, ...patch } });
|
||||
};
|
||||
@@ -76,7 +92,7 @@ export default function NodeInspector({ node, readOnly, onChange, onDelete }: No
|
||||
<div className="dataflow-panel-heading">
|
||||
<span>
|
||||
<strong>Inspector</strong>
|
||||
<small>{node.type}</small>
|
||||
<small>{definition?.label ?? node.type}</small>
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
@@ -111,6 +127,50 @@ export default function NodeInspector({ node, readOnly, onChange, onDelete }: No
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "source.reference" ? (
|
||||
<>
|
||||
<FormField label="Source">
|
||||
<select
|
||||
value={textValue(node.config.source_ref)}
|
||||
onChange={(event) => {
|
||||
const source = sources.find((item) => item.ref === event.target.value);
|
||||
if (!source) {
|
||||
updateConfig({
|
||||
source_ref: "",
|
||||
expected_fingerprint: ""
|
||||
});
|
||||
return;
|
||||
}
|
||||
updateConfig({
|
||||
source_ref: source.ref,
|
||||
source_name: source.source_name,
|
||||
expected_fingerprint: source.fingerprint,
|
||||
source_columns: source.columns
|
||||
});
|
||||
}}
|
||||
disabled={readOnly || !sourceCatalogueAvailable}
|
||||
>
|
||||
<option value="">
|
||||
{sourceCatalogueAvailable ? "Choose a source" : "Connectors unavailable"}
|
||||
</option>
|
||||
{sources.map((source) => (
|
||||
<option key={source.ref} value={source.ref}>
|
||||
{source.name} ({source.row_count ?? "?"} rows)
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</FormField>
|
||||
{textValue(node.config.expected_fingerprint) ? (
|
||||
<FormField label="Pinned fingerprint">
|
||||
<input
|
||||
value={textValue(node.config.expected_fingerprint)}
|
||||
readOnly
|
||||
title={textValue(node.config.expected_fingerprint)}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "source.inline" ? (
|
||||
<FormField label="Rows">
|
||||
<textarea
|
||||
@@ -160,6 +220,65 @@ export default function NodeInspector({ node, readOnly, onChange, onDelete }: No
|
||||
) : null}
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "distinct" ? (
|
||||
<FormField label="Key columns">
|
||||
<input
|
||||
value={stringList(node.config.columns).join(", ")}
|
||||
onChange={(event) => updateConfig({ columns: commaList(event.target.value) })}
|
||||
placeholder="All columns"
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "combine.union" ? (
|
||||
<FormField label="Duplicates">
|
||||
<select
|
||||
value={textValue(node.config.mode) || "all"}
|
||||
onChange={(event) => updateConfig({ mode: event.target.value })}
|
||||
disabled={readOnly}
|
||||
>
|
||||
<option value="all">Keep all rows</option>
|
||||
<option value="distinct">Remove duplicates</option>
|
||||
</select>
|
||||
</FormField>
|
||||
) : null}
|
||||
{node.type === "combine.join" ? (
|
||||
<>
|
||||
<FormField label="Join type">
|
||||
<select
|
||||
value={textValue(node.config.join_type) || "inner"}
|
||||
onChange={(event) => updateConfig({ join_type: event.target.value })}
|
||||
disabled={readOnly}
|
||||
>
|
||||
<option value="inner">Matching rows</option>
|
||||
<option value="left">All left rows</option>
|
||||
<option value="right">All right rows</option>
|
||||
<option value="full">All rows</option>
|
||||
</select>
|
||||
</FormField>
|
||||
<FormField label="Left keys">
|
||||
<input
|
||||
value={stringList(node.config.left_keys).join(", ")}
|
||||
onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Right keys">
|
||||
<input
|
||||
value={stringList(node.config.right_keys).join(", ")}
|
||||
onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Right-column prefix">
|
||||
<input
|
||||
value={textValue(node.config.right_prefix)}
|
||||
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "select" ? (
|
||||
<FormField label="Columns">
|
||||
<input
|
||||
@@ -190,6 +309,51 @@ export default function NodeInspector({ node, readOnly, onChange, onDelete }: No
|
||||
</FormField>
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "derive" ? (
|
||||
<>
|
||||
<FormField label="Output column">
|
||||
<input
|
||||
value={textValue(node.config.target_column)}
|
||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
<FormField label="Operation">
|
||||
<select
|
||||
value={textValue(node.config.operation) || "copy"}
|
||||
onChange={(event) => updateConfig({ operation: event.target.value })}
|
||||
disabled={readOnly}
|
||||
>
|
||||
<option value="copy">Copy</option>
|
||||
<option value="upper">Uppercase</option>
|
||||
<option value="lower">Lowercase</option>
|
||||
<option value="trim">Trim whitespace</option>
|
||||
<option value="concat">Concatenate</option>
|
||||
<option value="coalesce">First non-empty</option>
|
||||
<option value="add">Add</option>
|
||||
<option value="subtract">Subtract</option>
|
||||
<option value="multiply">Multiply</option>
|
||||
<option value="divide">Divide</option>
|
||||
</select>
|
||||
</FormField>
|
||||
<FormField label="Source columns">
|
||||
<input
|
||||
value={stringList(node.config.source_columns).join(", ")}
|
||||
onChange={(event) => updateConfig({ source_columns: commaList(event.target.value) })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
{textValue(node.config.operation) === "concat" ? (
|
||||
<FormField label="Separator">
|
||||
<input
|
||||
value={textValue(node.config.separator)}
|
||||
onChange={(event) => updateConfig({ separator: event.target.value })}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
</FormField>
|
||||
) : null}
|
||||
</>
|
||||
) : null}
|
||||
{node.type === "sort" ? (
|
||||
<FormField label="Sort fields">
|
||||
<textarea
|
||||
|
||||
@@ -4,7 +4,8 @@ import type {
|
||||
PipelineGraph,
|
||||
PipelineGraphNode,
|
||||
PipelinePayload,
|
||||
PipelineStatus
|
||||
PipelineStatus,
|
||||
NodeTypeDefinition
|
||||
} from "../../api/dataflow";
|
||||
|
||||
export type PipelineDraft = {
|
||||
@@ -18,26 +19,125 @@ export type PipelineDraft = {
|
||||
editorMode: EditorMode;
|
||||
};
|
||||
|
||||
export const NODE_LABELS: Record<string, string> = {
|
||||
"source.inline": "Inline source",
|
||||
"source.reference": "Connector source",
|
||||
filter: "Filter rows",
|
||||
select: "Select columns",
|
||||
aggregate: "Aggregate",
|
||||
sort: "Sort rows",
|
||||
limit: "Limit rows",
|
||||
output: "Output"
|
||||
};
|
||||
const input = [{ id: "input", label: "Input", required: true, multiple: false, minimum_connections: 1 }];
|
||||
const output = [{ id: "output", label: "Output", required: true, multiple: false, minimum_connections: 1 }];
|
||||
|
||||
export const PALETTE_NODE_TYPES = [
|
||||
"source.inline",
|
||||
"filter",
|
||||
"select",
|
||||
"aggregate",
|
||||
"sort",
|
||||
"limit",
|
||||
"output"
|
||||
] as const;
|
||||
export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
|
||||
nodeType("source.inline", "load", "Load", "Inline data", "Enter a small JSON table.", "braces", [], output, {
|
||||
source_name: "inline_source",
|
||||
rows: []
|
||||
}),
|
||||
nodeType(
|
||||
"source.reference",
|
||||
"load",
|
||||
"Load",
|
||||
"Connector source",
|
||||
"Load a governed tabular source.",
|
||||
"database",
|
||||
[],
|
||||
output,
|
||||
{ source_ref: "", source_name: "connector_source", expected_fingerprint: "" }
|
||||
),
|
||||
nodeType(
|
||||
"combine.union",
|
||||
"combine",
|
||||
"Combine",
|
||||
"Append rows",
|
||||
"Append two or more inputs.",
|
||||
"combine",
|
||||
[{ id: "input", label: "Inputs", required: true, multiple: true, minimum_connections: 2 }],
|
||||
output,
|
||||
{ mode: "all" }
|
||||
),
|
||||
nodeType(
|
||||
"combine.join",
|
||||
"combine",
|
||||
"Combine",
|
||||
"Join tables",
|
||||
"Match two inputs using key columns.",
|
||||
"git-merge",
|
||||
[
|
||||
{ id: "left", label: "Left", required: true, multiple: false, minimum_connections: 1 },
|
||||
{ id: "right", label: "Right", required: true, multiple: false, minimum_connections: 1 }
|
||||
],
|
||||
output,
|
||||
{ join_type: "inner", left_keys: [""], right_keys: [""], right_prefix: "right_" }
|
||||
),
|
||||
nodeType("filter", "filter", "Filter", "Filter rows", "Keep matching rows.", "filter", input, output, {
|
||||
column: "",
|
||||
operator: "eq",
|
||||
value: ""
|
||||
}),
|
||||
nodeType(
|
||||
"distinct",
|
||||
"filter",
|
||||
"Filter",
|
||||
"Remove duplicates",
|
||||
"Keep one row per key.",
|
||||
"list-filter",
|
||||
input,
|
||||
output,
|
||||
{ columns: [] }
|
||||
),
|
||||
nodeType(
|
||||
"select",
|
||||
"transform",
|
||||
"Transform",
|
||||
"Select columns",
|
||||
"Choose and rename fields.",
|
||||
"columns-3",
|
||||
input,
|
||||
output,
|
||||
{ fields: [{ column: "", alias: "" }] }
|
||||
),
|
||||
nodeType(
|
||||
"derive",
|
||||
"transform",
|
||||
"Transform",
|
||||
"Derive column",
|
||||
"Create a constrained calculated field.",
|
||||
"variable",
|
||||
input,
|
||||
output,
|
||||
{ target_column: "", operation: "copy", source_columns: [""], separator: " " }
|
||||
),
|
||||
nodeType(
|
||||
"aggregate",
|
||||
"transform",
|
||||
"Transform",
|
||||
"Aggregate",
|
||||
"Group and summarize rows.",
|
||||
"sigma",
|
||||
input,
|
||||
output,
|
||||
{ group_by: [], aggregates: [{ function: "count", column: "*", alias: "row_count" }] }
|
||||
),
|
||||
nodeType(
|
||||
"sort",
|
||||
"transform",
|
||||
"Transform",
|
||||
"Sort rows",
|
||||
"Order rows by fields.",
|
||||
"arrow-up-down",
|
||||
input,
|
||||
output,
|
||||
{ fields: [{ column: "", direction: "asc" }] }
|
||||
),
|
||||
nodeType("limit", "transform", "Transform", "Limit rows", "Keep the first rows.", "list-end", input, output, {
|
||||
count: 100
|
||||
}),
|
||||
nodeType(
|
||||
"output",
|
||||
"output",
|
||||
"Output",
|
||||
"Preview output",
|
||||
"Expose the terminal table.",
|
||||
"panel-top-open",
|
||||
input,
|
||||
[],
|
||||
{}
|
||||
)
|
||||
];
|
||||
|
||||
export function draftFromPipeline(pipeline: Pipeline): PipelineDraft {
|
||||
return {
|
||||
@@ -152,26 +252,45 @@ export function draftFingerprint(draft: PipelineDraft | null): string {
|
||||
});
|
||||
}
|
||||
|
||||
export function newNode(type: string, position: { x: number; y: number }): PipelineGraphNode {
|
||||
export function newNode(
|
||||
type: string,
|
||||
position: { x: number; y: number },
|
||||
library: NodeTypeDefinition[] = FALLBACK_NODE_LIBRARY
|
||||
): PipelineGraphNode {
|
||||
const id = `${type.replace(".", "-")}-${crypto.randomUUID()}`;
|
||||
const config: Record<string, unknown> = defaultNodeConfig(type);
|
||||
const definition = library.find((item) => item.type === type);
|
||||
const config = structuredClone(definition?.default_config ?? {});
|
||||
return {
|
||||
id,
|
||||
type,
|
||||
label: NODE_LABELS[type] ?? type,
|
||||
label: definition?.label ?? type,
|
||||
position,
|
||||
config
|
||||
};
|
||||
}
|
||||
|
||||
function defaultNodeConfig(type: string): Record<string, unknown> {
|
||||
if (type === "source.inline") return { source_name: "new_source", rows: [] };
|
||||
if (type === "filter") return { column: "", operator: "eq", value: "" };
|
||||
if (type === "select") return { fields: [{ column: "", alias: "" }] };
|
||||
if (type === "aggregate") {
|
||||
return { group_by: [], aggregates: [{ function: "count", column: "*", alias: "row_count" }] };
|
||||
}
|
||||
if (type === "sort") return { fields: [{ column: "", direction: "asc" }] };
|
||||
if (type === "limit") return { count: 100 };
|
||||
return {};
|
||||
function nodeType(
|
||||
type: string,
|
||||
category: NodeTypeDefinition["category"],
|
||||
categoryLabel: string,
|
||||
label: string,
|
||||
description: string,
|
||||
icon: string,
|
||||
inputPorts: NodeTypeDefinition["input_ports"],
|
||||
outputPorts: NodeTypeDefinition["output_ports"],
|
||||
defaultConfig: Record<string, unknown>
|
||||
): NodeTypeDefinition {
|
||||
return {
|
||||
type,
|
||||
category,
|
||||
category_label: categoryLabel,
|
||||
label,
|
||||
description,
|
||||
icon,
|
||||
input_ports: inputPorts,
|
||||
output_ports: outputPorts,
|
||||
config_fields: [],
|
||||
default_config: defaultConfig,
|
||||
sql_support: ["combine.union", "combine.join", "distinct", "derive"].includes(type) ? "partial" : "full"
|
||||
};
|
||||
}
|
||||
|
||||
34
webui/src/features/dataflow/nodeIcons.ts
Normal file
34
webui/src/features/dataflow/nodeIcons.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import {
|
||||
ArrowUpDown,
|
||||
Braces,
|
||||
Columns3,
|
||||
Combine,
|
||||
Database,
|
||||
Filter,
|
||||
GitMerge,
|
||||
ListEnd,
|
||||
ListFilter,
|
||||
PanelTopOpen,
|
||||
Sigma,
|
||||
Variable,
|
||||
type LucideIcon
|
||||
} from "lucide-react";
|
||||
|
||||
const icons: Record<string, LucideIcon> = {
|
||||
"arrow-up-down": ArrowUpDown,
|
||||
braces: Braces,
|
||||
"columns-3": Columns3,
|
||||
combine: Combine,
|
||||
database: Database,
|
||||
filter: Filter,
|
||||
"git-merge": GitMerge,
|
||||
"list-end": ListEnd,
|
||||
"list-filter": ListFilter,
|
||||
"panel-top-open": PanelTopOpen,
|
||||
sigma: Sigma,
|
||||
variable: Variable
|
||||
};
|
||||
|
||||
export function dataflowNodeIcon(icon: string): LucideIcon {
|
||||
return icons[icon] ?? Braces;
|
||||
}
|
||||
@@ -240,7 +240,7 @@
|
||||
.dataflow-editor {
|
||||
flex: 1 1 auto;
|
||||
display: grid;
|
||||
grid-template-columns: 170px minmax(0, 1fr) minmax(260px, 310px);
|
||||
grid-template-columns: 210px minmax(0, 1fr) minmax(260px, 310px);
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
@@ -297,10 +297,32 @@
|
||||
.dataflow-palette-items {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
height: calc(100% - 44px);
|
||||
padding: 8px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.dataflow-palette-group {
|
||||
display: grid;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.dataflow-palette-group + .dataflow-palette-group {
|
||||
margin-top: 6px;
|
||||
padding-top: 8px;
|
||||
border-top: var(--border-line);
|
||||
}
|
||||
|
||||
.dataflow-palette-group h3 {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
padding: 3px 8px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.dataflow-palette-items button {
|
||||
display: grid;
|
||||
grid-template-columns: 18px minmax(0, 1fr) 14px;
|
||||
@@ -425,6 +447,17 @@
|
||||
border-left-color: #b7791f;
|
||||
}
|
||||
|
||||
.dataflow-node-combine-union,
|
||||
.dataflow-node-combine-join {
|
||||
min-height: 64px;
|
||||
border-left-color: #2f7d6d;
|
||||
}
|
||||
|
||||
.dataflow-node-distinct {
|
||||
border-left-color: #b7791f;
|
||||
}
|
||||
|
||||
.dataflow-node-derive,
|
||||
.dataflow-node-aggregate {
|
||||
border-left-color: #76569b;
|
||||
}
|
||||
@@ -507,6 +540,24 @@
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.dataflow-node-port-label {
|
||||
position: absolute;
|
||||
left: 8px;
|
||||
max-width: 42px;
|
||||
overflow: hidden;
|
||||
color: var(--muted);
|
||||
font-size: 8px;
|
||||
line-height: 1;
|
||||
pointer-events: none;
|
||||
text-overflow: ellipsis;
|
||||
transform: translateY(-50%);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.dataflow-node-combine-join .dataflow-node-icon {
|
||||
margin-left: 31px;
|
||||
}
|
||||
|
||||
.dataflow-inspector-fields {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
@@ -546,6 +597,27 @@
|
||||
color: var(--danger-text);
|
||||
}
|
||||
|
||||
.dataflow-source-dialog {
|
||||
width: min(720px, calc(100vw - 32px));
|
||||
}
|
||||
|
||||
.dataflow-source-dialog-fields {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.dataflow-source-dialog-fields .alert,
|
||||
.dataflow-source-dialog-format,
|
||||
.dataflow-source-dialog-fields .form-field:last-child {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.dataflow-source-dialog-fields .dataflow-json-editor {
|
||||
width: 100%;
|
||||
min-height: 260px;
|
||||
}
|
||||
|
||||
.dataflow-sql-workbench {
|
||||
display: flex;
|
||||
height: 100%;
|
||||
@@ -651,9 +723,12 @@
|
||||
background: var(--primary-soft);
|
||||
}
|
||||
|
||||
.dataflow-preview-truncated {
|
||||
.dataflow-preview-summary {
|
||||
position: sticky;
|
||||
bottom: 0;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px 16px;
|
||||
padding: 6px 10px;
|
||||
border-top: var(--border-line);
|
||||
background: var(--panel-header);
|
||||
@@ -754,7 +829,7 @@
|
||||
}
|
||||
|
||||
.dataflow-editor {
|
||||
grid-template-columns: 150px minmax(0, 1fr) 270px;
|
||||
grid-template-columns: 180px minmax(0, 1fr) 270px;
|
||||
}
|
||||
|
||||
.dataflow-editor.is-sql {
|
||||
@@ -839,6 +914,23 @@
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.dataflow-palette-group {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.dataflow-palette-group + .dataflow-palette-group {
|
||||
margin-top: 0;
|
||||
padding-top: 0;
|
||||
padding-left: 6px;
|
||||
border-top: 0;
|
||||
border-left: var(--border-line);
|
||||
}
|
||||
|
||||
.dataflow-palette-group h3 {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.dataflow-palette-items button {
|
||||
min-width: 128px;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user