GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Initialize governed Dataflow module

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 01:28:17 +02:00
commit 223c008ff3
44 changed files with 6204 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
webui/package.json Normal file
View File

@@ -0,0 +1,34 @@
{
"name": "@govoplan/dataflow-webui",
"version": "0.1.14",
"private": true,
"type": "module",
"main": "src/index.ts",
"module": "src/index.ts",
"types": "src/index.ts",
"exports": {
".": {
"types": "./src/index.ts",
"import": "./src/index.ts"
},
"./styles/dataflow.css": "./src/styles/dataflow.css"
},
"scripts": {
"typecheck": "tsc --noEmit",
"test:structure": "node scripts/test-dataflow-page-structure.mjs"
},
"peerDependencies": {
"@govoplan/core-webui": "^0.1.14",
"@xyflow/react": "^12.11.2",
"lucide-react": "^1.23.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.1",
"typescript": "^5.7.2"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
"optional": true
}
}
}

25
webui/scripts/test-dataflow-page-structure.mjs Normal file
View File

@@ -0,0 +1,25 @@
import { readFileSync } from "node:fs";
const page = readFileSync(new URL("../src/features/dataflow/DataflowPage.tsx", import.meta.url), "utf8");
const canvas = readFileSync(new URL("../src/features/dataflow/DataflowCanvas.tsx", import.meta.url), "utf8");
const css = readFileSync(new URL("../src/styles/dataflow.css", import.meta.url), "utf8");
const checks = [
[page.includes("<DataflowCanvas"), "graph canvas"],
[page.includes("Apply SQL"), "SQL workbench"],
[page.includes("<NodeInspector"), "node inspector"],
[page.includes("previewDataflowPipeline"), "preview action"],
[page.includes("useUnsavedDraftGuard"), "unsaved-change guard"],
[canvas.includes("application/x-govoplan-dataflow-node"), "palette drop handling"],
[canvas.includes("isValidConnection"), "edge validation"],
[canvas.includes("connectionRadius={32}"), "visible connection drop radius"],
[css.includes("height: calc(100vh - 115px)"), "full-height workspace"],
[css.includes(".dataflow-preview-table-wrap"), "bounded preview scrolling"]
];
const missing = checks.filter(([present]) => !present).map(([, label]) => label);
if (missing.length) {
throw new Error(`Dataflow UI structure is missing: ${missing.join(", ")}`);
}
console.log("Dataflow page structure checks passed.");

187
webui/src/api/dataflow.ts Normal file
View File

@@ -0,0 +1,187 @@
import { apiFetch, type ApiSettings } from "@govoplan/core-webui";
export type PipelineStatus = "draft" | "active" | "archived";
export type EditorMode = "graph" | "sql";
export type GraphPosition = { x: number; y: number };
export type PipelineGraphNode = {
id: string;
type: string;
label: string;
position: GraphPosition;
config: Record<string, unknown>;
};
export type PipelineGraphEdge = {
id: string;
source: string;
target: string;
source_port?: string;
target_port?: string;
};
export type PipelineGraph = {
schema_version: 1;
nodes: PipelineGraphNode[];
edges: PipelineGraphEdge[];
};
export type DataflowDiagnostic = {
severity: "info" | "warning" | "error";
code: string;
message: string;
node_id?: string | null;
field?: string | null;
};
export type PipelineRevision = {
id: string;
revision: number;
schema_version: number;
graph: PipelineGraph;
sql_text?: string | null;
editor_mode: EditorMode;
content_hash: string;
created_by?: string | null;
created_at: string;
};
export type Pipeline = {
id: string;
tenant_id: string;
name: string;
description?: string | null;
status: PipelineStatus;
current_revision: number;
created_by?: string | null;
updated_by?: string | null;
created_at: string;
updated_at: string;
revision: PipelineRevision;
};
export type PipelinePayload = {
name: string;
description?: string | null;
status: PipelineStatus;
graph: PipelineGraph;
sql_text?: string | null;
editor_mode: EditorMode;
};
export type PipelineValidation = {
valid: boolean;
graph?: PipelineGraph | null;
sql_text?: string | null;
diagnostics: DataflowDiagnostic[];
};
export type PreviewColumn = {
name: string;
type: string;
nullable: boolean;
};
export type NodePreviewDiagnostic = {
node_id: string;
status: "succeeded" | "failed" | "skipped";
input_rows: number;
output_rows: number;
duration_ms: number;
columns: PreviewColumn[];
messages: string[];
};
export type PipelinePreview = {
run_id?: string | null;
pipeline_id?: string | null;
revision?: number | null;
status: "succeeded" | "failed";
columns: PreviewColumn[];
rows: Record<string, unknown>[];
total_rows: number;
truncated: boolean;
diagnostics: DataflowDiagnostic[];
node_diagnostics: NodePreviewDiagnostic[];
definition_hash: string;
executor_version: string;
};
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
return response.pipelines;
}
export function createDataflowPipeline(
settings: ApiSettings,
payload: PipelinePayload
): Promise<Pipeline> {
return apiFetch<Pipeline>(settings, "/api/v1/dataflow/pipelines", {
method: "POST",
body: JSON.stringify(payload)
});
}
export function updateDataflowPipeline(
settings: ApiSettings,
pipelineId: string,
payload: PipelinePayload & { expected_revision: number }
): Promise<Pipeline> {
return apiFetch<Pipeline>(settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`, {
method: "PUT",
body: JSON.stringify(payload)
});
}
export function deleteDataflowPipeline(settings: ApiSettings, pipelineId: string): Promise<{ deleted: boolean }> {
return apiFetch<{ deleted: boolean }>(
settings,
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`,
{ method: "DELETE" }
);
}
export function validateDataflowPipeline(
settings: ApiSettings,
payload: { graph?: PipelineGraph; sql_text?: string; source_nodes?: PipelineGraphNode[] }
): Promise<PipelineValidation> {
return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/validate", {
method: "POST",
body: JSON.stringify(payload)
});
}
export function compileDataflowSql(
settings: ApiSettings,
payload: { graph?: PipelineGraph; sql_text: string; source_nodes?: PipelineGraphNode[] }
): Promise<PipelineValidation> {
return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/sql/compile", {
method: "POST",
body: JSON.stringify(payload)
});
}
export function renderDataflowSql(
settings: ApiSettings,
graph: PipelineGraph
): Promise<PipelineValidation> {
return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/sql/render", {
method: "POST",
body: JSON.stringify({ graph })
});
}
export function previewDataflowPipeline(
settings: ApiSettings,
payload: {
pipeline_id?: string;
revision?: number;
graph?: PipelineGraph;
sql_text?: string;
source_nodes?: PipelineGraphNode[];
row_limit?: number;
}
): Promise<PipelinePreview> {
return apiFetch<PipelinePreview>(settings, "/api/v1/dataflow/preview", {
method: "POST",
body: JSON.stringify(payload)
});
}

210
webui/src/features/dataflow/DataflowCanvas.tsx Normal file
View File

@@ -0,0 +1,210 @@
import { useMemo, useState, type DragEvent } from "react";
import {
addEdge,
applyEdgeChanges,
applyNodeChanges,
Background,
BackgroundVariant,
ConnectionLineType,
Controls,
MiniMap,
ReactFlow,
type Connection,
type Edge,
type ReactFlowInstance
} from "@xyflow/react";
import type {
DataflowDiagnostic,
NodePreviewDiagnostic,
PipelineGraph,
PipelineGraphNode
} from "../../api/dataflow";
import DataflowNode, { type DataflowFlowNode } from "./DataflowNode";
import { newNode } from "./model";
const nodeTypes = { dataflow: DataflowNode };
type DataflowCanvasProps = {
graph: PipelineGraph;
diagnostics: DataflowDiagnostic[];
nodeDiagnostics: NodePreviewDiagnostic[];
selectedNodeId: string | null;
readOnly: boolean;
onGraphChange: (graph: PipelineGraph) => void;
onSelectNode: (nodeId: string | null) => void;
};
export default function DataflowCanvas({
graph,
diagnostics,
nodeDiagnostics,
selectedNodeId,
readOnly,
onGraphChange,
onSelectNode
}: DataflowCanvasProps) {
const [instance, setInstance] = useState<ReactFlowInstance<DataflowFlowNode, Edge> | null>(null);
const errorNodeIds = useMemo(
() => new Set(diagnostics.filter((item) => item.severity === "error" && item.node_id).map((item) => item.node_id)),
[diagnostics]
);
const rowCounts = useMemo(
() => new Map(nodeDiagnostics.map((item) => [item.node_id, item.output_rows])),
[nodeDiagnostics]
);
const nodes = useMemo<DataflowFlowNode[]>(
() => graph.nodes.map((node) => ({
id: node.id,
type: "dataflow",
position: node.position,
selected: node.id === selectedNodeId,
data: {
label: node.label,
transformType: node.type,
config: node.config,
hasError: errorNodeIds.has(node.id),
outputRows: rowCounts.get(node.id)
}
})),
[errorNodeIds, graph.nodes, rowCounts, selectedNodeId]
);
const edges = useMemo<Edge[]>(
() => graph.edges.map((edge) => ({
id: edge.id,
source: edge.source,
target: edge.target,
type: "smoothstep",
className: "dataflow-edge"
})),
[graph.edges]
);
const updateNodes = (nextNodes: DataflowFlowNode[]) => {
const ids = new Set(nextNodes.map((node) => node.id));
const nextGraphNodes = nextNodes.map((flowNode) => {
const current = graph.nodes.find((node) => node.id === flowNode.id);
if (!current) {
throw new Error(`Unknown Dataflow node: ${flowNode.id}`);
}
return { ...current, position: flowNode.position };
});
onGraphChange({
...graph,
nodes: nextGraphNodes,
edges: graph.edges.filter((edge) => ids.has(edge.source) && ids.has(edge.target))
});
};
const updateEdges = (nextEdges: Edge[]) => {
onGraphChange({
...graph,
edges: nextEdges.map((edge) => ({
id: edge.id,
source: edge.source,
target: edge.target,
source_port: edge.sourceHandle ?? "output",
target_port: edge.targetHandle ?? "input"
}))
});
};
const isValidConnection = (connection: Connection | Edge): boolean => {
if (readOnly || !connection.source || !connection.target || connection.source === connection.target) return false;
const source = graph.nodes.find((node) => node.id === connection.source);
const target = graph.nodes.find((node) => node.id === connection.target);
if (!source || !target || source.type === "output" || target.type.startsWith("source.")) return false;
const targetAlreadyConnected = graph.edges.some(
(edge) => edge.target === connection.target && edge.source !== connection.source
);
return !targetAlreadyConnected;
};
const onConnect = (connection: Connection) => {
if (!isValidConnection(connection)) return;
const next = addEdge(
{
...connection,
id: `edge-${connection.source}-${connection.target}-${crypto.randomUUID()}`,
type: "smoothstep"
},
edges
);
updateEdges(next);
};
const onDrop = (event: DragEvent<HTMLDivElement>) => {
event.preventDefault();
if (readOnly || !instance) return;
const type = event.dataTransfer.getData("application/x-govoplan-dataflow-node");
if (!type) return;
const position = instance.screenToFlowPosition({ x: event.clientX, y: event.clientY });
const node = newNode(type, position);
onGraphChange({ ...graph, nodes: [...graph.nodes, node] });
onSelectNode(node.id);
};
return (
<div
className="dataflow-canvas"
onDragOver={(event) => {
if (event.dataTransfer.types.includes("application/x-govoplan-dataflow-node")) {
event.preventDefault();
event.dataTransfer.dropEffect = "copy";
}
}}
onDrop={onDrop}
>
<ReactFlow<DataflowFlowNode, Edge>
nodes={nodes}
edges={edges}
nodeTypes={nodeTypes}
onInit={setInstance}
onNodesChange={(changes) => {
if (readOnly) return;
updateNodes(applyNodeChanges(changes, nodes));
}}
onEdgesChange={(changes) => {
if (readOnly) return;
updateEdges(applyEdgeChanges(changes, edges));
}}
onConnect={onConnect}
isValidConnection={isValidConnection}
onNodeClick={(_event, node) => onSelectNode(node.id)}
onPaneClick={() => onSelectNode(null)}
nodesDraggable={!readOnly}
nodesConnectable={!readOnly}
elementsSelectable
deleteKeyCode={readOnly ? null : ["Backspace", "Delete"]}
connectionLineType={ConnectionLineType.SmoothStep}
connectionLineStyle={{ stroke: "var(--accent)", strokeWidth: 3 }}
connectionRadius={32}
fitView
fitViewOptions={{ padding: 0.22, maxZoom: 1.25 }}
minZoom={0.25}
maxZoom={1.8}
>
<Background variant={BackgroundVariant.Dots} gap={20} size={1.3} />
<MiniMap
pannable
zoomable
nodeStrokeWidth={2}
nodeColor={(node) => node.data.hasError ? "var(--danger)" : "var(--accent)"}
/>
<Controls showInteractive={false} />
</ReactFlow>
{!graph.nodes.length ? (
<div className="dataflow-canvas-empty">Drop a source here</div>
) : null}
</div>
);
}
export function updateGraphNode(
graph: PipelineGraph,
updatedNode: PipelineGraphNode
): PipelineGraph {
return {
...graph,
nodes: graph.nodes.map((node) => node.id === updatedNode.id ? updatedNode : node)
};
}

87
webui/src/features/dataflow/DataflowNode.tsx Normal file
View File

@@ -0,0 +1,87 @@
import type { ComponentType } from "react";
import {
Braces,
Columns3,
Database,
Filter,
ListEnd,
Sigma,
SortAsc,
Unplug
} from "lucide-react";
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
export type DataflowNodeData = {
label: string;
transformType: string;
config: Record<string, unknown>;
hasError?: boolean;
outputRows?: number;
};
export type DataflowFlowNode = Node<DataflowNodeData, "dataflow">;
const iconByType: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
"source.inline": Braces,
"source.reference": Database,
filter: Filter,
select: Columns3,
aggregate: Sigma,
sort: SortAsc,
limit: ListEnd,
output: Unplug
};
export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowNode>) {
const Icon = iconByType[data.transformType] ?? Braces;
const isSource = data.transformType.startsWith("source.");
const isOutput = data.transformType === "output";
return (
<div
className={[
"dataflow-node",
`dataflow-node-${data.transformType.replace(".", "-")}`,
selected ? "is-selected" : "",
data.hasError ? "has-error" : ""
].filter(Boolean).join(" ")}
>
{!isSource ? (
<Handle
type="target"
position={Position.Left}
className="dataflow-node-handle dataflow-node-handle-input"
/>
) : null}
<span className="dataflow-node-icon" aria-hidden="true">
<Icon size={17} strokeWidth={1.8} />
</span>
<span className="dataflow-node-copy">
<strong>{data.label}</strong>
<small>{transformLabel(data.transformType)}</small>
</span>
{typeof data.outputRows === "number" ? (
<span className="dataflow-node-count">{data.outputRows}</span>
) : null}
{!isOutput ? (
<Handle
type="source"
position={Position.Right}
className="dataflow-node-handle dataflow-node-handle-output"
/>
) : null}
</div>
);
}
function transformLabel(type: string): string {
if (type === "source.inline") return "Inline source";
if (type === "source.reference") return "Connector source";
if (type === "filter") return "Filter";
if (type === "select") return "Projection";
if (type === "aggregate") return "Aggregation";
if (type === "sort") return "Sort";
if (type === "limit") return "Limit";
if (type === "output") return "Output";
return type;
}

775
webui/src/features/dataflow/DataflowPage.tsx Normal file
View File

@@ -0,0 +1,775 @@
import {
useCallback,
useEffect,
useMemo,
useState,
type ComponentType,
type DragEvent
} from "react";
import {
Braces,
CheckCircle2,
Code2,
Columns3,
Database,
Filter,
ListEnd,
Network,
Play,
Plus,
RefreshCw,
RotateCcw,
Save,
Sigma,
SortAsc,
Trash2,
TriangleAlert
} from "lucide-react";
import {
Button,
ConfirmDialog,
DismissibleAlert,
IconButton,
LoadingFrame,
SegmentedControl,
StatusBadge,
hasScope,
isApiError,
useUnsavedChanges,
useUnsavedDraftGuard,
type ApiSettings,
type AuthInfo
} from "@govoplan/core-webui";
import { ReactFlowProvider } from "@xyflow/react";
import {
compileDataflowSql,
createDataflowPipeline,
deleteDataflowPipeline,
listDataflowPipelines,
previewDataflowPipeline,
renderDataflowSql,
updateDataflowPipeline,
validateDataflowPipeline,
type DataflowDiagnostic,
type EditorMode,
type NodePreviewDiagnostic,
type Pipeline,
type PipelineGraphNode,
type PipelinePreview
} from "../../api/dataflow";
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
import NodeInspector from "./NodeInspector";
import {
NODE_LABELS,
PALETTE_NODE_TYPES,
draftFingerprint,
draftFromPipeline,
newNode,
pipelinePayload,
sampleDraft,
sourceNodes,
type PipelineDraft
} from "./model";
type ResultTab = "preview" | "diagnostics";
const paletteIcons: Record<string, ComponentType<{ size?: number; strokeWidth?: number }>> = {
"source.inline": Braces,
filter: Filter,
select: Columns3,
aggregate: Sigma,
sort: SortAsc,
limit: ListEnd,
output: Network
};
export default function DataflowPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo }) {
const { requestNavigation, requestDiscard } = useUnsavedChanges();
const [pipelines, setPipelines] = useState<Pipeline[]>([]);
const [draft, setDraft] = useState<PipelineDraft | null>(null);
const [savedDraft, setSavedDraft] = useState<PipelineDraft | null>(null);
const [selectedNodeId, setSelectedNodeId] = useState<string | null>(null);
const [search, setSearch] = useState("");
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [working, setWorking] = useState(false);
const [error, setError] = useState("");
const [success, setSuccess] = useState("");
const [diagnostics, setDiagnostics] = useState<DataflowDiagnostic[]>([]);
const [nodeDiagnostics, setNodeDiagnostics] = useState<NodePreviewDiagnostic[]>([]);
const [preview, setPreview] = useState<PipelinePreview | null>(null);
const [resultOpen, setResultOpen] = useState(false);
const [resultTab, setResultTab] = useState<ResultTab>("preview");
const [deleteOpen, setDeleteOpen] = useState(false);
const canWrite = hasScope(auth, "dataflow:pipeline:write") || hasScope(auth, "dataflow:pipeline:admin");
const canRun = hasScope(auth, "dataflow:pipeline:run") || hasScope(auth, "dataflow:pipeline:admin");
const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft);
const selectedNode = useMemo(
() => draft?.graph.nodes.find((node) => node.id === selectedNodeId) ?? null,
[draft, selectedNodeId]
);
const filteredPipelines = useMemo(() => {
const query = search.trim().toLocaleLowerCase();
if (!query) return pipelines;
return pipelines.filter((pipeline) =>
`${pipeline.name} ${pipeline.description ?? ""} ${pipeline.status}`.toLocaleLowerCase().includes(query)
);
}, [pipelines, search]);
const loadPipelines = useCallback(async (preferredId?: string | null) => {
setLoading(true);
setError("");
try {
const items = await listDataflowPipelines(settings);
setPipelines(items);
const selected = items.find((item) => item.id === (preferredId ?? draft?.id)) ?? items[0];
if (selected) {
const next = draftFromPipeline(selected);
setDraft(next);
setSavedDraft(structuredClone(next));
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
} else if (!draft?.id) {
setDraft(null);
setSavedDraft(null);
setSelectedNodeId(null);
}
} catch (loadError) {
setError(apiErrorMessage(loadError));
} finally {
setLoading(false);
}
}, [draft?.id, settings]);
useEffect(() => {
void loadPipelines();
}, []);
const discardDraft = useCallback(() => {
if (savedDraft) {
const reset = structuredClone(savedDraft);
setDraft(reset);
setSelectedNodeId(reset.graph.nodes[0]?.id ?? null);
} else {
setDraft(null);
setSelectedNodeId(null);
}
setDiagnostics([]);
setNodeDiagnostics([]);
setPreview(null);
setError("");
setSuccess("");
}, [savedDraft]);
const saveDraft = useCallback(async (): Promise<boolean> => {
if (!draft || !canWrite || !draft.name.trim()) {
setError(!draft?.name.trim() ? "Pipeline name is required." : "You cannot save this pipeline.");
return false;
}
setSaving(true);
setError("");
setSuccess("");
try {
const payload = pipelinePayload(draft);
const saved = draft.id && draft.currentRevision
? await updateDataflowPipeline(settings, draft.id, {
...payload,
expected_revision: draft.currentRevision
})
: await createDataflowPipeline(settings, payload);
const next = draftFromPipeline(saved);
setDraft(next);
setSavedDraft(structuredClone(next));
setPipelines((current) => [saved, ...current.filter((item) => item.id !== saved.id)]);
setSelectedNodeId((current) => current && next.graph.nodes.some((node) => node.id === current)
? current
: next.graph.nodes[0]?.id ?? null);
setDiagnostics([]);
setSuccess(`Saved revision ${saved.current_revision}.`);
return true;
} catch (saveError) {
setError(apiErrorMessage(saveError));
return false;
} finally {
setSaving(false);
}
}, [canWrite, draft, settings]);
useUnsavedDraftGuard({
dirty,
title: "Unsaved pipeline",
message: "Save or discard the pipeline changes before leaving this workspace.",
onSave: saveDraft,
onDiscard: discardDraft
});
useEffect(() => {
const beforeUnload = (event: BeforeUnloadEvent) => {
if (!dirty) return;
event.preventDefault();
event.returnValue = "";
};
window.addEventListener("beforeunload", beforeUnload);
return () => window.removeEventListener("beforeunload", beforeUnload);
}, [dirty]);
const selectPipeline = (pipeline: Pipeline) => {
requestNavigation(() => {
const next = draftFromPipeline(pipeline);
setDraft(next);
setSavedDraft(structuredClone(next));
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
setDiagnostics([]);
setNodeDiagnostics([]);
setPreview(null);
setError("");
setSuccess("");
});
};
const createNew = () => {
requestNavigation(() => {
const next = sampleDraft();
setDraft(next);
setSavedDraft(null);
setSelectedNodeId(next.graph.nodes[0]?.id ?? null);
setDiagnostics([]);
setNodeDiagnostics([]);
setPreview(null);
setError("");
setSuccess("");
});
};
const updateDraft = (patch: Partial<PipelineDraft>) => {
setDraft((current) => current ? { ...current, ...patch } : current);
setSuccess("");
};
const updateGraph = (graph: PipelineDraft["graph"]) => {
updateDraft({ graph });
setDiagnostics([]);
setNodeDiagnostics([]);
};
const validate = async () => {
if (!draft) return;
setWorking(true);
setError("");
setSuccess("");
try {
const response = await validateDataflowPipeline(settings, draft.editorMode === "sql"
? { graph: draft.graph, sql_text: draft.sqlText, source_nodes: sourceNodes(draft.graph) }
: { graph: draft.graph });
setDiagnostics(response.diagnostics);
if (response.valid) setSuccess("Pipeline definition is valid.");
setResultOpen(true);
setResultTab("diagnostics");
} catch (validationError) {
setError(apiErrorMessage(validationError));
} finally {
setWorking(false);
}
};
const applySql = async (switchToGraph = false): Promise<boolean> => {
if (!draft) return false;
setWorking(true);
setError("");
setSuccess("");
try {
const response = await compileDataflowSql(settings, {
graph: draft.graph,
sql_text: draft.sqlText,
source_nodes: sourceNodes(draft.graph)
});
setDiagnostics(response.diagnostics);
setResultOpen(true);
setResultTab("diagnostics");
if (!response.valid || !response.graph) return false;
updateDraft({
graph: response.graph,
sqlText: response.sql_text ?? draft.sqlText,
editorMode: switchToGraph ? "graph" : "sql"
});
setSelectedNodeId(response.graph.nodes[0]?.id ?? null);
setSuccess("SQL compiled into the pipeline graph.");
return true;
} catch (compileError) {
setError(apiErrorMessage(compileError));
return false;
} finally {
setWorking(false);
}
};
const changeEditorMode = async (mode: EditorMode) => {
if (!draft || mode === draft.editorMode) return;
if (mode === "sql") {
setWorking(true);
setError("");
try {
const response = await renderDataflowSql(settings, draft.graph);
setDiagnostics(response.diagnostics);
if (!response.valid || !response.sql_text) {
setResultOpen(true);
setResultTab("diagnostics");
return;
}
updateDraft({ editorMode: "sql", sqlText: response.sql_text });
} catch (renderError) {
setError(apiErrorMessage(renderError));
} finally {
setWorking(false);
}
return;
}
await applySql(true);
};
const runPreview = async () => {
if (!draft || !canRun) return;
setWorking(true);
setError("");
setSuccess("");
try {
const response = await previewDataflowPipeline(settings, !dirty && draft.id && draft.currentRevision
? { pipeline_id: draft.id, revision: draft.currentRevision, row_limit: 100 }
: {
graph: draft.graph,
sql_text: draft.editorMode === "sql" ? draft.sqlText : undefined,
source_nodes: sourceNodes(draft.graph),
row_limit: 100
});
setPreview(response);
setDiagnostics(response.diagnostics);
setNodeDiagnostics(response.node_diagnostics);
setResultOpen(true);
setResultTab(response.status === "succeeded" ? "preview" : "diagnostics");
if (response.status === "succeeded") {
setSuccess(`Preview produced ${response.total_rows} row${response.total_rows === 1 ? "" : "s"}.`);
}
} catch (previewError) {
setError(apiErrorMessage(previewError));
} finally {
setWorking(false);
}
};
const removePipeline = async () => {
if (!draft?.id) return;
setSaving(true);
setError("");
try {
await deleteDataflowPipeline(settings, draft.id);
const remaining = pipelines.filter((item) => item.id !== draft.id);
setPipelines(remaining);
const next = remaining[0] ? draftFromPipeline(remaining[0]) : null;
setDraft(next);
setSavedDraft(next ? structuredClone(next) : null);
setSelectedNodeId(next?.graph.nodes[0]?.id ?? null);
setDeleteOpen(false);
setPreview(null);
setDiagnostics([]);
} catch (deleteError) {
setError(apiErrorMessage(deleteError));
} finally {
setSaving(false);
}
};
const addNode = (type: string) => {
if (!draft || !canWrite || uniqueNodeExists(draft, type)) return;
const node = newNode(type, {
x: 120 + draft.graph.nodes.length * 70,
y: 100 + (draft.graph.nodes.length % 4) * 80
});
updateGraph({ ...draft.graph, nodes: [...draft.graph.nodes, node] });
setSelectedNodeId(node.id);
};
return (
<main className="dataflow-page">
<div className="dataflow-shell">
<aside className="dataflow-pipeline-panel" aria-label="Data pipelines">
<div className="dataflow-panel-toolbar">
<strong>Pipelines</strong>
<span className="dataflow-toolbar-actions">
<IconButton
label="Refresh pipelines"
icon={<RefreshCw size={16} />}
variant="ghost"
onClick={() => requestNavigation(() => void loadPipelines(draft?.id))}
disabled={loading}
/>
<IconButton
label="New pipeline"
icon={<Plus size={17} />}
variant="primary"
onClick={createNew}
disabled={!canWrite}
/>
</span>
</div>
<div className="dataflow-pipeline-search">
<input
type="search"
value={search}
onChange={(event) => setSearch(event.target.value)}
placeholder="Search pipelines"
aria-label="Search pipelines"
/>
</div>
<LoadingFrame loading={loading} label="Loading pipelines" className="dataflow-pipeline-list-frame">
<div className="dataflow-pipeline-list">
{filteredPipelines.map((pipeline) => (
<button
key={pipeline.id}
type="button"
className={pipeline.id === draft?.id ? "is-selected" : ""}
onClick={() => selectPipeline(pipeline)}
>
<span>
<strong>{pipeline.name}</strong>
<small>Revision {pipeline.current_revision}</small>
</span>
<StatusBadge status={pipeline.status} />
</button>
))}
{!loading && !filteredPipelines.length ? (
<div className="dataflow-pipeline-empty">
{search ? "No matching pipelines" : "No pipelines yet"}
</div>
) : null}
</div>
</LoadingFrame>
</aside>
<section className="dataflow-workspace">
{draft ? (
<>
<div className="dataflow-workspace-toolbar">
<div className="dataflow-identity-fields">
<input
className="dataflow-name-input"
value={draft.name}
onChange={(event) => updateDraft({ name: event.target.value })}
aria-label="Pipeline name"
disabled={!canWrite}
/>
<input
className="dataflow-description-input"
value={draft.description}
onChange={(event) => updateDraft({ description: event.target.value })}
placeholder="Description"
aria-label="Pipeline description"
disabled={!canWrite}
/>
<select
value={draft.status}
onChange={(event) => updateDraft({ status: event.target.value as PipelineDraft["status"] })}
aria-label="Pipeline status"
disabled={!canWrite}
>
<option value="draft">Draft</option>
<option value="active">Active</option>
<option value="archived">Archived</option>
</select>
</div>
<div className="dataflow-command-bar">
<SegmentedControl<EditorMode>
ariaLabel="Pipeline editor mode"
options={[
{ id: "graph", label: <><Network size={15} /> Graph</> },
{ id: "sql", label: <><Code2 size={15} /> SQL</> }
]}
value={draft.editorMode}
onChange={(mode) => void changeEditorMode(mode)}
disabled={working}
/>
<Button onClick={() => void validate()} disabled={working}>
<CheckCircle2 size={16} /> Validate
</Button>
<Button variant="primary" onClick={() => void runPreview()} disabled={working || !canRun}>
<Play size={16} /> Preview
</Button>
<IconButton
label="Discard changes"
icon={<RotateCcw size={16} />}
variant="ghost"
onClick={() => requestDiscard(() => undefined)}
disabled={!dirty || saving}
/>
{draft.id ? (
<IconButton
label="Delete pipeline"
icon={<Trash2 size={16} />}
variant="danger"
onClick={() => setDeleteOpen(true)}
disabled={!canWrite || saving}
/>
) : null}
<Button
variant="primary"
onClick={() => void saveDraft()}
disabled={saving || working || !dirty || !canWrite}
>
<Save size={16} /> {saving ? "Saving..." : "Save"}
</Button>
</div>
</div>
{(error || success) ? (
<div className="dataflow-alerts">
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null}
{success ? <DismissibleAlert tone="success" resetKey={success}>{success}</DismissibleAlert> : null}
</div>
) : null}
<div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}>
{draft.editorMode === "graph" ? (
<aside className="dataflow-palette" aria-label="Transform palette">
<div className="dataflow-panel-heading">
<strong>Transforms</strong>
</div>
<div className="dataflow-palette-items">
{PALETTE_NODE_TYPES.map((type) => {
const Icon = paletteIcons[type] ?? Database;
const disabled = !canWrite || uniqueNodeExists(draft, type);
return (
<button
key={type}
type="button"
draggable={!disabled}
disabled={disabled}
onDragStart={(event) => startPaletteDrag(event, type)}
onClick={() => addNode(type)}
>
<Icon size={16} />
<span>{NODE_LABELS[type]}</span>
<Plus size={14} className="dataflow-palette-add" />
</button>
);
})}
</div>
</aside>
) : null}
<section className="dataflow-editor-surface">
{draft.editorMode === "graph" ? (
<ReactFlowProvider>
<DataflowCanvas
graph={draft.graph}
diagnostics={diagnostics}
nodeDiagnostics={nodeDiagnostics}
selectedNodeId={selectedNodeId}
readOnly={!canWrite}
onGraphChange={updateGraph}
onSelectNode={setSelectedNodeId}
/>
</ReactFlowProvider>
) : (
<div className="dataflow-sql-workbench">
<div className="dataflow-sql-toolbar">
<span>Constrained Dataflow SQL</span>
<Button
variant="primary"
onClick={() => void applySql(false)}
disabled={working || !canWrite}
>
<Code2 size={16} /> Apply SQL
</Button>
</div>
<textarea
value={draft.sqlText}
onChange={(event) => updateDraft({ sqlText: event.target.value })}
spellCheck={false}
disabled={!canWrite}
aria-label="Dataflow SQL"
/>
</div>
)}
</section>
<NodeInspector
node={selectedNode}
readOnly={!canWrite}
onChange={(node: PipelineGraphNode) => updateGraph(updateGraphNode(draft.graph, node))}
onDelete={(nodeId) => {
updateGraph({
...draft.graph,
nodes: draft.graph.nodes.filter((node) => node.id !== nodeId),
edges: draft.graph.edges.filter((edge) => edge.source !== nodeId && edge.target !== nodeId)
});
setSelectedNodeId(null);
}}
/>
</div>
{resultOpen ? (
<ResultPanel
tab={resultTab}
onTabChange={setResultTab}
preview={preview}
diagnostics={diagnostics}
nodeDiagnostics={nodeDiagnostics}
onClose={() => setResultOpen(false)}
/>
) : null}
</>
) : (
<div className="dataflow-workspace-empty">
<Network size={30} />
<strong>No pipeline selected</strong>
<Button variant="primary" onClick={createNew} disabled={!canWrite}>
<Plus size={16} /> New pipeline
</Button>
</div>
)}
{working ? <div className="dataflow-working-indicator" role="status">Working...</div> : null}
</section>
</div>
<ConfirmDialog
open={deleteOpen}
title="Delete pipeline"
message={`Delete ${draft?.name ?? "this pipeline"} and all of its saved revisions and run evidence?`}
confirmLabel="Delete"
tone="danger"
busy={saving}
onCancel={() => setDeleteOpen(false)}
onConfirm={() => void removePipeline()}
/>
</main>
);
}
function ResultPanel({
tab,
onTabChange,
preview,
diagnostics,
nodeDiagnostics,
onClose
}: {
tab: ResultTab;
onTabChange: (tab: ResultTab) => void;
preview: PipelinePreview | null;
diagnostics: DataflowDiagnostic[];
nodeDiagnostics: NodePreviewDiagnostic[];
onClose: () => void;
}) {
return (
<section className="dataflow-results" aria-label="Pipeline results">
<div className="dataflow-results-toolbar">
<SegmentedControl<ResultTab>
ariaLabel="Result view"
options={[
{ id: "preview", label: `Preview${preview ? ` (${preview.total_rows})` : ""}` },
{ id: "diagnostics", label: `Diagnostics (${diagnostics.length + nodeDiagnostics.length})` }
]}
value={tab}
onChange={onTabChange}
/>
<Button variant="ghost" onClick={onClose}>Close</Button>
</div>
{tab === "preview" ? (
<PreviewTable preview={preview} />
) : (
<DiagnosticsPanel diagnostics={diagnostics} nodeDiagnostics={nodeDiagnostics} />
)}
</section>
);
}
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
if (!preview) return <div className="dataflow-results-empty">No preview has been run.</div>;
if (preview.status === "failed") return <div className="dataflow-results-empty">Preview failed.</div>;
return (
<div className="dataflow-preview-table-wrap">
<table className="dataflow-preview-table">
<thead>
<tr>
{preview.columns.map((column) => (
<th key={column.name}>
<span>{column.name}</span>
<small>{column.type}{column.nullable ? " · nullable" : ""}</small>
</th>
))}
</tr>
</thead>
<tbody>
{preview.rows.map((row, rowIndex) => (
<tr key={rowIndex}>
{preview.columns.map((column) => (
<td key={column.name}>{formatCell(row[column.name])}</td>
))}
</tr>
))}
</tbody>
</table>
{preview.truncated ? (
<div className="dataflow-preview-truncated">Showing {preview.rows.length} of {preview.total_rows} rows</div>
) : null}
</div>
);
}
function DiagnosticsPanel({
diagnostics,
nodeDiagnostics
}: {
diagnostics: DataflowDiagnostic[];
nodeDiagnostics: NodePreviewDiagnostic[];
}) {
if (!diagnostics.length && !nodeDiagnostics.length) {
return <div className="dataflow-results-empty">No diagnostics.</div>;
}
return (
<div className="dataflow-diagnostics-list">
{diagnostics.map((item, index) => (
<div key={`${item.code}-${item.node_id ?? "pipeline"}-${index}`} className={`is-${item.severity}`}>
{item.severity === "error" ? <TriangleAlert size={15} /> : <CheckCircle2 size={15} />}
<span>
<strong>{item.code}</strong>
<small>{item.message}</small>
</span>
{item.node_id ? <code>{item.node_id}</code> : null}
</div>
))}
{nodeDiagnostics.map((item) => (
<div key={item.node_id} className={`is-${item.status}`}>
<CheckCircle2 size={15} />
<span>
<strong>{item.node_id}</strong>
<small>{item.input_rows} in · {item.output_rows} out · {item.duration_ms.toFixed(2)} ms</small>
</span>
<code>{item.columns.length} columns</code>
</div>
))}
</div>
);
}
function uniqueNodeExists(draft: PipelineDraft, type: string): boolean {
if (type === "output") return draft.graph.nodes.some((node) => node.type === "output");
if (type.startsWith("source.")) return draft.graph.nodes.some((node) => node.type.startsWith("source."));
return false;
}
function startPaletteDrag(event: DragEvent<HTMLButtonElement>, type: string) {
event.dataTransfer.setData("application/x-govoplan-dataflow-node", type);
event.dataTransfer.effectAllowed = "copy";
}
function formatCell(value: unknown): string {
if (value === null || value === undefined) return "—";
if (typeof value === "object") return JSON.stringify(value);
return String(value);
}
function apiErrorMessage(error: unknown): string {
if (!isApiError(error)) return error instanceof Error ? error.message : "The request failed.";
try {
const parsed = JSON.parse(error.body) as { detail?: string | { message?: string } };
if (typeof parsed.detail === "string") return parsed.detail;
if (parsed.detail?.message) return parsed.detail.message;
} catch {
// Fall through to the API error message.
}
return error.message;
}

316
webui/src/features/dataflow/NodeInspector.tsx Normal file
View File

@@ -0,0 +1,316 @@
import { useEffect, useState } from "react";
import { Trash2 } from "lucide-react";
import { Button, DismissibleAlert, FormField } from "@govoplan/core-webui";
import type { PipelineGraphNode } from "../../api/dataflow";
type NodeInspectorProps = {
node: PipelineGraphNode | null;
readOnly: boolean;
onChange: (node: PipelineGraphNode) => void;
onDelete: (nodeId: string) => void;
};
export default function NodeInspector({ node, readOnly, onChange, onDelete }: NodeInspectorProps) {
const [rowsText, setRowsText] = useState("");
const [aggregateText, setAggregateText] = useState("");
const [sortText, setSortText] = useState("");
const [localError, setLocalError] = useState("");
useEffect(() => {
setRowsText(node ? JSON.stringify(node.config.rows ?? [], null, 2) : "");
setAggregateText(node ? aggregatesToText(node.config.aggregates) : "");
setSortText(node ? sortFieldsToText(node.config.fields) : "");
setLocalError("");
}, [node?.id]);
if (!node) {
return (
<aside className="dataflow-inspector" aria-label="Node inspector">
<div className="dataflow-panel-heading">
<strong>Inspector</strong>
</div>
<div className="dataflow-inspector-empty">No node selected</div>
</aside>
);
}
const updateConfig = (patch: Record<string, unknown>) => {
onChange({ ...node, config: { ...node.config, ...patch } });
};
const commitRows = () => {
try {
const parsed: unknown = JSON.parse(rowsText);
if (!Array.isArray(parsed) || parsed.some((item) => !isRecord(item))) {
throw new Error("Rows must be a JSON array of objects.");
}
setLocalError("");
updateConfig({ rows: parsed });
} catch (error) {
setLocalError(error instanceof Error ? error.message : "Rows could not be parsed.");
}
};
const commitAggregates = () => {
try {
const parsed = aggregatesFromText(aggregateText);
setLocalError("");
updateConfig({ aggregates: parsed });
} catch (error) {
setLocalError(error instanceof Error ? error.message : "Aggregates could not be parsed.");
}
};
const commitSort = () => {
try {
const parsed = sortFieldsFromText(sortText);
setLocalError("");
updateConfig({ fields: parsed });
} catch (error) {
setLocalError(error instanceof Error ? error.message : "Sort fields could not be parsed.");
}
};
return (
<aside className="dataflow-inspector" aria-label="Node inspector">
<div className="dataflow-panel-heading">
<span>
<strong>Inspector</strong>
<small>{node.type}</small>
</span>
<Button
variant="ghost"
className="dataflow-inspector-delete"
onClick={() => onDelete(node.id)}
disabled={readOnly}
aria-label="Delete node"
title="Delete node"
>
<Trash2 size={16} />
</Button>
</div>
<div className="dataflow-inspector-fields">
{localError ? (
<DismissibleAlert tone="danger" resetKey={localError}>
{localError}
</DismissibleAlert>
) : null}
<FormField label="Name">
<input
value={node.label}
onChange={(event) => onChange({ ...node, label: event.target.value })}
disabled={readOnly}
/>
</FormField>
{node.type.startsWith("source.") ? (
<FormField label="Logical source name">
<input
value={textValue(node.config.source_name)}
onChange={(event) => updateConfig({ source_name: event.target.value })}
disabled={readOnly}
/>
</FormField>
) : null}
{node.type === "source.inline" ? (
<FormField label="Rows">
<textarea
className="dataflow-json-editor"
value={rowsText}
onChange={(event) => setRowsText(event.target.value)}
onBlur={commitRows}
spellCheck={false}
disabled={readOnly}
/>
</FormField>
) : null}
{node.type === "filter" ? (
<>
<FormField label="Column">
<input
value={textValue(node.config.column)}
onChange={(event) => updateConfig({ column: event.target.value })}
disabled={readOnly}
/>
</FormField>
<FormField label="Operator">
<select
value={textValue(node.config.operator) || "eq"}
onChange={(event) => updateConfig({ operator: event.target.value })}
disabled={readOnly}
>
<option value="eq">equals</option>
<option value="ne">does not equal</option>
<option value="gt">greater than</option>
<option value="gte">greater than or equal</option>
<option value="lt">less than</option>
<option value="lte">less than or equal</option>
<option value="contains">contains</option>
<option value="is_null">is null</option>
<option value="not_null">is not null</option>
</select>
</FormField>
{!["is_null", "not_null"].includes(textValue(node.config.operator)) ? (
<FormField label="Value">
<input
value={displayScalar(node.config.value)}
onChange={(event) => updateConfig({ value: parseScalar(event.target.value) })}
disabled={readOnly}
/>
</FormField>
) : null}
</>
) : null}
{node.type === "select" ? (
<FormField label="Columns">
<input
value={selectFieldsToText(node.config.fields)}
onChange={(event) => updateConfig({ fields: selectFieldsFromText(event.target.value) })}
disabled={readOnly}
/>
</FormField>
) : null}
{node.type === "aggregate" ? (
<>
<FormField label="Group by">
<input
value={stringList(node.config.group_by).join(", ")}
onChange={(event) => updateConfig({ group_by: commaList(event.target.value) })}
disabled={readOnly}
/>
</FormField>
<FormField label="Aggregates">
<textarea
className="dataflow-expression-editor"
value={aggregateText}
onChange={(event) => setAggregateText(event.target.value)}
onBlur={commitAggregates}
spellCheck={false}
disabled={readOnly}
/>
</FormField>
</>
) : null}
{node.type === "sort" ? (
<FormField label="Sort fields">
<textarea
className="dataflow-expression-editor"
value={sortText}
onChange={(event) => setSortText(event.target.value)}
onBlur={commitSort}
spellCheck={false}
disabled={readOnly}
/>
</FormField>
) : null}
{node.type === "limit" ? (
<FormField label="Maximum rows">
<input
type="number"
min={1}
max={100000}
value={numberValue(node.config.count, 100)}
onChange={(event) => updateConfig({ count: Number(event.target.value) })}
disabled={readOnly}
/>
</FormField>
) : null}
</div>
</aside>
);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function textValue(value: unknown): string {
return typeof value === "string" ? value : "";
}
function numberValue(value: unknown, fallback: number): number {
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
}
function displayScalar(value: unknown): string {
if (value === null) return "null";
if (typeof value === "string") return value;
if (typeof value === "number" || typeof value === "boolean") return String(value);
return "";
}
function parseScalar(value: string): string | number | boolean | null {
const trimmed = value.trim();
if (trimmed === "null") return null;
if (trimmed === "true") return true;
if (trimmed === "false") return false;
if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) return Number(trimmed);
return value;
}
function stringList(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
}
function commaList(value: string): string[] {
return value.split(",").map((item) => item.trim()).filter(Boolean);
}
function selectFieldsToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
if (typeof item === "string") return item;
if (!isRecord(item)) return "";
const column = textValue(item.column);
const alias = textValue(item.alias);
return alias && alias !== column ? `${column} as ${alias}` : column;
}).filter(Boolean).join(", ");
}
function selectFieldsFromText(value: string): Array<{ column: string; alias: string }> {
return commaList(value).map((item) => {
const match = item.match(/^(.+?)\s+as\s+(.+)$/i);
const column = (match?.[1] ?? item).trim();
return { column, alias: (match?.[2] ?? column).trim() };
});
}
function aggregatesToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
if (!isRecord(item)) return "";
return `${textValue(item.function)}(${textValue(item.column) || "*"}) as ${textValue(item.alias)}`;
}).filter(Boolean).join("\n");
}
function aggregatesFromText(value: string): Array<Record<string, string>> {
const lines = value.split("\n").map((line) => line.trim()).filter(Boolean);
if (!lines.length) throw new Error("Add at least one aggregate.");
return lines.map((line) => {
const match = line.match(/^(count|sum|avg|min|max)\(([^)]+)\)\s+as\s+([A-Za-z_][A-Za-z0-9_]*)$/i);
if (!match) throw new Error(`Invalid aggregate: ${line}`);
return {
function: match[1].toLowerCase(),
column: match[2].trim(),
alias: match[3]
};
});
}
function sortFieldsToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
if (!isRecord(item)) return "";
return `${textValue(item.column)} ${textValue(item.direction) || "asc"}`;
}).filter(Boolean).join("\n");
}
function sortFieldsFromText(value: string): Array<Record<string, string>> {
const lines = value.split("\n").map((line) => line.trim()).filter(Boolean);
if (!lines.length) throw new Error("Add at least one sort field.");
return lines.map((line) => {
const match = line.match(/^(.+?)(?:\s+(asc|desc))?$/i);
const column = match?.[1]?.trim() ?? "";
if (!column) throw new Error(`Invalid sort field: ${line}`);
return { column, direction: (match?.[2] ?? "asc").toLowerCase() };
});
}

177
webui/src/features/dataflow/model.ts Normal file
View File

@@ -0,0 +1,177 @@
import type {
EditorMode,
Pipeline,
PipelineGraph,
PipelineGraphNode,
PipelinePayload,
PipelineStatus
} from "../../api/dataflow";
export type PipelineDraft = {
id: string | null;
currentRevision: number | null;
name: string;
description: string;
status: PipelineStatus;
graph: PipelineGraph;
sqlText: string;
editorMode: EditorMode;
};
export const NODE_LABELS: Record<string, string> = {
"source.inline": "Inline source",
"source.reference": "Connector source",
filter: "Filter rows",
select: "Select columns",
aggregate: "Aggregate",
sort: "Sort rows",
limit: "Limit rows",
output: "Output"
};
export const PALETTE_NODE_TYPES = [
"source.inline",
"filter",
"select",
"aggregate",
"sort",
"limit",
"output"
] as const;
export function draftFromPipeline(pipeline: Pipeline): PipelineDraft {
return {
id: pipeline.id,
currentRevision: pipeline.current_revision,
name: pipeline.name,
description: pipeline.description ?? "",
status: pipeline.status,
graph: structuredClone(pipeline.revision.graph),
sqlText: pipeline.revision.sql_text ?? "",
editorMode: pipeline.revision.editor_mode
};
}
export function sampleDraft(): PipelineDraft {
return {
id: null,
currentRevision: null,
name: "Monthly case overview",
description: "Filter open cases and aggregate their value by district.",
status: "draft",
editorMode: "graph",
sqlText: "",
graph: {
schema_version: 1,
nodes: [
{
id: "source",
type: "source.inline",
label: "Monthly cases",
position: { x: 60, y: 170 },
config: {
source_name: "monthly_cases",
rows: [
{ case_id: "A-1001", district: "North", status: "open", amount: 1200 },
{ case_id: "A-1002", district: "South", status: "closed", amount: 800 },
{ case_id: "A-1003", district: "North", status: "open", amount: 450 },
{ case_id: "A-1004", district: "West", status: "open", amount: 2100 },
{ case_id: "A-1005", district: "South", status: "open", amount: 650 }
]
}
},
{
id: "filter-open",
type: "filter",
label: "Only open cases",
position: { x: 300, y: 170 },
config: { column: "status", operator: "eq", value: "open" }
},
{
id: "aggregate-district",
type: "aggregate",
label: "Totals by district",
position: { x: 540, y: 170 },
config: {
group_by: ["district"],
aggregates: [
{ function: "count", column: "*", alias: "cases" },
{ function: "sum", column: "amount", alias: "total_amount" }
]
}
},
{
id: "sort-total",
type: "sort",
label: "Largest total first",
position: { x: 780, y: 170 },
config: { fields: [{ column: "total_amount", direction: "desc" }] }
},
{
id: "output",
type: "output",
label: "Preview output",
position: { x: 1020, y: 170 },
config: {}
}
],
edges: [
{ id: "source-filter", source: "source", target: "filter-open" },
{ id: "filter-aggregate", source: "filter-open", target: "aggregate-district" },
{ id: "aggregate-sort", source: "aggregate-district", target: "sort-total" },
{ id: "sort-output", source: "sort-total", target: "output" }
]
}
};
}
export function pipelinePayload(draft: PipelineDraft): PipelinePayload {
return {
name: draft.name.trim(),
description: draft.description.trim() || null,
status: draft.status,
graph: draft.graph,
sql_text: draft.sqlText.trim() || null,
editor_mode: draft.editorMode
};
}
export function sourceNodes(graph: PipelineGraph): PipelineGraphNode[] {
return graph.nodes.filter((node) => node.type.startsWith("source."));
}
export function draftFingerprint(draft: PipelineDraft | null): string {
if (!draft) return "";
return JSON.stringify({
name: draft.name,
description: draft.description,
status: draft.status,
graph: draft.graph,
sqlText: draft.sqlText,
editorMode: draft.editorMode
});
}
export function newNode(type: string, position: { x: number; y: number }): PipelineGraphNode {
const id = `${type.replace(".", "-")}-${crypto.randomUUID()}`;
const config: Record<string, unknown> = defaultNodeConfig(type);
return {
id,
type,
label: NODE_LABELS[type] ?? type,
position,
config
};
}
function defaultNodeConfig(type: string): Record<string, unknown> {
if (type === "source.inline") return { source_name: "new_source", rows: [] };
if (type === "filter") return { column: "", operator: "eq", value: "" };
if (type === "select") return { fields: [{ column: "", alias: "" }] };
if (type === "aggregate") {
return { group_by: [], aggregates: [{ function: "count", column: "*", alias: "row_count" }] };
}
if (type === "sort") return { fields: [{ column: "", direction: "asc" }] };
if (type === "limit") return { count: 100 };
return {};
}

2
webui/src/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export { dataflowModule as default, dataflowModule } from "./module";
export * from "./api/dataflow";

38
webui/src/module.ts Normal file
View File

@@ -0,0 +1,38 @@
import { createElement, lazy } from "react";
import type { PlatformWebModule } from "@govoplan/core-webui";
import "./styles/dataflow.css";
import "@xyflow/react/dist/style.css";
const DataflowPage = lazy(() => import("./features/dataflow/DataflowPage"));
const readScopes = ["dataflow:pipeline:read", "dataflow:pipeline:admin"];
export const dataflowModule: PlatformWebModule = {
id: "dataflow",
label: "Dataflow",
version: "0.1.14",
optionalDependencies: [
"access",
"audit",
"connectors",
"files",
"notifications",
"policy",
"reporting",
"risk_compliance",
"workflow"
],
navItems: [
{ to: "/dataflow", label: "Dataflow", iconName: "waypoints", anyOf: readScopes, order: 72 }
],
routes: [
{
path: "/dataflow",
anyOf: readScopes,
order: 72,
render: ({ settings, auth }) => createElement(DataflowPage, { settings, auth })
}
]
};
export default dataflowModule;

853
webui/src/styles/dataflow.css Normal file
View File

@@ -0,0 +1,853 @@
.dataflow-page {
position: relative;
height: calc(100vh - 115px);
min-width: 0;
min-height: 0;
padding: 0;
overflow: hidden;
color: var(--text);
background: var(--bg);
}
.dataflow-page *,
.dataflow-page *::before,
.dataflow-page *::after {
box-sizing: border-box;
}
.dataflow-shell {
display: grid;
grid-template-columns: minmax(250px, 300px) minmax(0, 1fr);
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
overflow: hidden;
border: var(--border-line);
background: var(--panel);
}
.dataflow-pipeline-panel,
.dataflow-workspace,
.dataflow-editor,
.dataflow-editor-surface,
.dataflow-canvas,
.dataflow-pipeline-list-frame {
min-width: 0;
min-height: 0;
}
.dataflow-pipeline-panel {
display: flex;
flex-direction: column;
overflow: hidden;
border-right: var(--border-line);
background: var(--panel-soft);
}
.dataflow-panel-toolbar,
.dataflow-workspace-toolbar,
.dataflow-results-toolbar,
.dataflow-sql-toolbar,
.dataflow-panel-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 10px;
flex: 0 0 auto;
border-bottom: var(--border-line);
background: var(--panel-header);
}
.dataflow-panel-toolbar {
min-height: 52px;
padding: 8px 10px 8px 14px;
}
.dataflow-toolbar-actions,
.dataflow-command-bar,
.dataflow-identity-fields,
.dataflow-results-toolbar .segmented-control {
display: flex;
align-items: center;
gap: 7px;
}
.dataflow-pipeline-search {
padding: 10px;
border-bottom: var(--border-line);
}
.dataflow-pipeline-search input {
min-height: 34px;
padding: 7px 9px;
}
.dataflow-pipeline-list-frame {
flex: 1 1 auto;
overflow: hidden;
}
.dataflow-pipeline-list {
height: 100%;
overflow: auto;
padding: 6px;
}
.dataflow-pipeline-list > button {
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
align-items: center;
gap: 8px;
width: 100%;
min-height: 56px;
border: 0;
border-radius: var(--radius-sm);
background: transparent;
color: var(--text);
cursor: pointer;
padding: 8px 9px;
text-align: left;
}
.dataflow-pipeline-list > button:hover,
.dataflow-pipeline-list > button:focus-visible {
background: var(--primary-soft);
outline: none;
}
.dataflow-pipeline-list > button.is-selected {
background: var(--primary-soft-strong);
box-shadow: inset 3px 0 0 var(--accent);
}
.dataflow-pipeline-list > button > span:first-child {
min-width: 0;
}
.dataflow-pipeline-list strong,
.dataflow-pipeline-list small {
display: block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.dataflow-pipeline-list strong {
color: var(--text-strong);
font-size: 13px;
}
.dataflow-pipeline-list small {
margin-top: 4px;
color: var(--muted);
font-size: 11px;
}
.dataflow-pipeline-list .status-badge {
max-width: 74px;
overflow: hidden;
text-overflow: ellipsis;
}
.dataflow-pipeline-empty,
.dataflow-inspector-empty,
.dataflow-results-empty {
display: grid;
place-items: center;
min-height: 100px;
color: var(--muted);
font-size: 13px;
text-align: center;
}
.dataflow-workspace {
position: relative;
display: flex;
flex-direction: column;
overflow: hidden;
background: var(--bg);
}
.dataflow-workspace-toolbar {
min-height: 58px;
padding: 8px 10px;
}
.dataflow-identity-fields {
min-width: 260px;
flex: 1 1 auto;
}
.dataflow-identity-fields input,
.dataflow-identity-fields select {
min-height: 34px;
padding: 7px 9px;
}
.dataflow-name-input {
max-width: 260px;
color: var(--text-strong);
font-weight: 700;
}
.dataflow-description-input {
min-width: 140px;
flex: 1 1 240px;
}
.dataflow-identity-fields select {
width: 104px;
flex: 0 0 104px;
}
.dataflow-command-bar {
justify-content: flex-end;
flex-wrap: wrap;
}
.dataflow-command-bar .btn,
.dataflow-sql-toolbar .btn {
display: inline-flex;
align-items: center;
gap: 6px;
min-height: 34px;
white-space: nowrap;
}
.dataflow-command-bar .segmented-control-option {
display: inline-flex;
align-items: center;
gap: 5px;
}
.dataflow-alerts {
position: absolute;
z-index: 12;
top: 66px;
right: 12px;
display: grid;
width: min(500px, calc(100% - 24px));
gap: 6px;
pointer-events: none;
}
.dataflow-alerts .alert {
pointer-events: auto;
box-shadow: var(--shadow-popover);
}
.dataflow-editor {
flex: 1 1 auto;
display: grid;
grid-template-columns: 170px minmax(0, 1fr) minmax(260px, 310px);
min-height: 0;
overflow: hidden;
}
.dataflow-editor.is-sql {
grid-template-columns: minmax(0, 1fr) minmax(260px, 310px);
}
.dataflow-palette,
.dataflow-inspector {
min-width: 0;
min-height: 0;
overflow: hidden;
background: var(--panel-soft);
}
.dataflow-palette {
border-right: var(--border-line);
}
.dataflow-inspector {
border-left: var(--border-line);
}
.dataflow-panel-heading {
min-height: 44px;
padding: 8px 10px 8px 12px;
}
.dataflow-panel-heading > span {
min-width: 0;
}
.dataflow-panel-heading strong,
.dataflow-panel-heading small {
display: block;
}
.dataflow-panel-heading strong {
color: var(--text-strong);
font-size: 13px;
}
.dataflow-panel-heading small {
overflow: hidden;
margin-top: 2px;
color: var(--muted);
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
font-size: 10px;
text-overflow: ellipsis;
white-space: nowrap;
}
.dataflow-palette-items {
display: grid;
gap: 4px;
padding: 8px;
overflow: auto;
}
.dataflow-palette-items button {
display: grid;
grid-template-columns: 18px minmax(0, 1fr) 14px;
align-items: center;
gap: 7px;
min-height: 38px;
border: 0;
border-radius: var(--radius-sm);
background: transparent;
color: var(--text);
cursor: grab;
font: inherit;
font-size: 12px;
padding: 7px 8px;
text-align: left;
}
.dataflow-palette-items button:hover:not(:disabled),
.dataflow-palette-items button:focus-visible:not(:disabled) {
background: var(--primary-soft);
color: var(--text-strong);
outline: none;
}
.dataflow-palette-items button:active:not(:disabled) {
cursor: grabbing;
}
.dataflow-palette-items button:disabled {
cursor: default;
opacity: .42;
}
.dataflow-palette-add {
color: var(--accent);
}
.dataflow-editor-surface {
position: relative;
overflow: hidden;
background: var(--bg);
}
.dataflow-canvas {
position: absolute;
inset: 0;
}
.dataflow-canvas .react-flow {
background: var(--bg);
}
.dataflow-canvas .react-flow__background {
color: var(--line-dark);
}
.dataflow-canvas .react-flow__controls,
.dataflow-canvas .react-flow__minimap {
overflow: hidden;
border: var(--border-line);
border-radius: var(--radius-sm);
background: var(--panel);
box-shadow: var(--shadow-xs);
}
.dataflow-canvas .react-flow__controls-button {
border-bottom: var(--border-line);
background: var(--panel);
color: var(--text);
}
.dataflow-canvas .react-flow__minimap-mask {
fill: color-mix(in srgb, var(--bg) 76%, transparent);
}
.dataflow-canvas .react-flow__edge-path {
stroke: var(--line-dark);
stroke-width: 2;
}
.dataflow-canvas .react-flow__edge.selected .react-flow__edge-path,
.dataflow-canvas .react-flow__edge:hover .react-flow__edge-path {
stroke: var(--accent);
stroke-width: 3;
}
.dataflow-canvas .react-flow__connection-path {
stroke: var(--accent);
stroke-width: 3;
}
.dataflow-canvas-empty {
position: absolute;
inset: 0;
display: grid;
place-items: center;
color: var(--muted);
pointer-events: none;
}
.dataflow-node {
position: relative;
display: flex;
align-items: center;
gap: 9px;
width: 180px;
min-height: 54px;
border: 1px solid var(--line-dark);
border-left: 4px solid #3d6f9e;
border-radius: 6px;
background: var(--panel);
box-shadow: var(--shadow-xs);
padding: 8px 10px;
}
.dataflow-node-source-inline,
.dataflow-node-source-reference {
border-left-color: #2f7d6d;
}
.dataflow-node-filter {
border-left-color: #b7791f;
}
.dataflow-node-aggregate {
border-left-color: #76569b;
}
.dataflow-node-sort {
border-left-color: #3d6f9e;
}
.dataflow-node-output {
border-left-color: #9d4e63;
}
.dataflow-node.is-selected {
border-color: var(--accent);
box-shadow: 0 0 0 2px color-mix(in srgb, var(--accent) 24%, transparent), var(--shadow-xs);
}
.dataflow-node.has-error {
border-color: var(--danger-text);
}
.dataflow-node-icon {
display: grid;
width: 28px;
height: 28px;
flex: 0 0 28px;
place-items: center;
border-radius: 5px;
background: var(--panel-soft);
color: var(--text-strong);
}
.dataflow-node-copy {
min-width: 0;
flex: 1 1 auto;
}
.dataflow-node-copy strong,
.dataflow-node-copy small {
display: block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.dataflow-node-copy strong {
color: var(--text-strong);
font-size: 12px;
}
.dataflow-node-copy small {
margin-top: 3px;
color: var(--muted);
font-size: 10px;
}
.dataflow-node-count {
min-width: 22px;
border-radius: 999px;
background: var(--primary-soft);
color: var(--text-strong);
font-size: 10px;
font-variant-numeric: tabular-nums;
padding: 3px 5px;
text-align: center;
}
.dataflow-node-handle {
width: 13px;
height: 13px;
border: 3px solid var(--panel);
background: var(--line-dark);
}
.dataflow-node-handle:hover,
.dataflow-node-handle.connectingto,
.dataflow-node-handle.valid {
width: 17px;
height: 17px;
background: var(--accent);
}
.dataflow-inspector-fields {
display: grid;
gap: 12px;
height: calc(100% - 44px);
overflow: auto;
padding: 12px;
}
.dataflow-inspector-fields .form-field {
min-width: 0;
}
.dataflow-inspector-fields input,
.dataflow-inspector-fields select,
.dataflow-inspector-fields textarea {
margin-top: 5px;
padding: 7px 8px;
font-size: 12px;
}
.dataflow-json-editor {
min-height: 190px;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
line-height: 1.45;
tab-size: 2;
}
.dataflow-expression-editor {
min-height: 78px;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
line-height: 1.45;
}
.dataflow-inspector-delete {
color: var(--danger-text);
}
.dataflow-sql-workbench {
display: flex;
height: 100%;
min-height: 0;
flex-direction: column;
}
.dataflow-sql-toolbar {
min-height: 44px;
padding: 7px 10px 7px 14px;
color: var(--text-strong);
font-size: 12px;
font-weight: 700;
}
.dataflow-sql-workbench > textarea {
flex: 1 1 auto;
min-height: 0;
resize: none;
border: 0;
border-radius: 0;
background: var(--bg);
color: var(--text-strong);
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
font-size: 14px;
line-height: 1.65;
padding: 20px;
tab-size: 2;
}
.dataflow-sql-workbench > textarea:focus {
outline: 2px solid color-mix(in srgb, var(--accent) 46%, transparent);
outline-offset: -2px;
}
.dataflow-results {
flex: 0 0 clamp(190px, 28vh, 320px);
display: flex;
min-height: 0;
flex-direction: column;
overflow: hidden;
border-top: var(--border-line-dark);
background: var(--panel);
}
.dataflow-results-toolbar {
min-height: 42px;
padding: 5px 9px;
}
.dataflow-results-toolbar .btn {
min-height: 30px;
}
.dataflow-preview-table-wrap,
.dataflow-diagnostics-list {
min-height: 0;
flex: 1 1 auto;
overflow: auto;
}
.dataflow-preview-table {
width: 100%;
min-width: max-content;
border-collapse: collapse;
font-size: 12px;
}
.dataflow-preview-table th,
.dataflow-preview-table td {
min-width: 130px;
max-width: 360px;
border-right: var(--border-line);
border-bottom: var(--border-line);
padding: 7px 10px;
overflow: hidden;
text-align: left;
text-overflow: ellipsis;
white-space: nowrap;
}
.dataflow-preview-table th {
position: sticky;
z-index: 2;
top: 0;
background: var(--panel-soft);
color: var(--text-strong);
}
.dataflow-preview-table th span,
.dataflow-preview-table th small {
display: block;
}
.dataflow-preview-table th small {
margin-top: 2px;
color: var(--muted);
font-size: 9px;
font-weight: 500;
}
.dataflow-preview-table tbody tr:hover {
background: var(--primary-soft);
}
.dataflow-preview-truncated {
position: sticky;
bottom: 0;
padding: 6px 10px;
border-top: var(--border-line);
background: var(--panel-header);
color: var(--muted);
font-size: 11px;
}
.dataflow-diagnostics-list {
padding: 6px;
}
.dataflow-diagnostics-list > div {
display: grid;
grid-template-columns: 18px minmax(0, 1fr) auto;
align-items: center;
gap: 8px;
min-height: 44px;
border-bottom: var(--border-line);
padding: 6px 8px;
}
.dataflow-diagnostics-list > div.is-error {
color: var(--danger-text);
}
.dataflow-diagnostics-list > div.is-warning {
color: var(--warning-deep);
}
.dataflow-diagnostics-list span {
min-width: 0;
}
.dataflow-diagnostics-list strong,
.dataflow-diagnostics-list small {
display: block;
}
.dataflow-diagnostics-list small {
margin-top: 2px;
color: var(--muted);
}
.dataflow-diagnostics-list code {
max-width: 220px;
overflow: hidden;
color: var(--muted);
font-size: 10px;
text-overflow: ellipsis;
white-space: nowrap;
}
.dataflow-workspace-empty {
display: grid;
place-content: center;
justify-items: center;
gap: 12px;
height: 100%;
color: var(--muted);
}
.dataflow-workspace-empty strong {
color: var(--text-strong);
}
.dataflow-workspace-empty .btn {
display: inline-flex;
align-items: center;
gap: 6px;
}
.dataflow-working-indicator {
position: absolute;
z-index: 20;
right: 12px;
bottom: 10px;
border: var(--border-line-dark);
border-radius: var(--radius-sm);
background: var(--panel);
box-shadow: var(--shadow-popover);
color: var(--text-strong);
font-size: 12px;
padding: 7px 10px;
}
@media (max-width: 1180px) {
.dataflow-shell {
grid-template-columns: 240px minmax(0, 1fr);
}
.dataflow-workspace-toolbar {
align-items: stretch;
flex-direction: column;
}
.dataflow-command-bar {
justify-content: flex-start;
}
.dataflow-editor {
grid-template-columns: 150px minmax(0, 1fr) 270px;
}
.dataflow-editor.is-sql {
grid-template-columns: minmax(0, 1fr) 270px;
}
}
@media (max-width: 900px) {
.dataflow-shell {
grid-template-columns: 210px minmax(0, 1fr);
}
.dataflow-editor {
grid-template-columns: 135px minmax(0, 1fr);
grid-template-rows: minmax(0, 1fr) minmax(170px, 32%);
}
.dataflow-editor.is-sql {
grid-template-columns: minmax(0, 1fr);
grid-template-rows: minmax(0, 1fr) minmax(170px, 32%);
}
.dataflow-inspector {
grid-column: 1 / -1;
border-top: var(--border-line);
border-left: 0;
}
.dataflow-inspector-fields {
grid-template-columns: repeat(3, minmax(0, 1fr));
height: calc(100% - 44px);
}
.dataflow-inspector-fields .alert,
.dataflow-inspector-fields .dataflow-json-editor {
grid-column: 1 / -1;
}
}
@media (max-width: 680px) {
.dataflow-page {
height: calc(100vh - 94px);
}
.dataflow-shell {
grid-template-columns: 1fr;
grid-template-rows: minmax(150px, 24%) minmax(0, 1fr);
}
.dataflow-pipeline-panel {
border-right: 0;
border-bottom: var(--border-line);
}
.dataflow-identity-fields,
.dataflow-command-bar {
flex-wrap: wrap;
}
.dataflow-name-input,
.dataflow-description-input {
max-width: none;
min-width: 180px;
}
.dataflow-editor {
grid-template-columns: 1fr;
grid-template-rows: auto minmax(0, 1fr) minmax(160px, 30%);
}
.dataflow-palette {
border-right: 0;
border-bottom: var(--border-line);
}
.dataflow-palette .dataflow-panel-heading {
display: none;
}
.dataflow-palette-items {
display: flex;
overflow-x: auto;
}
.dataflow-palette-items button {
min-width: 128px;
}
.dataflow-inspector-fields {
grid-template-columns: 1fr 1fr;
}
.dataflow-results {
flex-basis: 220px;
}
}

1
webui/src/vite-env.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
/// <reference path="../../../govoplan-core/webui/src/vite-env.d.ts" />

32
webui/tsconfig.json Normal file
View File

@@ -0,0 +1,32 @@
{
"compilerOptions": {
"target": "ES2020",
"useDefineForClassFields": true,
"lib": ["DOM", "DOM.Iterable", "ES2020"],
"allowJs": false,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"forceConsistentCasingInFileNames": true,
"module": "ESNext",
"moduleResolution": "Bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
"preserveSymlinks": true,
"baseUrl": ".",
"paths": {
"@govoplan/core-webui": ["../../govoplan-core/webui/src/index.ts"],
"@govoplan/core-webui/*": ["../../govoplan-core/webui/src/*"],
"@xyflow/react": ["../../govoplan-core/webui/node_modules/@xyflow/react/dist/esm/index.d.ts"],
"lucide-react": ["../../govoplan-core/webui/node_modules/lucide-react/dist/lucide-react.d.ts"],
"react": ["../../govoplan-core/webui/node_modules/@types/react/index.d.ts"],
"react/jsx-runtime": ["../../govoplan-core/webui/node_modules/@types/react/jsx-runtime.d.ts"]
}
},
"include": ["src"]
}