GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dataflow): model production SQL flow patterns

Browse Source
This commit is contained in:
zemion
2026-07-30 18:19:18 +02:00
parent 0664570607
commit e091042948
26 changed files with 2169 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui/src/features/dataflow/NodeInspector.tsx
+133 -8
View File
@@ -38,7 +38,9 @@ export default function NodeInspector({
}: NodeInspectorProps) {
const [rowsText, setRowsText] = useState("");
const [aggregateText, setAggregateText] = useState("");
const [calculationText, setCalculationText] = useState("");
const [sortText, setSortText] = useState("");
const [rankSortText, setRankSortText] = useState("");
const [rulesText, setRulesText] = useState("");
const [parametersText, setParametersText] = useState("");
const [subflowGraphText, setSubflowGraphText] = useState("");
@@ -47,7 +49,9 @@ export default function NodeInspector({
useEffect(() => {
setRowsText(node ? JSON.stringify(node.config.rows ?? [], null, 2) : "");
setAggregateText(node ? aggregatesToText(node.config.aggregates) : "");
setCalculationText(node ? calculationsToText(node.config.calculations) : "");
setSortText(node ? sortFieldsToText(node.config.fields) : "");
setRankSortText(node ? sortFieldsToText(node.config.order_by) : "");
setRulesText(node ? JSON.stringify(node.config.rules ?? [], null, 2) : "");
setParametersText(node ? JSON.stringify(node.config.parameters ?? {}, null, 2) : "");
setSubflowGraphText(node ? JSON.stringify(node.config.graph ?? {}, null, 2) : "");
@@ -103,6 +107,26 @@ export default function NodeInspector({
}
};
const commitCalculations = () => {
try {
const parsed = calculationsFromText(calculationText);
setLocalError("");
updateConfig({ calculations: parsed });
} catch (error) {
setLocalError(error instanceof Error ? error.message : "Calculations could not be parsed.");
}
};
const commitRankSort = () => {
try {
const parsed = sortFieldsFromText(rankSortText);
setLocalError("");
updateConfig({ order_by: parsed });
} catch (error) {
setLocalError(error instanceof Error ? error.message : "Rank order could not be parsed.");
}
};
const commitJsonConfig = (
field: string,
value: string,
@@ -318,6 +342,8 @@ export default function NodeInspector({
<option value="left">All left rows</option>
<option value="right">All right rows</option>
<option value="full">All rows</option>
<option value="semi">Left rows with a match</option>
<option value="anti">Left rows without a match</option>
</select>
</FormField>
<FormField label="Left keys">
@@ -334,13 +360,15 @@ export default function NodeInspector({
disabled={readOnly}
/>
</FormField>
<FormField label="Right-column prefix">
<input
value={textValue(node.config.right_prefix)}
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
disabled={readOnly}
/>
</FormField>
{!["semi", "anti"].includes(textValue(node.config.join_type)) ? (
<FormField label="Right-column prefix">
<input
value={textValue(node.config.right_prefix)}
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
disabled={readOnly}
/>
</FormField>
) : null}
</>
) : null}
{node.type === "select" ? (
@@ -453,6 +481,18 @@ export default function NodeInspector({
</FormField>
</>
) : null}
{node.type === "calculate" ? (
<FormField label="Calculated columns">
<textarea
className="dataflow-expression-editor"
value={calculationText}
onChange={(event) => setCalculationText(event.target.value)}
onBlur={commitCalculations}
spellCheck={false}
disabled={readOnly}
/>
</FormField>
) : null}
{node.type === "convert" ? (
<>
<FormField label="Source column">
@@ -550,6 +590,45 @@ export default function NodeInspector({
/>
</FormField>
) : null}
{node.type === "window.rank" ? (
<>
<FormField label="Method">
<select
value={textValue(node.config.method) || "row_number"}
onChange={(event) => updateConfig({ method: event.target.value })}
disabled={readOnly}
>
<option value="row_number">Row number</option>
<option value="rank">Rank with gaps</option>
<option value="dense_rank">Dense rank</option>
</select>
</FormField>
<FormField label="Output column">
<input
value={textValue(node.config.target_column)}
onChange={(event) => updateConfig({ target_column: event.target.value })}
disabled={readOnly}
/>
</FormField>
<FormField label="Partition by">
<input
value={stringList(node.config.partition_by).join(", ")}
onChange={(event) => updateConfig({ partition_by: commaList(event.target.value) })}
disabled={readOnly}
/>
</FormField>
<FormField label="Order by">
<textarea
className="dataflow-expression-editor"
value={rankSortText}
onChange={(event) => setRankSortText(event.target.value)}
onBlur={commitRankSort}
spellCheck={false}
disabled={readOnly}
/>
</FormField>
</>
) : null}
{node.type === "limit" ? (
<FormField label="Maximum rows">
<input
@@ -719,6 +798,51 @@ function selectFieldsFromText(value: string): Array<{ column: string; alias: str
});
}
function calculationsToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
if (!isRecord(item)) return "";
const target = textValue(item.target_column);
const expression = textValue(item.expression);
const resultType = textValue(item.result_type) || "unknown";
const typeSuffix = resultType === "unknown" ? "" : `:${resultType}`;
return target && expression ? `${target}${typeSuffix} = ${expression}` : "";
}).filter(Boolean).join("\n");
}
function calculationsFromText(
value: string
): Array<{ target_column: string; expression: string; result_type: string }> {
const lines = value.split("\n").map((line) => line.trim()).filter(Boolean);
if (!lines.length) throw new Error("Add at least one calculated column.");
const supportedTypes = new Set([
"unknown",
"string",
"integer",
"number",
"boolean",
"date",
"datetime"
]);
return lines.map((line) => {
const separator = line.indexOf("=");
if (separator < 1) throw new Error(`Invalid calculation: ${line}`);
const declaration = line.slice(0, separator).trim();
const expression = line.slice(separator + 1).trim();
const [targetColumn, declaredType = "unknown"] = declaration
.split(":", 2)
.map((item) => item.trim());
if (!targetColumn || !expression || !supportedTypes.has(declaredType)) {
throw new Error(`Invalid calculation: ${line}`);
}
return {
target_column: targetColumn,
expression,
result_type: declaredType
};
});
}
function aggregatesToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
@@ -745,7 +869,8 @@ function sortFieldsToText(value: unknown): string {
if (!Array.isArray(value)) return "";
return value.map((item) => {
if (!isRecord(item)) return "";
return `${textValue(item.column)} ${textValue(item.direction) || "asc"}`;
const column = textValue(item.column);
return column ? `${column} ${textValue(item.direction) || "asc"}` : "";
}).filter(Boolean).join("\n");
}
webui/src/features/dataflow/model.ts
+31
View File
@@ -140,6 +140,21 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
output,
{ target_column: "", expression: "", result_type: "unknown" }
),
nodeType(
"calculate",
"transform",
"Transform",
"Calculate columns",
"Create or replace several ordered calculated fields.",
"calculator",
input,
output,
{
calculations: [
{ target_column: "", expression: "", result_type: "unknown" }
]
}
),
nodeType(
"convert",
"transform",
@@ -184,6 +199,22 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
output,
{ fields: [{ column: "", direction: "asc" }] }
),
nodeType(
"window.rank",
"transform",
"Transform",
"Rank rows",
"Number or rank rows within partitions.",
"list-ordered",
input,
output,
{
method: "row_number",
target_column: "row_number",
partition_by: [],
order_by: [{ column: "", direction: "asc" }]
}
),
nodeType("limit", "transform", "Transform", "Limit rows", "Keep the first rows.", "list-end", input, output, {
count: 100
}),
webui/src/features/dataflow/nodeIcons.ts
+4
View File
@@ -3,6 +3,7 @@ import {
BadgeCheck,
Boxes,
Braces,
Calculator,
Columns3,
Combine,
Database,
@@ -11,6 +12,7 @@ import {
ListEnd,
ListFilter,
ListChecks,
ListOrdered,
PanelTopOpen,
Replace,
ReplaceAll,
@@ -26,6 +28,7 @@ const icons: Record<string, LucideIcon> = {
"badge-check": BadgeCheck,
boxes: Boxes,
braces: Braces,
calculator: Calculator,
"columns-3": Columns3,
combine: Combine,
database: Database,
@@ -34,6 +37,7 @@ const icons: Record<string, LucideIcon> = {
"list-end": ListEnd,
"list-filter": ListFilter,
"list-checks": ListChecks,
"list-ordered": ListOrdered,
"panel-top-open": PanelTopOpen,
replace: Replace,
"replace-all": ReplaceAll,
webui/tsconfig.json
+2 -1
View File
@@ -26,7 +26,8 @@
"@xyflow/react": ["../../govoplan-core/webui/node_modules/@xyflow/react/dist/esm/index.d.ts"],
"lucide-react": ["../../govoplan-core/webui/node_modules/lucide-react/dist/lucide-react.d.ts"],
"react": ["../../govoplan-core/webui/node_modules/@types/react/index.d.ts"],
"react/jsx-runtime": ["../../govoplan-core/webui/node_modules/@types/react/jsx-runtime.d.ts"]
"react/jsx-runtime": ["../../govoplan-core/webui/node_modules/@types/react/jsx-runtime.d.ts"],
"react-router-dom": ["../../govoplan-core/webui/node_modules/react-router-dom/dist/index.d.ts"]
}
},
"include": ["src"]