feat: add extensible operators and golden flows

2026-07-29 15:50:15 +02:00
parent 09c98087c5
commit 946202ef01
27 changed files with 3584 additions and 218 deletions
@@ -10,7 +10,7 @@ export type PipelineStatus = "draft" | "active" | "archived";
 export type DefinitionScopeType = "system" | "tenant" | "group" | "user";
 export type DefinitionKind = "flow" | "template";
 export type EditorMode = "graph" | "sql";
-export type NodeCategory = "load" | "combine" | "filter" | "transform" | "output";
+export type NodeCategory = "load" | "combine" | "filter" | "transform" | "quality" | "output";

 export type GraphPosition = { x: number; y: number };
 export type PipelineGraphNode = {
@@ -39,12 +39,18 @@ export default function NodeInspector({
  const [rowsText, setRowsText] = useState("");
  const [aggregateText, setAggregateText] = useState("");
  const [sortText, setSortText] = useState("");
+  const [rulesText, setRulesText] = useState("");
+  const [parametersText, setParametersText] = useState("");
+  const [subflowGraphText, setSubflowGraphText] = useState("");
  const [localError, setLocalError] = useState("");

  useEffect(() => {
    setRowsText(node ? JSON.stringify(node.config.rows ?? [], null, 2) : "");
    setAggregateText(node ? aggregatesToText(node.config.aggregates) : "");
    setSortText(node ? sortFieldsToText(node.config.fields) : "");
+    setRulesText(node ? JSON.stringify(node.config.rules ?? [], null, 2) : "");
+    setParametersText(node ? JSON.stringify(node.config.parameters ?? {}, null, 2) : "");
+    setSubflowGraphText(node ? JSON.stringify(node.config.graph ?? {}, null, 2) : "");
    setLocalError("");
  }, [node?.id]);

@@ -97,6 +103,26 @@ export default function NodeInspector({
    }
  };

+  const commitJsonConfig = (
+    field: string,
+    value: string,
+    expected: "array" | "object"
+  ) => {
+    try {
+      const parsed: unknown = JSON.parse(value);
+      if (
+        (expected === "array" && !Array.isArray(parsed)) ||
+        (expected === "object" && !isRecord(parsed))
+      ) {
+        throw new Error(`Value must be a JSON ${expected}.`);
+      }
+      setLocalError("");
+      updateConfig({ [field]: parsed });
+    } catch (error) {
+      setLocalError(error instanceof Error ? error.message : "JSON could not be parsed.");
+    }
+  };
+
  return (
    <aside className="dataflow-inspector" aria-label="Node inspector">
      <div className="dataflow-panel-heading">
@@ -247,6 +273,17 @@ export default function NodeInspector({
            ) : null}
          </>
        ) : null}
+        {node.type === "filter.expression" ? (
+          <FormField label="Expression">
+            <textarea
+              className="dataflow-expression-editor"
+              value={textValue(node.config.expression)}
+              onChange={(event) => updateConfig({ expression: event.target.value })}
+              spellCheck={false}
+              disabled={readOnly}
+            />
+          </FormField>
+        ) : null}
        {node.type === "distinct" ? (
          <FormField label="Key columns">
            <input
@@ -381,6 +418,126 @@ export default function NodeInspector({
            ) : null}
          </>
        ) : null}
+        {node.type === "expression" ? (
+          <>
+            <FormField label="Output column">
+              <input
+                value={textValue(node.config.target_column)}
+                onChange={(event) => updateConfig({ target_column: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Expression">
+              <textarea
+                className="dataflow-expression-editor"
+                value={textValue(node.config.expression)}
+                onChange={(event) => updateConfig({ expression: event.target.value })}
+                spellCheck={false}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Expected type">
+              <select
+                value={textValue(node.config.result_type) || "unknown"}
+                onChange={(event) => updateConfig({ result_type: event.target.value })}
+                disabled={readOnly}
+              >
+                <option value="unknown">Infer</option>
+                <option value="string">Text</option>
+                <option value="integer">Integer</option>
+                <option value="number">Number</option>
+                <option value="boolean">Boolean</option>
+                <option value="date">Date</option>
+                <option value="datetime">Date and time</option>
+              </select>
+            </FormField>
+          </>
+        ) : null}
+        {node.type === "convert" ? (
+          <>
+            <FormField label="Source column">
+              <input
+                value={textValue(node.config.source_column)}
+                onChange={(event) => updateConfig({ source_column: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Output column">
+              <input
+                value={textValue(node.config.target_column)}
+                onChange={(event) => updateConfig({ target_column: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Data type">
+              <select
+                value={textValue(node.config.target_type) || "string"}
+                onChange={(event) => updateConfig({ target_type: event.target.value })}
+                disabled={readOnly}
+              >
+                <option value="string">Text</option>
+                <option value="integer">Integer</option>
+                <option value="number">Number</option>
+                <option value="boolean">Boolean</option>
+                <option value="date">Date</option>
+                <option value="datetime">Date and time</option>
+              </select>
+            </FormField>
+            <FormField label="Conversion error">
+              <select
+                value={textValue(node.config.on_error) || "fail"}
+                onChange={(event) => updateConfig({ on_error: event.target.value })}
+                disabled={readOnly}
+              >
+                <option value="fail">Stop</option>
+                <option value="null">Use null</option>
+                <option value="keep">Keep original</option>
+              </select>
+            </FormField>
+          </>
+        ) : null}
+        {node.type === "replace" ? (
+          <>
+            <FormField label="Source column">
+              <input
+                value={textValue(node.config.source_column)}
+                onChange={(event) => updateConfig({ source_column: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Output column">
+              <input
+                value={textValue(node.config.target_column)}
+                onChange={(event) => updateConfig({ target_column: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Mode">
+              <select
+                value={textValue(node.config.mode) || "exact"}
+                onChange={(event) => updateConfig({ mode: event.target.value })}
+                disabled={readOnly}
+              >
+                <option value="exact">Exact value</option>
+                <option value="text">Text fragment</option>
+              </select>
+            </FormField>
+            <FormField label="Find">
+              <input
+                value={displayScalar(node.config.find)}
+                onChange={(event) => updateConfig({ find: parseScalar(event.target.value) })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Replacement">
+              <input
+                value={displayScalar(node.config.replacement)}
+                onChange={(event) => updateConfig({ replacement: parseScalar(event.target.value) })}
+                disabled={readOnly}
+              />
+            </FormField>
+          </>
+        ) : null}
        {node.type === "sort" ? (
          <FormField label="Sort fields">
            <textarea
@@ -405,6 +562,103 @@ export default function NodeInspector({
            />
          </FormField>
        ) : null}
+        {node.type === "quality.rules" ? (
+          <>
+            <FormField label="Rules">
+              <textarea
+                className="dataflow-json-editor"
+                value={rulesText}
+                onChange={(event) => setRulesText(event.target.value)}
+                onBlur={() => commitJsonConfig("rules", rulesText, "array")}
+                spellCheck={false}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Invalid rows">
+              <select
+                value={textValue(node.config.action) || "annotate"}
+                onChange={(event) => updateConfig({ action: event.target.value })}
+                disabled={readOnly}
+              >
+                <option value="annotate">Annotate</option>
+                <option value="drop">Drop</option>
+                <option value="fail">Stop</option>
+              </select>
+            </FormField>
+          </>
+        ) : null}
+        {node.type === "reconcile.compare" ? (
+          <>
+            <FormField label="Expected keys">
+              <input
+                value={stringList(node.config.left_keys).join(", ")}
+                onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Observed keys">
+              <input
+                value={stringList(node.config.right_keys).join(", ")}
+                onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Compared columns">
+              <input
+                value={comparisonFieldsToText(node.config.compare_columns)}
+                onChange={(event) => updateConfig({
+                  compare_columns: comparisonFieldsFromText(event.target.value)
+                })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Observed prefix">
+              <input
+                value={textValue(node.config.right_prefix)}
+                onChange={(event) => updateConfig({ right_prefix: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+          </>
+        ) : null}
+        {node.type === "subflow" ? (
+          <>
+            <FormField label="Template reference">
+              <input
+                value={textValue(node.config.template_ref)}
+                onChange={(event) => updateConfig({ template_ref: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Template version">
+              <input
+                value={textValue(node.config.template_version)}
+                onChange={(event) => updateConfig({ template_version: event.target.value })}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Parameters">
+              <textarea
+                className="dataflow-json-editor"
+                value={parametersText}
+                onChange={(event) => setParametersText(event.target.value)}
+                onBlur={() => commitJsonConfig("parameters", parametersText, "object")}
+                spellCheck={false}
+                disabled={readOnly}
+              />
+            </FormField>
+            <FormField label="Pinned graph">
+              <textarea
+                className="dataflow-json-editor"
+                value={subflowGraphText}
+                onChange={(event) => setSubflowGraphText(event.target.value)}
+                onBlur={() => commitJsonConfig("graph", subflowGraphText, "object")}
+                spellCheck={false}
+                disabled={readOnly}
+              />
+            </FormField>
+          </>
+        ) : null}
      </div>
    </aside>
  );
@@ -505,3 +759,21 @@ function sortFieldsFromText(value: string): Array<Record<string, string>> {
    return { column, direction: (match?.[2] ?? "asc").toLowerCase() };
  });
 }
+
+function comparisonFieldsToText(value: unknown): string {
+  if (!Array.isArray(value)) return "";
+  return value.map((item) => {
+    if (typeof item === "string") return item;
+    if (!isRecord(item)) return "";
+    const left = textValue(item.left) || textValue(item.column);
+    const right = textValue(item.right) || left;
+    return left === right ? left : `${left}=${right}`;
+  }).filter(Boolean).join(", ");
+}
+
+function comparisonFieldsFromText(value: string): Array<{ left: string; right: string }> {
+  return commaList(value).map((item) => {
+    const [left, right] = item.split("=", 2).map((part) => part.trim());
+    return { left, right: right || left };
+  });
+}
@@ -85,6 +85,17 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
    operator: "eq",
    value: ""
  }),
+  nodeType(
+    "filter.expression",
+    "filter",
+    "Filter",
+    "Expression filter",
+    "Keep rows matching a typed expression.",
+    "list-checks",
+    input,
+    output,
+    { expression: "true" }
+  ),
  nodeType(
    "distinct",
    "filter",
@@ -118,6 +129,39 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
    output,
    { target_column: "", operation: "copy", source_columns: [""], separator: " " }
  ),
+  nodeType(
+    "expression",
+    "transform",
+    "Transform",
+    "Expression",
+    "Create a typed calculated field.",
+    "function-square",
+    input,
+    output,
+    { target_column: "", expression: "", result_type: "unknown" }
+  ),
+  nodeType(
+    "convert",
+    "transform",
+    "Transform",
+    "Convert type",
+    "Convert a field with explicit error handling.",
+    "replace-all",
+    input,
+    output,
+    { source_column: "", target_column: "", target_type: "string", on_error: "fail" }
+  ),
+  nodeType(
+    "replace",
+    "transform",
+    "Transform",
+    "Replace values",
+    "Replace exact values or text fragments.",
+    "replace",
+    input,
+    output,
+    { source_column: "", target_column: "", mode: "exact", find: "", replacement: "" }
+  ),
  nodeType(
    "aggregate",
    "transform",
@@ -143,6 +187,47 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
  nodeType("limit", "transform", "Transform", "Limit rows", "Keep the first rows.", "list-end", input, output, {
    count: 100
  }),
+  nodeType(
+    "quality.rules",
+    "quality",
+    "Quality",
+    "Quality rules",
+    "Validate and annotate, drop, or reject rows.",
+    "badge-check",
+    input,
+    output,
+    { rules: [{ id: "required", column: "", operator: "not_null" }], action: "annotate" }
+  ),
+  nodeType(
+    "reconcile.compare",
+    "quality",
+    "Quality",
+    "Reconcile tables",
+    "Compare keyed expected and observed rows.",
+    "scan-search",
+    [
+      { id: "left", label: "Expected", required: true, multiple: false, minimum_connections: 1 },
+      { id: "right", label: "Observed", required: true, multiple: false, minimum_connections: 1 }
+    ],
+    output,
+    { left_keys: [""], right_keys: [""], compare_columns: [], right_prefix: "observed_" }
+  ),
+  nodeType(
+    "subflow",
+    "transform",
+    "Transform",
+    "Reusable subflow",
+    "Run a pinned parameterized template snapshot.",
+    "boxes",
+    input,
+    output,
+    {
+      template_ref: "",
+      template_version: "",
+      parameters: {},
+      graph: { schema_version: 1, nodes: [], edges: [] }
+    }
+  ),
  nodeType(
    "output",
    "output",
@@ -329,6 +414,11 @@ function nodeType(
    output_ports: outputPorts,
    config_fields: [],
    default_config: defaultConfig,
-    sql_support: ["combine.union", "combine.join", "distinct", "derive"].includes(type) ? "partial" : "full"
+    sql_support:
+      ["replace", "quality.rules", "reconcile.compare", "subflow"].includes(type)
+        ? "none"
+        : ["combine.union", "combine.join", "distinct", "derive", "expression", "convert", "filter.expression"].includes(type)
+          ? "partial"
+          : "full"
  };
 }
@@ -1,5 +1,7 @@
 import {
  ArrowUpDown,
+  BadgeCheck,
+  Boxes,
  Braces,
  Columns3,
  Combine,
@@ -8,14 +10,21 @@ import {
  GitMerge,
  ListEnd,
  ListFilter,
+  ListChecks,
  PanelTopOpen,
+  Replace,
+  ReplaceAll,
+  ScanSearch,
  Sigma,
+  SquareFunction,
  Variable,
  type LucideIcon
 } from "lucide-react";

 const icons: Record<string, LucideIcon> = {
  "arrow-up-down": ArrowUpDown,
+  "badge-check": BadgeCheck,
+  boxes: Boxes,
  braces: Braces,
  "columns-3": Columns3,
  combine: Combine,
@@ -24,8 +33,13 @@ const icons: Record<string, LucideIcon> = {
  "git-merge": GitMerge,
  "list-end": ListEnd,
  "list-filter": ListFilter,
+  "list-checks": ListChecks,
  "panel-top-open": PanelTopOpen,
+  replace: Replace,
+  "replace-all": ReplaceAll,
+  "scan-search": ScanSearch,
  sigma: Sigma,
+  "function-square": SquareFunction,
  variable: Variable
 };