feat: run pinned pipelines and publish outputs

2026-07-28 13:47:59 +02:00
parent 6ca3058021
commit 6305ef9cef
13 changed files with 1461 additions and 7 deletions
@@ -8,6 +8,7 @@ import {
 import {
  CheckCircle2,
  Code2,
+  DatabaseZap,
  Network,
  Play,
  Plus,
@@ -28,6 +29,7 @@ import {
  LoadingFrame,
  SegmentedControl,
  StatusBadge,
+  ToggleSwitch,
  hasScope,
  isApiError,
  useUnsavedChanges,
@@ -42,9 +44,11 @@ import {
  createDataflowSourceSnapshot,
  deleteDataflowPipeline,
  listDataflowNodeTypes,
+  listDataflowPipelineRuns,
  listDataflowPipelines,
  listDataflowSources,
  previewDataflowPipeline,
+  runDataflowPipeline,
  renderDataflowSql,
  updateDataflowPipeline,
  validateDataflowPipeline,
@@ -56,6 +60,7 @@ import {
  type Pipeline,
  type PipelineGraphNode,
  type PipelinePreview,
+  type PipelineRun,
  type TabularSource
 } from "../../api/dataflow";
 import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
@@ -96,6 +101,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
  const [resultTab, setResultTab] = useState<ResultTab>("preview");
  const [deleteOpen, setDeleteOpen] = useState(false);
  const [snapshotOpen, setSnapshotOpen] = useState(false);
+  const [runOpen, setRunOpen] = useState(false);
  const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
  const [sources, setSources] = useState<TabularSource[]>([]);
  const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
@@ -533,6 +539,13 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
                  <Button variant="primary" onClick={() => void runPreview()} disabled={working || !canRun}>
                    <Play size={16} /> Preview
                  </Button>
+                  <Button
+                    onClick={() => setRunOpen(true)}
+                    disabled={working || !canRun || dirty || !draft.id || !draft.currentRevision}
+                    title={dirty ? "Save the pipeline before starting a pinned run." : undefined}
+                  >
+                    <DatabaseZap size={16} /> Run
+                  </Button>
                  <IconButton
                    label="Discard changes"
                    icon={<RotateCcw size={16} />}
@@ -724,10 +737,265 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
          setSelectedNodeId(node.id);
        }}
      />
+      <RunPipelineDialog
+        open={runOpen}
+        settings={settings}
+        pipeline={draft?.id && draft.currentRevision ? {
+          id: draft.id,
+          name: draft.name,
+          revision: draft.currentRevision
+        } : null}
+        sources={sources}
+        onClose={() => setRunOpen(false)}
+        onCompleted={(run) => {
+          if (run.status === "succeeded") {
+            setSuccess(
+              run.output_datasource_ref
+                ? `Run published ${run.output_row_count} rows to ${run.output_datasource_ref}.`
+                : `Run completed with ${run.output_row_count} output rows.`
+            );
+          } else {
+            setError(run.error || "Dataflow run failed.");
+          }
+          if (run.output_datasource_ref) {
+            void listDataflowSources(settings).then((catalogue) => {
+              setSources(catalogue.sources);
+              setSourceCatalogueAvailable(catalogue.available);
+              setSourceCatalogueWritable(catalogue.writable);
+            });
+          }
+        }}
+      />
    </main>
  );
 }

+type RunMode = "run" | "publish";
+
+function RunPipelineDialog({
+  open,
+  settings,
+  pipeline,
+  sources,
+  onClose,
+  onCompleted
+}: {
+  open: boolean;
+  settings: ApiSettings;
+  pipeline: { id: string; name: string; revision: number } | null;
+  sources: TabularSource[];
+  onClose: () => void;
+  onCompleted: (run: PipelineRun) => void;
+}) {
+  const [mode, setMode] = useState<RunMode>("run");
+  const [targetRef, setTargetRef] = useState("new");
+  const [name, setName] = useState("");
+  const [sourceName, setSourceName] = useState("");
+  const [description, setDescription] = useState("");
+  const [freeze, setFreeze] = useState(false);
+  const [frozenLabel, setFrozenLabel] = useState("");
+  const [idempotencyKey, setIdempotencyKey] = useState("");
+  const [runs, setRuns] = useState<PipelineRun[]>([]);
+  const [loadingRuns, setLoadingRuns] = useState(false);
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState("");
+  const publicationTargets = sources.filter((source) =>
+    source.mode !== "live" && source.capabilities.includes("read")
+  );
+
+  useEffect(() => {
+    if (!open || !pipeline) return;
+    setMode("run");
+    setTargetRef("new");
+    setName(`${pipeline.name} output`);
+    setSourceName(sourceNameFromFile(`${pipeline.name}_output`));
+    setDescription("");
+    setFreeze(false);
+    setFrozenLabel("");
+    setIdempotencyKey(crypto.randomUUID());
+    setError("");
+    setLoadingRuns(true);
+    void listDataflowPipelineRuns(settings, pipeline.id)
+      .then(setRuns)
+      .catch((loadError) => setError(apiErrorMessage(loadError)))
+      .finally(() => setLoadingRuns(false));
+  }, [open, pipeline?.id, pipeline?.revision, settings]);
+
+  const start = async () => {
+    if (!pipeline) return;
+    setBusy(true);
+    setError("");
+    try {
+      const run = await runDataflowPipeline(settings, pipeline.id, {
+        revision: pipeline.revision,
+        idempotency_key: idempotencyKey,
+        row_limit: 500,
+        publication: mode === "publish" ? {
+          target_datasource_ref: targetRef === "new" ? null : targetRef,
+          name: targetRef === "new" ? name.trim() : null,
+          source_name: targetRef === "new" ? sourceName.trim() : null,
+          description: targetRef === "new" ? description.trim() || null : null,
+          freeze,
+          frozen_label: freeze ? frozenLabel.trim() || null : null,
+          set_current: true
+        } : null
+      });
+      setRuns((current) => [
+        run,
+        ...current.filter((item) => item.ref !== run.ref)
+      ]);
+      onCompleted(run);
+      setIdempotencyKey(crypto.randomUUID());
+      if (run.status !== "succeeded") {
+        setError(run.error || "Dataflow run failed.");
+      }
+    } catch (runError) {
+      setError(apiErrorMessage(runError));
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const targetComplete = mode === "run"
+    || targetRef !== "new"
+    || Boolean(name.trim() && sourceName.trim());
+
+  return (
+    <Dialog
+      open={open}
+      title={`Run ${pipeline?.name ?? "pipeline"}`}
+      className="dataflow-run-dialog"
+      onClose={() => {
+        if (!busy) onClose();
+      }}
+      footer={(
+        <>
+          <Button onClick={onClose} disabled={busy}>Close</Button>
+          <Button
+            variant="primary"
+            onClick={() => void start()}
+            disabled={busy || !pipeline || !targetComplete}
+          >
+            <Play size={16} /> {busy ? "Running..." : "Run revision"}
+          </Button>
+        </>
+      )}
+    >
+      <div className="dataflow-run-dialog-content">
+        {error ? (
+          <DismissibleAlert tone="danger" resetKey={error}>
+            {error}
+          </DismissibleAlert>
+        ) : null}
+        <div className="dataflow-run-controls">
+          <SegmentedControl<RunMode>
+            ariaLabel="Run output"
+            options={[
+              { id: "run", label: "Run only" },
+              { id: "publish", label: "Publish result" }
+            ]}
+            value={mode}
+            onChange={setMode}
+          />
+          <span className="dataflow-run-revision">
+            Revision {pipeline?.revision ?? "-"}
+          </span>
+        </div>
+        {mode === "publish" ? (
+          <div className="dataflow-run-publication-fields">
+            <FormField label="Target">
+              <select
+                value={targetRef}
+                onChange={(event) => setTargetRef(event.target.value)}
+              >
+                <option value="new">New datasource</option>
+                {publicationTargets.map((source) => (
+                  <option key={source.ref} value={source.ref}>
+                    {source.name} ({source.mode})
+                  </option>
+                ))}
+              </select>
+            </FormField>
+            {targetRef === "new" ? (
+              <>
+                <FormField label="Name">
+                  <input
+                    value={name}
+                    onChange={(event) => setName(event.target.value)}
+                  />
+                </FormField>
+                <FormField label="Logical source name">
+                  <input
+                    value={sourceName}
+                    onChange={(event) => setSourceName(event.target.value)}
+                    pattern="[A-Za-z_][A-Za-z0-9_]*"
+                  />
+                </FormField>
+                <FormField label="Description">
+                  <input
+                    value={description}
+                    onChange={(event) => setDescription(event.target.value)}
+                  />
+                </FormField>
+              </>
+            ) : null}
+            <div className="dataflow-run-freeze">
+              <ToggleSwitch
+                label="Freeze published state"
+                checked={freeze}
+                onChange={setFreeze}
+              />
+            </div>
+            {freeze ? (
+              <FormField label="Frozen-state label">
+                <input
+                  value={frozenLabel}
+                  onChange={(event) => setFrozenLabel(event.target.value)}
+                />
+              </FormField>
+            ) : null}
+          </div>
+        ) : null}
+        <section className="dataflow-run-history">
+          <div className="dataflow-run-history-heading">
+            <strong>Recent runs</strong>
+            <small>{loadingRuns ? "Loading..." : `${runs.length} shown`}</small>
+          </div>
+          <div>
+            <table>
+              <thead>
+                <tr>
+                  <th>Started</th>
+                  <th>Revision</th>
+                  <th>Status</th>
+                  <th>Rows</th>
+                  <th>Output</th>
+                </tr>
+              </thead>
+              <tbody>
+                {runs.map((run) => (
+                  <tr key={run.ref}>
+                    <td>{new Date(run.started_at).toLocaleString()}</td>
+                    <td>{run.revision}</td>
+                    <td><StatusBadge status={run.status} label={run.status} /></td>
+                    <td>{run.output_row_count.toLocaleString()}</td>
+                    <td title={run.output_materialization_ref ?? ""}>
+                      {run.output_datasource_ref ?? "Run evidence"}
+                    </td>
+                  </tr>
+                ))}
+                {!loadingRuns && !runs.length ? (
+                  <tr><td colSpan={5}>No runs yet</td></tr>
+                ) : null}
+              </tbody>
+            </table>
+          </div>
+        </section>
+      </div>
+    </Dialog>
+  );
+}
+
 function SourceSnapshotDialog({
  open,
  settings,