feat: run pinned pipelines and publish outputs

webui/src/api/dataflow.ts

@@ -146,6 +146,7 @@ export type TabularSource = {
   source_name: string;
   name: string;
   description?: string | null;
+  mode: "live" | "cached" | "static";
   columns: TabularSourceColumn[];
   schema_version: string;
   fingerprint: string;
@@ -161,6 +162,31 @@ export type TabularSourceCatalogue = {
   sources: TabularSource[];
 };
 
+export type PipelineRun = {
+  ref: string;
+  pipeline_id: string;
+  revision: number;
+  run_type: string;
+  status: "running" | "succeeded" | "failed" | "cancelled";
+  idempotency_key?: string | null;
+  definition_hash: string;
+  executor_version: string;
+  source_fingerprints: Record<string, unknown>[];
+  result_schema: PreviewColumn[];
+  diagnostics: DataflowDiagnostic[];
+  input_row_count: number;
+  output_row_count: number;
+  output_publication_ref?: string | null;
+  output_datasource_ref?: string | null;
+  output_materialization_ref?: string | null;
+  error?: string | null;
+  started_at: string;
+  finished_at?: string | null;
+  created_by?: string | null;
+  created_at: string;
+  replayed: boolean;
+};
+
 export async function listDataflowNodeTypes(settings: ApiSettings): Promise<NodeTypeDefinition[]> {
   const response = await apiFetch<{ nodes: NodeTypeDefinition[] }>(
     settings,
@@ -280,3 +306,43 @@ export function previewDataflowPipeline(
     body: JSON.stringify(payload)
   });
 }
+
+export async function listDataflowPipelineRuns(
+  settings: ApiSettings,
+  pipelineId: string
+): Promise<PipelineRun[]> {
+  const response = await apiFetch<{ runs: PipelineRun[] }>(
+    settings,
+    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs?limit=25`
+  );
+  return response.runs;
+}
+
+export function runDataflowPipeline(
+  settings: ApiSettings,
+  pipelineId: string,
+  payload: {
+    revision: number;
+    idempotency_key: string;
+    row_limit?: number;
+    publication?: {
+      target_datasource_ref?: string | null;
+      name?: string | null;
+      source_name?: string | null;
+      description?: string | null;
+      freeze?: boolean;
+      frozen_label?: string | null;
+      set_current?: boolean;
+      metadata?: Record<string, unknown>;
+    } | null;
+  }
+): Promise<PipelineRun> {
+  return apiFetch(
+    settings,
+    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs`,
+    {
+      method: "POST",
+      body: JSON.stringify(payload)
+    }
+  );
+}

webui/src/features/dataflow/DataflowPage.tsx

@@ -8,6 +8,7 @@ import {
 import {
   CheckCircle2,
   Code2,
+  DatabaseZap,
   Network,
   Play,
   Plus,
@@ -28,6 +29,7 @@ import {
   LoadingFrame,
   SegmentedControl,
   StatusBadge,
+  ToggleSwitch,
   hasScope,
   isApiError,
   useUnsavedChanges,
@@ -42,9 +44,11 @@ import {
   createDataflowSourceSnapshot,
   deleteDataflowPipeline,
   listDataflowNodeTypes,
+  listDataflowPipelineRuns,
   listDataflowPipelines,
   listDataflowSources,
   previewDataflowPipeline,
+  runDataflowPipeline,
   renderDataflowSql,
   updateDataflowPipeline,
   validateDataflowPipeline,
@@ -56,6 +60,7 @@ import {
   type Pipeline,
   type PipelineGraphNode,
   type PipelinePreview,
+  type PipelineRun,
   type TabularSource
 } from "../../api/dataflow";
 import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
@@ -96,6 +101,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
   const [resultTab, setResultTab] = useState<ResultTab>("preview");
   const [deleteOpen, setDeleteOpen] = useState(false);
   const [snapshotOpen, setSnapshotOpen] = useState(false);
+  const [runOpen, setRunOpen] = useState(false);
   const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
   const [sources, setSources] = useState<TabularSource[]>([]);
   const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
@@ -533,6 +539,13 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
                   <Button variant="primary" onClick={() => void runPreview()} disabled={working || !canRun}>
                     <Play size={16} /> Preview
                   </Button>
+                  <Button
+                    onClick={() => setRunOpen(true)}
+                    disabled={working || !canRun || dirty || !draft.id || !draft.currentRevision}
+                    title={dirty ? "Save the pipeline before starting a pinned run." : undefined}
+                  >
+                    <DatabaseZap size={16} /> Run
+                  </Button>
                   <IconButton
                     label="Discard changes"
                     icon={<RotateCcw size={16} />}
@@ -724,10 +737,265 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
           setSelectedNodeId(node.id);
         }}
       />
+      <RunPipelineDialog
+        open={runOpen}
+        settings={settings}
+        pipeline={draft?.id && draft.currentRevision ? {
+          id: draft.id,
+          name: draft.name,
+          revision: draft.currentRevision
+        } : null}
+        sources={sources}
+        onClose={() => setRunOpen(false)}
+        onCompleted={(run) => {
+          if (run.status === "succeeded") {
+            setSuccess(
+              run.output_datasource_ref
+                ? `Run published ${run.output_row_count} rows to ${run.output_datasource_ref}.`
+                : `Run completed with ${run.output_row_count} output rows.`
+            );
+          } else {
+            setError(run.error || "Dataflow run failed.");
+          }
+          if (run.output_datasource_ref) {
+            void listDataflowSources(settings).then((catalogue) => {
+              setSources(catalogue.sources);
+              setSourceCatalogueAvailable(catalogue.available);
+              setSourceCatalogueWritable(catalogue.writable);
+            });
+          }
+        }}
+      />
     </main>
   );
 }
 
+type RunMode = "run" | "publish";
+
+function RunPipelineDialog({
+  open,
+  settings,
+  pipeline,
+  sources,
+  onClose,
+  onCompleted
+}: {
+  open: boolean;
+  settings: ApiSettings;
+  pipeline: { id: string; name: string; revision: number } | null;
+  sources: TabularSource[];
+  onClose: () => void;
+  onCompleted: (run: PipelineRun) => void;
+}) {
+  const [mode, setMode] = useState<RunMode>("run");
+  const [targetRef, setTargetRef] = useState("new");
+  const [name, setName] = useState("");
+  const [sourceName, setSourceName] = useState("");
+  const [description, setDescription] = useState("");
+  const [freeze, setFreeze] = useState(false);
+  const [frozenLabel, setFrozenLabel] = useState("");
+  const [idempotencyKey, setIdempotencyKey] = useState("");
+  const [runs, setRuns] = useState<PipelineRun[]>([]);
+  const [loadingRuns, setLoadingRuns] = useState(false);
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState("");
+  const publicationTargets = sources.filter((source) =>
+    source.mode !== "live" && source.capabilities.includes("read")
+  );
+
+  useEffect(() => {
+    if (!open || !pipeline) return;
+    setMode("run");
+    setTargetRef("new");
+    setName(`${pipeline.name} output`);
+    setSourceName(sourceNameFromFile(`${pipeline.name}_output`));
+    setDescription("");
+    setFreeze(false);
+    setFrozenLabel("");
+    setIdempotencyKey(crypto.randomUUID());
+    setError("");
+    setLoadingRuns(true);
+    void listDataflowPipelineRuns(settings, pipeline.id)
+      .then(setRuns)
+      .catch((loadError) => setError(apiErrorMessage(loadError)))
+      .finally(() => setLoadingRuns(false));
+  }, [open, pipeline?.id, pipeline?.revision, settings]);
+
+  const start = async () => {
+    if (!pipeline) return;
+    setBusy(true);
+    setError("");
+    try {
+      const run = await runDataflowPipeline(settings, pipeline.id, {
+        revision: pipeline.revision,
+        idempotency_key: idempotencyKey,
+        row_limit: 500,
+        publication: mode === "publish" ? {
+          target_datasource_ref: targetRef === "new" ? null : targetRef,
+          name: targetRef === "new" ? name.trim() : null,
+          source_name: targetRef === "new" ? sourceName.trim() : null,
+          description: targetRef === "new" ? description.trim() || null : null,
+          freeze,
+          frozen_label: freeze ? frozenLabel.trim() || null : null,
+          set_current: true
+        } : null
+      });
+      setRuns((current) => [
+        run,
+        ...current.filter((item) => item.ref !== run.ref)
+      ]);
+      onCompleted(run);
+      setIdempotencyKey(crypto.randomUUID());
+      if (run.status !== "succeeded") {
+        setError(run.error || "Dataflow run failed.");
+      }
+    } catch (runError) {
+      setError(apiErrorMessage(runError));
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const targetComplete = mode === "run"
+    || targetRef !== "new"
+    || Boolean(name.trim() && sourceName.trim());
+
+  return (
+    <Dialog
+      open={open}
+      title={`Run ${pipeline?.name ?? "pipeline"}`}
+      className="dataflow-run-dialog"
+      onClose={() => {
+        if (!busy) onClose();
+      }}
+      footer={(
+        <>
+          <Button onClick={onClose} disabled={busy}>Close</Button>
+          <Button
+            variant="primary"
+            onClick={() => void start()}
+            disabled={busy || !pipeline || !targetComplete}
+          >
+            <Play size={16} /> {busy ? "Running..." : "Run revision"}
+          </Button>
+        </>
+      )}
+    >
+      <div className="dataflow-run-dialog-content">
+        {error ? (
+          <DismissibleAlert tone="danger" resetKey={error}>
+            {error}
+          </DismissibleAlert>
+        ) : null}
+        <div className="dataflow-run-controls">
+          <SegmentedControl<RunMode>
+            ariaLabel="Run output"
+            options={[
+              { id: "run", label: "Run only" },
+              { id: "publish", label: "Publish result" }
+            ]}
+            value={mode}
+            onChange={setMode}
+          />
+          <span className="dataflow-run-revision">
+            Revision {pipeline?.revision ?? "-"}
+          </span>
+        </div>
+        {mode === "publish" ? (
+          <div className="dataflow-run-publication-fields">
+            <FormField label="Target">
+              <select
+                value={targetRef}
+                onChange={(event) => setTargetRef(event.target.value)}
+              >
+                <option value="new">New datasource</option>
+                {publicationTargets.map((source) => (
+                  <option key={source.ref} value={source.ref}>
+                    {source.name} ({source.mode})
+                  </option>
+                ))}
+              </select>
+            </FormField>
+            {targetRef === "new" ? (
+              <>
+                <FormField label="Name">
+                  <input
+                    value={name}
+                    onChange={(event) => setName(event.target.value)}
+                  />
+                </FormField>
+                <FormField label="Logical source name">
+                  <input
+                    value={sourceName}
+                    onChange={(event) => setSourceName(event.target.value)}
+                    pattern="[A-Za-z_][A-Za-z0-9_]*"
+                  />
+                </FormField>
+                <FormField label="Description">
+                  <input
+                    value={description}
+                    onChange={(event) => setDescription(event.target.value)}
+                  />
+                </FormField>
+              </>
+            ) : null}
+            <div className="dataflow-run-freeze">
+              <ToggleSwitch
+                label="Freeze published state"
+                checked={freeze}
+                onChange={setFreeze}
+              />
+            </div>
+            {freeze ? (
+              <FormField label="Frozen-state label">
+                <input
+                  value={frozenLabel}
+                  onChange={(event) => setFrozenLabel(event.target.value)}
+                />
+              </FormField>
+            ) : null}
+          </div>
+        ) : null}
+        <section className="dataflow-run-history">
+          <div className="dataflow-run-history-heading">
+            <strong>Recent runs</strong>
+            <small>{loadingRuns ? "Loading..." : `${runs.length} shown`}</small>
+          </div>
+          <div>
+            <table>
+              <thead>
+                <tr>
+                  <th>Started</th>
+                  <th>Revision</th>
+                  <th>Status</th>
+                  <th>Rows</th>
+                  <th>Output</th>
+                </tr>
+              </thead>
+              <tbody>
+                {runs.map((run) => (
+                  <tr key={run.ref}>
+                    <td>{new Date(run.started_at).toLocaleString()}</td>
+                    <td>{run.revision}</td>
+                    <td><StatusBadge status={run.status} label={run.status} /></td>
+                    <td>{run.output_row_count.toLocaleString()}</td>
+                    <td title={run.output_materialization_ref ?? ""}>
+                      {run.output_datasource_ref ?? "Run evidence"}
+                    </td>
+                  </tr>
+                ))}
+                {!loadingRuns && !runs.length ? (
+                  <tr><td colSpan={5}>No runs yet</td></tr>
+                ) : null}
+              </tbody>
+            </table>
+          </div>
+        </section>
+      </div>
+    </Dialog>
+  );
+}
+
 function SourceSnapshotDialog({
   open,
   settings,

webui/src/styles/dataflow.css

@@ -601,6 +601,106 @@
   width: min(720px, calc(100vw - 32px));
 }
 
+.dataflow-run-dialog {
+  width: min(860px, calc(100vw - 32px));
+}
+
+.dataflow-run-dialog-content {
+  display: grid;
+  gap: 14px;
+  max-height: min(680px, calc(100vh - 210px));
+  overflow: hidden;
+}
+
+.dataflow-run-controls,
+.dataflow-run-history-heading {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+}
+
+.dataflow-run-revision,
+.dataflow-run-history-heading small {
+  color: var(--muted);
+  font-size: 11px;
+}
+
+.dataflow-run-publication-fields {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 12px;
+  padding: 12px 0;
+  border-top: var(--border-line);
+  border-bottom: var(--border-line);
+}
+
+.dataflow-run-publication-fields input,
+.dataflow-run-publication-fields select {
+  margin-top: 5px;
+  padding: 7px 8px;
+}
+
+.dataflow-run-freeze {
+  display: flex;
+  min-height: 54px;
+  align-items: end;
+  padding-bottom: 4px;
+}
+
+.dataflow-run-history {
+  display: flex;
+  min-height: 190px;
+  flex-direction: column;
+  overflow: hidden;
+}
+
+.dataflow-run-history-heading {
+  min-height: 36px;
+  flex: 0 0 auto;
+  border-bottom: var(--border-line);
+}
+
+.dataflow-run-history-heading strong {
+  color: var(--text-strong);
+  font-size: 12px;
+}
+
+.dataflow-run-history > div:last-child {
+  min-height: 0;
+  flex: 1 1 auto;
+  overflow: auto;
+}
+
+.dataflow-run-history table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 11px;
+}
+
+.dataflow-run-history th,
+.dataflow-run-history td {
+  max-width: 230px;
+  border-bottom: var(--border-line);
+  padding: 7px 9px;
+  overflow: hidden;
+  text-align: left;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.dataflow-run-history th {
+  position: sticky;
+  z-index: 1;
+  top: 0;
+  background: var(--panel-soft);
+  color: var(--text-strong);
+}
+
+.dataflow-run-history tbody tr:hover {
+  background: var(--primary-soft);
+}
+
 .dataflow-source-dialog-fields {
   display: grid;
   grid-template-columns: 1fr 1fr;
@@ -942,4 +1042,8 @@
   .dataflow-results {
     flex-basis: 220px;
   }
+
+  .dataflow-run-publication-fields {
+    grid-template-columns: 1fr;
+  }
 }