feat: run dataflows through durable workers

webui/src/api/dataflow.ts

@@ -214,8 +214,10 @@ export type PipelineRun = {
   pipeline_id: string;
   revision: number;
   run_type: string;
-  status: "running" | "succeeded" | "failed" | "cancelled";
+  status: "queued" | "retrying" | "running" | "succeeded" | "failed" | "cancelled";
   idempotency_key?: string | null;
+  execution_backend: "auto" | "reference" | "duckdb";
+  environment: "development" | "staging" | "production";
   definition_hash: string;
   executor_version: string;
   source_fingerprints: Record<string, unknown>[];
@@ -231,14 +233,37 @@ export type PipelineRun = {
   delivery_ref?: string | null;
   correlation_id?: string | null;
   causation_id?: string | null;
+  attempts: number;
+  max_attempts: number;
+  available_at?: string | null;
+  claimed_at?: string | null;
+  lease_expires_at?: string | null;
+  cancellation_requested_at?: string | null;
+  progress_percent: number;
+  progress_phase: string;
+  retention_until?: string | null;
+  purged_at?: string | null;
   error?: string | null;
-  started_at: string;
+  started_at?: string | null;
   finished_at?: string | null;
   created_by?: string | null;
   created_at: string;
   replayed: boolean;
 };
 
+export type PipelineDeployment = {
+  id: string;
+  pipeline_id: string;
+  revision: number;
+  environment: "development" | "staging" | "production";
+  source_environment: "development" | "staging" | "production";
+  status: string;
+  provenance: Record<string, unknown>;
+  promoted_by?: string | null;
+  created_at: string;
+  updated_at: string;
+};
+
 export type DataflowTriggerKind = "once" | "interval" | "event";
 export type DataflowTrigger = {
   id: string;
@@ -505,6 +530,10 @@ export function runDataflowPipeline(
     revision: number;
     idempotency_key: string;
     row_limit?: number;
+    execution_backend?: "reference" | "duckdb" | "auto";
+    environment?: "development" | "staging" | "production";
+    max_attempts?: number;
+    retention_days?: number;
     publication?: {
       target_datasource_ref?: string | null;
       name?: string | null;
@@ -526,3 +555,45 @@ export function runDataflowPipeline(
     }
   );
 }
+
+export function cancelDataflowPipelineRun(
+  settings: ApiSettings,
+  runRef: string
+): Promise<PipelineRun> {
+  const runId = runRef.replace(/^dataflow-run:/, "");
+  return apiFetch(
+    settings,
+    `/api/v1/dataflow/runs/${encodeURIComponent(runId)}/cancel`,
+    { method: "POST" }
+  );
+}
+
+export async function listDataflowPipelineDeployments(
+  settings: ApiSettings,
+  pipelineId: string
+): Promise<PipelineDeployment[]> {
+  const response = await apiFetch<{ deployments: PipelineDeployment[] }>(
+    settings,
+    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/deployments`
+  );
+  return response.deployments;
+}
+
+export function promoteDataflowPipeline(
+  settings: ApiSettings,
+  pipelineId: string,
+  payload: {
+    revision: number;
+    source_environment: "development" | "staging";
+    target_environment: "staging" | "production";
+  }
+): Promise<PipelineDeployment> {
+  return apiFetch(
+    settings,
+    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/promotions`,
+    {
+      method: "POST",
+      body: JSON.stringify(payload)
+    }
+  );
+}

webui/src/features/dataflow/DataflowPage.tsx

@@ -8,6 +8,7 @@ import {
 } from "react";
 import {
   CheckCircle2,
+  CircleStop,
   Clock3,
   Code2,
   CopyPlus,
@@ -16,6 +17,7 @@ import {
   Play,
   Plus,
   RefreshCw,
+  Rocket,
   RotateCcw,
   Save,
   Settings2,
@@ -46,6 +48,7 @@ import { ReactFlowProvider } from "@xyflow/react";
 import { useLocation } from "react-router-dom";
 import {
   compileDataflowSql,
+  cancelDataflowPipelineRun,
   createDataflowTrigger,
   createDataflowPipeline,
   createDataflowSourceSnapshot,
@@ -55,10 +58,12 @@ import {
   deriveDataflowPipeline,
   listDataflowNodeTypes,
   listDataflowPipelineRuns,
+  listDataflowPipelineDeployments,
   listDataflowPipelines,
   listDataflowSources,
   listDataflowTriggers,
   previewDataflowPipeline,
+  promoteDataflowPipeline,
   runDataflowPipeline,
   renderDataflowSql,
   updateDataflowPipeline,
@@ -75,6 +80,7 @@ import {
   type Pipeline,
   type PipelineGraphNode,
   type PipelinePreview,
+  type PipelineDeployment,
   type PipelineRun,
   type TabularSource
 } from "../../api/dataflow";
@@ -925,6 +931,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
           revision: draft.currentRevision
         } : null}
         sources={sources}
+        canPromote={canWrite}
         onClose={() => setRunOpen(false)}
         onCompleted={(run) => {
           if (run.status === "succeeded") {
@@ -933,8 +940,10 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
                 ? `Run published ${run.output_row_count} rows to ${run.output_datasource_ref}.`
                 : `Run completed with ${run.output_row_count} output rows.`
             );
-          } else {
+          } else if (run.status === "failed") {
             setError(run.error || "Dataflow run failed.");
+          } else {
+            setSuccess(`Run ${run.ref} was queued.`);
           }
           if (run.output_datasource_ref) {
             void listDataflowSources(settings).then((catalogue) => {
@@ -1586,6 +1595,7 @@ function RunPipelineDialog({
   settings,
   pipeline,
   sources,
+  canPromote,
   onClose,
   onCompleted
 }: {
@@ -1593,6 +1603,7 @@ function RunPipelineDialog({
   settings: ApiSettings;
   pipeline: { id: string; name: string; revision: number } | null;
   sources: TabularSource[];
+  canPromote: boolean;
   onClose: () => void;
   onCompleted: (run: PipelineRun) => void;
 }) {
@@ -1604,6 +1615,8 @@ function RunPipelineDialog({
   const [freeze, setFreeze] = useState(false);
   const [frozenLabel, setFrozenLabel] = useState("");
   const [idempotencyKey, setIdempotencyKey] = useState("");
+  const [environment, setEnvironment] = useState<"development" | "staging" | "production">("development");
+  const [deployments, setDeployments] = useState<PipelineDeployment[]>([]);
   const [runs, setRuns] = useState<PipelineRun[]>([]);
   const [loadingRuns, setLoadingRuns] = useState(false);
   const [busy, setBusy] = useState(false);
@@ -1622,14 +1635,35 @@ function RunPipelineDialog({
     setFreeze(false);
     setFrozenLabel("");
     setIdempotencyKey(crypto.randomUUID());
+    setEnvironment("development");
     setError("");
     setLoadingRuns(true);
-    void listDataflowPipelineRuns(settings, pipeline.id)
-      .then(setRuns)
+    void Promise.all([
+      listDataflowPipelineRuns(settings, pipeline.id),
+      listDataflowPipelineDeployments(settings, pipeline.id)
+    ])
+      .then(([loadedRuns, loadedDeployments]) => {
+        setRuns(loadedRuns);
+        setDeployments(loadedDeployments);
+      })
       .catch((loadError) => setError(apiErrorMessage(loadError)))
       .finally(() => setLoadingRuns(false));
   }, [open, pipeline?.id, pipeline?.revision, settings]);
 
+  const hasActiveRuns = runs.some((run) =>
+    run.status === "queued" || run.status === "retrying" || run.status === "running"
+  );
+
+  useEffect(() => {
+    if (!open || !pipeline || !hasActiveRuns) return;
+    const timer = window.setInterval(() => {
+      void listDataflowPipelineRuns(settings, pipeline.id)
+        .then(setRuns)
+        .catch(() => undefined);
+    }, 2000);
+    return () => window.clearInterval(timer);
+  }, [hasActiveRuns, open, pipeline?.id, settings]);
+
   const start = async () => {
     if (!pipeline) return;
     setBusy(true);
@@ -1639,6 +1673,8 @@ function RunPipelineDialog({
         revision: pipeline.revision,
         idempotency_key: idempotencyKey,
         row_limit: 500,
+        execution_backend: "auto",
+        environment,
         publication: mode === "publish" ? {
           target_datasource_ref: targetRef === "new" ? null : targetRef,
           name: targetRef === "new" ? name.trim() : null,
@@ -1655,7 +1691,7 @@ function RunPipelineDialog({
       ]);
       onCompleted(run);
       setIdempotencyKey(crypto.randomUUID());
-      if (run.status !== "succeeded") {
+      if (run.status === "failed") {
         setError(run.error || "Dataflow run failed.");
       }
     } catch (runError) {
@@ -1665,6 +1701,39 @@ function RunPipelineDialog({
     }
   };
 
+  const cancelRun = async (run: PipelineRun) => {
+    setError("");
+    try {
+      const cancelled = await cancelDataflowPipelineRun(settings, run.ref);
+      setRuns((current) => current.map((item) =>
+        item.ref === cancelled.ref ? cancelled : item
+      ));
+    } catch (cancelError) {
+      setError(apiErrorMessage(cancelError));
+    }
+  };
+
+  const promote = async (target: "staging" | "production") => {
+    if (!pipeline) return;
+    setBusy(true);
+    setError("");
+    try {
+      const deployment = await promoteDataflowPipeline(settings, pipeline.id, {
+        revision: pipeline.revision,
+        source_environment: target === "staging" ? "development" : "staging",
+        target_environment: target
+      });
+      setDeployments((current) => [
+        ...current.filter((item) => item.environment !== target),
+        deployment
+      ]);
+    } catch (promotionError) {
+      setError(apiErrorMessage(promotionError));
+    } finally {
+      setBusy(false);
+    }
+  };
+
   const targetComplete = mode === "run"
     || targetRef !== "new"
     || Boolean(name.trim() && sourceName.trim());
@@ -1685,7 +1754,7 @@ function RunPipelineDialog({
             onClick={() => void start()}
             disabled={busy || !pipeline || !targetComplete}
           >
-            <Play size={16} /> {busy ? "Running..." : "Run revision"}
+            <Play size={16} /> {busy ? "Queueing..." : "Queue run"}
           </Button>
         </>
       )}
@@ -1710,6 +1779,44 @@ function RunPipelineDialog({
             Revision {pipeline?.revision ?? "-"}
           </span>
         </div>
+        <div className="dataflow-run-publication-fields">
+          <FormField label="Environment">
+            <select
+              value={environment}
+              onChange={(event) => setEnvironment(
+                event.target.value as "development" | "staging" | "production"
+              )}
+            >
+              <option value="development">Development</option>
+              <option value="staging">Staging</option>
+              <option value="production">Production</option>
+            </select>
+          </FormField>
+          {canPromote ? (
+            <>
+              <Button
+                onClick={() => void promote("staging")}
+                disabled={busy || deployments.some((item) =>
+                  item.environment === "staging" && item.revision === pipeline?.revision
+                )}
+              >
+                <Rocket size={16} /> Promote to staging
+              </Button>
+              <Button
+                onClick={() => void promote("production")}
+                disabled={busy
+                  || !deployments.some((item) =>
+                    item.environment === "staging" && item.revision === pipeline?.revision
+                  )
+                  || deployments.some((item) =>
+                    item.environment === "production" && item.revision === pipeline?.revision
+                  )}
+              >
+                <Rocket size={16} /> Promote to production
+              </Button>
+            </>
+          ) : null}
+        </div>
         {mode === "publish" ? (
           <div className="dataflow-run-publication-fields">
             <FormField label="Target">
@@ -1774,27 +1881,42 @@ function RunPipelineDialog({
             <table>
               <thead>
                 <tr>
-                  <th>Started</th>
+                  <th>Queued</th>
                   <th>Revision</th>
                   <th>Status</th>
+                  <th>Progress</th>
                   <th>Rows</th>
                   <th>Output</th>
+                  <th aria-label="Actions" />
                 </tr>
               </thead>
               <tbody>
                 {runs.map((run) => (
                   <tr key={run.ref}>
-                    <td>{new Date(run.started_at).toLocaleString()}</td>
+                    <td>{new Date(run.started_at ?? run.created_at).toLocaleString()}</td>
                     <td>{run.revision}</td>
                     <td><StatusBadge status={run.status} label={run.status} /></td>
+                    <td>{run.progress_percent}% · {run.progress_phase}</td>
                     <td>{run.output_row_count.toLocaleString()}</td>
                     <td title={run.output_materialization_ref ?? ""}>
                       {run.output_datasource_ref ?? "Run evidence"}
                     </td>
+                    <td>
+                      {run.status === "queued"
+                        || run.status === "retrying"
+                        || run.status === "running" ? (
+                          <IconButton
+                            label="Cancel run"
+                            icon={<CircleStop size={16} />}
+                            variant="danger"
+                            onClick={() => void cancelRun(run)}
+                          />
+                        ) : null}
+                    </td>
                   </tr>
                 ))}
                 {!loadingRuns && !runs.length ? (
-                  <tr><td colSpan={5}>No runs yet</td></tr>
+                  <tr><td colSpan={7}>No runs yet</td></tr>
                 ) : null}
               </tbody>
             </table>