Fence and reconcile Dataflow runs

This commit is contained in:
2026-08-03 06:09:53 +02:00
parent e1c092ece7
commit 3fa7a29f48
13 changed files with 1314 additions and 44 deletions
+7 -1
View File
@@ -222,7 +222,7 @@ export type PipelineRun = {
pipeline_id: string;
revision: number;
run_type: string;
status: "queued" | "retrying" | "running" | "succeeded" | "failed" | "cancelled";
status: "queued" | "retrying" | "running" | "succeeded" | "failed" | "cancelled" | "outcome_unknown";
idempotency_key?: string | null;
execution_backend: "auto" | "reference" | "duckdb";
environment: "development" | "staging" | "production";
@@ -256,6 +256,12 @@ export type PipelineRun = {
finished_at?: string | null;
created_by?: string | null;
created_at: string;
recovery_operation_id?: string | null;
recovery_operation_type?: string | null;
recovery_mode?: string | null;
recovery_status?: string | null;
recovery_requires_attention: boolean;
recovery_explanation?: string | null;
replayed: boolean;
};
+25 -1
View File
@@ -1653,6 +1653,7 @@ function RunPipelineDialog({
const hasActiveRuns = runs.some((run) =>
run.status === "queued" || run.status === "retrying" || run.status === "running"
);
const recoveryAttentionRuns = runs.filter((run) => run.recovery_requires_attention);
useEffect(() => {
if (!open || !pipeline || !hasActiveRuns) return;
@@ -1877,6 +1878,15 @@ function RunPipelineDialog({
<strong>Recent runs</strong>
<small>{loadingRuns ? "Loading..." : `${runs.length} shown`}</small>
</div>
{recoveryAttentionRuns.length ? (
<DismissibleAlert
tone="warning"
resetKey={recoveryAttentionRuns.map((run) => `${run.ref}:${run.recovery_status}`).join("|")}
dismissible={false}
>
{recoveryAttentionRuns.length} run(s) require output reconciliation. Verify the recorded sink result before retrying.
</DismissibleAlert>
) : null}
<div>
<table>
<thead>
@@ -1885,6 +1895,7 @@ function RunPipelineDialog({
<th>Revision</th>
<th>Status</th>
<th>Progress</th>
<th>Recovery</th>
<th>Rows</th>
<th>Output</th>
<th aria-label="Actions" />
@@ -1897,6 +1908,19 @@ function RunPipelineDialog({
<td>{run.revision}</td>
<td><StatusBadge status={run.status} label={run.status} /></td>
<td>{run.progress_percent}% · {run.progress_phase}</td>
<td className="dataflow-run-recovery" title={run.recovery_explanation ?? ""}>
{run.recovery_status ? (
<>
<StatusBadge
status={run.recovery_requires_attention ? "warning" : run.recovery_status}
label={run.recovery_status.replace(/_/g, " ")}
/>
{run.recovery_explanation ? <small>{run.recovery_explanation}</small> : null}
</>
) : (
<span>Not recorded</span>
)}
</td>
<td>{run.output_row_count.toLocaleString()}</td>
<td title={run.output_materialization_ref ?? ""}>
{run.output_datasource_ref ?? "Run evidence"}
@@ -1916,7 +1940,7 @@ function RunPipelineDialog({
</tr>
))}
{!loadingRuns && !runs.length ? (
<tr><td colSpan={7}>No runs yet</td></tr>
<tr><td colSpan={8}>No runs yet</td></tr>
) : null}
</tbody>
</table>
+12
View File
@@ -832,6 +832,18 @@
background: var(--primary-soft);
}
.dataflow-run-history td.dataflow-run-recovery {
min-width: 180px;
white-space: normal;
}
.dataflow-run-recovery small {
display: block;
margin-top: 4px;
color: var(--muted);
line-height: 1.35;
}
.dataflow-source-dialog-fields {
display: grid;
grid-template-columns: 1fr 1fr;