GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(release): harden durable run recovery

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 18:58:54 +02:00
parent ead697d049
commit 1dc9148ec3
5 changed files with 1046 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
tools/release/webui/index.html
View File

@@ -907,6 +907,7 @@
manualRepo: "",
currentRun: null,
runs: [],
runStoreAvailable: true,
};
elements.refresh.addEventListener("click", load);
@@ -959,7 +960,11 @@
return fetch(`${path}${suffix}`, {
headers: token ? { "X-Release-Console-Token": token } : {},
}).then((response) => {
if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
if (!response.ok) {
return response.json().catch(() => ({})).then((payload) => {
throw new Error(payload.detail || `${response.status} ${response.statusText}`);
});
}
return response.json();
});
}
@@ -985,14 +990,21 @@
async function load() {
elements.statusLine.textContent = "Refreshing...";
try {
const [dashboard, intelligence, runs] = await Promise.all([api("/api/dashboard"), api("/api/release-intelligence"), api("/api/release-runs")]);
const runsRequest = api("/api/release-runs")
.then((data) => ({ data, error: null }))
.catch((error) => ({ data: null, error }));
const [dashboard, intelligence, runsResult] = await Promise.all([api("/api/dashboard"), api("/api/release-intelligence"), runsRequest]);
renderDashboard(dashboard);
renderReleaseIntelligence(intelligence);
renderReleaseRunList(runs.runs || []);
if (releaseState.currentRun?.run_id) {
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
if (runsResult.error) {
renderReleaseRunStoreUnavailable(runsResult.error);
} else {
renderIdlePlan();
renderReleaseRunList(runsResult.data?.runs || []);
if (releaseState.currentRun?.run_id) {
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
} else {
renderIdlePlan();
}
}
elements.statusLine.textContent = `Generated ${dashboard.generated_at}`;
} catch (error) {
@@ -1019,7 +1031,10 @@
}
function renderReleaseRunList(runs) {
releaseState.runStoreAvailable = true;
releaseState.runs = Array.isArray(runs) ? runs : [];
elements.releaseRun.disabled = false;
elements.createReleaseRun.disabled = false;
const selectedId = releaseState.currentRun?.run_id || "";
const options = releaseState.runs.map((run) => {
const repositories = run.repo_versions ? Object.keys(run.repo_versions).length : 0;
@@ -1032,7 +1047,20 @@
}
}
function renderReleaseRunStoreUnavailable(error) {
releaseState.runStoreAvailable = false;
releaseState.runs = [];
releaseState.currentRun = null;
elements.releaseRun.innerHTML = `<option value="">Run storage unavailable</option>`;
elements.releaseRun.disabled = true;
elements.createReleaseRun.disabled = true;
elements.resumeReleaseRun.disabled = true;
elements.runStatus.textContent = "unavailable";
elements.runOutput.innerHTML = `<div class="action"><h3>${pill("unavailable", "block")} Durable run storage cannot be read</h3><p>${escapeHtml(error.message)}</p><p class="action-meta">Dashboard and release previews remain available. Correct the private state-directory problem, then refresh.</p></div>`;
}
async function createReleaseRun() {
if (!releaseState.runStoreAvailable) return;
const repoVersions = selectedRepoVersions();
if (!Object.keys(repoVersions).length) {
elements.runStatus.textContent = "selection required";
@@ -1110,7 +1138,7 @@
const recommendation = run.recommended_next || {};
const steps = Array.isArray(state.steps) ? state.steps : [];
const events = Array.isArray(state.events) ? state.events.slice(-5).reverse() : [];
const recommendationKind = state.status === "blocked" ? "block" : recommendation.available ? "ok" : "warn";
const recommendationKind = state.status === "blocked" ? "block" : state.status === "attention" ? "warn" : recommendation.available ? "ok" : "warn";
const recommendationHtml = recommendation.id ? `<div class="action recommended">
<h3>${pill("recommended next", recommendationKind)} ${escapeHtml(recommendation.title || recommendation.id)}</h3>
<p>${escapeHtml(recommendation.detail || "")}</p>
@@ -1119,20 +1147,37 @@
const stepHtml = steps.map((step) => {
const kind = step.state === "succeeded" ? "ok" : ["failed", "interrupted"].includes(step.state) ? "block" : step.available ? "ok" : "warn";
const retryReason = step.retry_available ? "Prepare this known failed or read-only interrupted step for an explicit retry." : (step.disabled_reason || "Retry is unavailable for this state.");
const reconciliationHtml = step.reconciliation_required ? `<div class="form-grid" data-run-reconcile-step="${escapeAttr(step.id)}">
<div><label>Observed external effect</label><select data-run-reconcile-outcome><option value="">Choose verified outcome</option><option value="effect_absent">Effect is absent — retry is safe</option><option value="effect_succeeded">Effect succeeded — advance the run</option><option value="unresolved">Still unresolved — keep blocked</option></select></div>
<div><label>Confirmation</label><input type="text" data-run-reconcile-confirm placeholder="Type RECONCILE" autocomplete="off" /></div>
<div class="button-row"><button class="secondary" data-run-reconcile-submit disabled title="Verify local and remote state independently, choose the observed outcome, and type RECONCILE.">Record Reconciliation</button></div>
</div>` : "";
return `<div class="action">
<div class="stage-title"><h3>${escapeHtml(`${step.order}. ${step.title || step.id}`)}</h3>${pill(step.state, kind)}</div>
<p>${escapeHtml(step.detail || "")}</p>
${step.result_code ? `<p class="action-meta">Result code: ${escapeHtml(step.result_code)}</p>` : ""}
<div class="button-row"><button class="secondary" data-run-retry-step="${escapeAttr(step.id)}" ${step.retry_available ? "" : "disabled"} title="${escapeAttr(retryReason)}">Prepare Retry</button></div>
${reconciliationHtml}
</div>`;
}).join("");
const eventHtml = events.length ? `<div class="action"><h3>Recent bounded state events</h3>${events.map((event) => `<p class="action-meta">${escapeHtml(`${event.at} · ${event.type}${event.step_id ? ` · ${event.step_id}` : ""}${event.result_code ? ` · ${event.result_code}` : ""}`)}</p>`).join("")}</div>` : "";
elements.runStatus.textContent = state.status || "unknown";
elements.resumeReleaseRun.disabled = state.status === "completed";
elements.resumeReleaseRun.disabled = state.status !== "running";
elements.runOutput.innerHTML = `<div class="action"><h3>${pill(state.status || "unknown", state.status === "blocked" ? "block" : state.status === "completed" ? "ok" : "warn")} ${escapeHtml(run.run_id)}</h3><p>Frozen plan ${escapeHtml((run.immutable?.digest || "").slice(0, 16))}… · updated ${escapeHtml(run.updated_at || "-")}</p><p class="action-meta">Executor controls remain separate and keep their existing confirmations.</p></div>${recommendationHtml}${stepHtml}${eventHtml}`;
for (const button of elements.runOutput.querySelectorAll("[data-run-retry-step]")) {
button.addEventListener("click", () => retryReleaseRunStep(button.dataset.runRetryStep));
}
for (const controls of elements.runOutput.querySelectorAll("[data-run-reconcile-step]")) {
const outcome = controls.querySelector("[data-run-reconcile-outcome]");
const confirmation = controls.querySelector("[data-run-reconcile-confirm]");
const submit = controls.querySelector("[data-run-reconcile-submit]");
const refresh = () => {
submit.disabled = !outcome.value || confirmation.value.trim() !== "RECONCILE";
};
outcome.addEventListener("change", refresh);
confirmation.addEventListener("input", refresh);
submit.addEventListener("click", () => reconcileReleaseRunStep(controls.dataset.runReconcileStep, outcome.value, confirmation.value.trim(), submit));
}
}
async function resumeReleaseRun() {
@@ -1147,7 +1192,7 @@
elements.runStatus.textContent = "error";
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
} finally {
elements.resumeReleaseRun.disabled = releaseState.currentRun?.state?.status === "completed";
elements.resumeReleaseRun.disabled = releaseState.currentRun?.state?.status !== "running";
}
}
@@ -1155,7 +1200,7 @@
const run = releaseState.currentRun;
if (!run || !stepId) return;
try {
const retried = await postJson(`/api/release-runs/${encodeURIComponent(run.run_id)}/steps/${encodeURIComponent(stepId)}/retry`, { request_id: requestId("retry"), reconciled: false });
const retried = await postJson(`/api/release-runs/${encodeURIComponent(run.run_id)}/steps/${encodeURIComponent(stepId)}/retry`, { request_id: requestId("retry") });
releaseState.currentRun = retried;
renderReleaseRun(retried);
} catch (error) {
@@ -1163,6 +1208,24 @@
}
}
async function reconcileReleaseRunStep(stepId, outcome, confirmation, button) {
const run = releaseState.currentRun;
if (!run || !stepId || !outcome || confirmation !== "RECONCILE") return;
setBusy([button], true);
try {
const reconciled = await postJson(`/api/release-runs/${encodeURIComponent(run.run_id)}/steps/${encodeURIComponent(stepId)}/reconcile`, {
request_id: requestId("reconcile"),
outcome,
confirm: confirmation,
});
releaseState.currentRun = reconciled;
renderReleaseRun(reconciled);
} catch (error) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
button.disabled = false;
}
}
function requestId(prefix) {
const identifier = window.crypto?.randomUUID ? window.crypto.randomUUID() : `${Date.now()}-${Math.random().toString(16).slice(2)}`;
return `${prefix}-${identifier}`;