fix(release): preserve durable recovery capacity

This commit is contained in:
2026-07-22 19:48:55 +02:00
parent 22f5c2ff4e
commit 4c16069f88
4 changed files with 537 additions and 33 deletions

View File

@@ -108,13 +108,17 @@ commands, process output, confirmation text, credentials, bearer tokens, or
signing material.
Before a step can enter `running`, the store reserves the two command-ledger
slots needed for worst-case recovery. An explicit resume consumes one slot and
is accepted only while a persisted attempt is actually running. A mutating
attempt always retains its final `effect_absent` or `effect_succeeded` slot;
slots needed for worst-case recovery. It also projects the serialized record
through start, finish/failure, resume, and the required terminal reconciliation
or read-only retry; the start is rejected unless every required atomic write
fits the record-size bound. An explicit resume consumes one slot and is
accepted only while a persisted attempt is actually running. A mutating attempt
always retains its final `effect_absent` or `effect_succeeded` slot;
`unresolved` may be recorded at most once for that attempt and only when an
additional slot is available. Read-only interruption and known failure retain
the slot needed to prepare a retry. Capacity exhaustion is therefore detected
before starting an effect rather than stranding an ambiguous attempt.
additional ledger slot and serialized terminal-write capacity are available.
Read-only interruption and known failure retain the slot and byte capacity
needed to prepare a retry. Capacity exhaustion is therefore detected before
starting an effect rather than stranding an ambiguous attempt.
An explicit resume after a process restart converts every persisted `running`
step to `interrupted`; it never guesses whether an external effect happened.
@@ -128,16 +132,20 @@ code-only state event. A known failed attempt can likewise be prepared for
retry. The UI keeps unavailable controls visible and disabled.
The browser likewise retains the request identifier for an uncertain
resume/retry/reconciliation response and reuses it on the operator's retry.
Identifiers are cleared only after the server returns the resulting run state.
resume/retry/reconciliation response and replays it after reload. A successful
replay selects the returned run state. Transport and server failures retain the
identifier; a deterministic `4xx` rejection clears it so a stale command cannot
poison a later attempt.
The run API is covered by the same local console token middleware as every
other `/api/` route:
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
freezes a selective plan only when that creation is not already known.
- `GET /api/release-runs` lists bounded summaries; unreadable entries are
reported as unavailable instead of being silently omitted.
- `GET /api/release-runs` lists bounded summaries ordered by verified
`updated_at`, then `created_at` and run identifier. Unreadable entries are
appended deterministically when room remains instead of displacing newer
verified runs.
- `GET /api/release-runs/{run_id}` reads and verifies one exact record.
- `POST /api/release-runs/{run_id}/resume` records explicit recovery.
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or

View File

@@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path:
from govoplan_release.release_run import ( # noqa: E402
MAX_EVENTS,
MAX_RECORD_BYTES,
ReleaseRunConflict,
ReleaseRunCorrupt,
ReleaseRunNotFound,
@@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase):
)
self.assertEqual("completed", completed["state"]["status"])
def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None:
rejected_plan = mutating_first_plan()
rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)]
rejected_run_id = create_run(
self.store,
request_id="byte-capacity-rejected-create-0001",
plan_snapshot=rejected_plan,
)["run_id"]
rejected_path = self.root / f"{rejected_run_id}.json"
self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000)
with self.assertRaisesRegex(
ReleaseRunConflict, "serialized recovery capacity"
):
self.store.start_step(
rejected_run_id,
"core:tag",
attempt_id="byte-capacity-rejected-attempt-0001",
)
untouched = self.store.get(rejected_run_id)
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
accepted_plan = mutating_first_plan()
accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
accepted_run_id = create_run(
self.store,
request_id="byte-capacity-accepted-create-0001",
plan_snapshot=accepted_plan,
)["run_id"]
accepted_path = self.root / f"{accepted_run_id}.json"
self.store.start_step(
accepted_run_id,
"core:tag",
attempt_id="byte-capacity-accepted-attempt-0001",
)
self.store.resume(
accepted_run_id,
request_id="byte-capacity-accepted-resume-0001",
)
with self.assertRaisesRegex(
ReleaseRunConflict, "reserved for a terminal reconciliation"
):
self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
completed = self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-terminal-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("completed", completed["state"]["status"])
self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES)
def test_read_only_start_reserves_serialized_resume_and_retry_capacity(
self,
) -> None:
plan = mutating_first_plan()
plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index]
plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
run_id = create_run(
self.store,
request_id="byte-capacity-read-only-create-0001",
plan_snapshot=plan,
)["run_id"]
self.store.start_step(
run_id,
"core:tag",
attempt_id="byte-capacity-read-only-attempt-0001",
)
self.store.resume(
run_id,
request_id="byte-capacity-read-only-resume-0001",
)
retried = self.store.retry_step(
run_id,
"core:tag",
request_id="byte-capacity-read-only-retry-0001",
)
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
run_id = create_run(
self.store,
@@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase):
listed = self.store.list(limit=1)
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable(
self,
) -> None:
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:00Z",
):
older = create_run(
self.store,
request_id="timestamp-order-request-older-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T01:00:00Z",
):
newer = create_run(
self.store,
request_id="timestamp-order-request-newer-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T02:00:00Z",
):
self.store.start_step(
older["run_id"],
"core:preflight",
attempt_id="timestamp-order-attempt-0001",
)
corrupt = create_run(
self.store,
request_id="timestamp-order-request-corrupt-0001",
)
corrupt_path = self.root / f"{corrupt['run_id']}.json"
corrupt_path.write_text("{}", encoding="utf-8")
corrupt_path.chmod(0o600)
listed = self.store.list(limit=3)
self.assertEqual(
[older["run_id"], newer["run_id"], corrupt["run_id"]],
[item["run_id"] for item in listed],
)
self.assertEqual("unavailable", listed[-1]["status"])
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
def test_list_enumeration_oserror_is_normalized(self) -> None:
self.store.list()
with patch.object(
Path,
"iterdir",
side_effect=PermissionError("denied"),
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"):
self.store.list()
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
run_id = create_run(
self.store,
@@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
self.assertIn("pendingReleaseRunPayload", webui)
self.assertIn("recoverPendingReleaseRun", webui)
self.assertIn("recoverPendingRunCommands", webui)
self.assertIn("inFlightRunCommandId", webui)
self.assertIn("Run saved; list refresh unavailable", webui)
self.assertIn("This foundation tracks state only.", runbook)
@@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase):
"submitReleaseRunCreate",
"refreshReleaseRunListAfterCreate",
"readInFlightRunCommands",
"pendingRunCommandRequest",
"recoverPendingRunCommands",
"deterministicRunCommandError",
"inFlightRunCommandId",
"clearInFlightRunCommand",
"requestId",
@@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru
const renderReleaseRunList = () => {};
let failCreate = true;
let failList = true;
let failureStatus = null;
const postedRequestIds = [];
async function postJson(_path, payload) {
const postedPaths = [];
async function postJson(path, payload) {
postedPaths.push(path);
postedRequestIds.push(payload.request_id);
if (failCreate) throw new Error("uncertain transport failure");
if (failCreate) {
const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure");
if (failureStatus) error.status = failureStatus;
throw error;
}
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
}
async function api(_path) {
@@ -1194,11 +1348,40 @@ async function api(_path) {
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry");
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry");
const commandKey = "retry:rr-known:core:preflight";
const commandOne = inFlightRunCommandId(commandKey, "retry");
const commandReplay = inFlightRunCommandId(commandKey, "retry");
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
clearInFlightRunCommand("retry:rr-known:step", commandOne);
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry");
const retryRequest = pendingRunCommandRequest(commandKey, commandOne);
assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable");
const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request");
assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable");
assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed");
failCreate = true;
const uncertainCommand = await recoverPendingRunCommands();
assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared");
failCreate = false;
const recoveredCommand = await recoverPendingRunCommands();
assert(recoveredCommand === true, "pending command did not recover after reload");
assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared");
assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID");
assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint");
assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run");
const conflictKey = "resume:rr-known";
const conflictId = inFlightRunCommandId(conflictKey, "resume");
failCreate = true;
failureStatus = 409;
const rejectedCommand = await recoverPendingRunCommands();
assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID");
assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained");
failCreate = false;
failureStatus = null;
const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry");
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
"""

View File

@@ -59,6 +59,8 @@ ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"}
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
RECONCILIATION_CONFIRMATION = "RECONCILE"
START_RECOVERY_RESERVE = 2
PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z"
PROJECTION_RESULT_CODE = "r" * 64
class ReleaseRunError(RuntimeError):
@@ -200,32 +202,32 @@ class ReleaseRunStore:
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
with self._locked():
self._ensure_root()
paths = sorted(
(
try:
paths = [
path
for path in self.root.iterdir()
if path.name.endswith(".json")
and RUN_ID_PATTERN.fullmatch(path.stem)
),
key=lambda item: item.name,
reverse=True,
)
]
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run storage could not be enumerated safely."
) from exc
summaries: list[dict[str, Any]] = []
unavailable: list[dict[str, Any]] = []
for path in paths:
try:
record = self._read(path.stem)
except ReleaseRunWorkspaceMismatch:
continue
except (ReleaseRunCorrupt, ReleaseRunNotFound):
summaries.append(
unavailable.append(
{
"run_id": path.stem,
"status": "unavailable",
"error": "Run record is unavailable or failed integrity validation.",
}
)
if len(summaries) >= limit:
break
continue
view = _view(record)
immutable_input = view["immutable"]["input"]
@@ -240,9 +242,16 @@ class ReleaseRunStore:
"recommended_next": view["recommended_next"],
}
)
if len(summaries) >= limit:
break
return summaries
summaries.sort(
key=lambda item: (
item["updated_at"],
item["created_at"],
item["run_id"],
),
reverse=True,
)
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
return (summaries + unavailable)[:limit]
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
request_fingerprint = _request_fingerprint(request_id)
@@ -386,6 +395,7 @@ class ReleaseRunStore:
"An unresolved outcome is already recorded for this attempt."
)
_ensure_command_capacity(record, reserve_after=1)
_ensure_unresolved_recovery_byte_capacity(record, step_id)
else:
_ensure_command_capacity(record)
if outcome == "effect_absent":
@@ -446,7 +456,7 @@ class ReleaseRunStore:
"The attempt identifier was already used for another release step."
)
return _view(record)
step, _plan_step = _step_pair(record, step_id)
step, plan_step = _step_pair(record, step_id)
available, reason = _step_available(record, step_id)
if not available:
raise ReleaseRunConflict(reason)
@@ -471,6 +481,11 @@ class ReleaseRunStore:
}
)
_append_event(record, event_type="step_started", step_id=step_id)
_ensure_start_recovery_byte_capacity(
record,
step_id,
mutating=bool(plan_step.get("mutating")),
)
self._persist_update(record)
return _view(record)
@@ -1337,12 +1352,13 @@ def _append_event(
event_type: str,
step_id: str | None = None,
result_code: str | None = None,
at: str | None = None,
) -> None:
if event_type not in EVENT_TYPES:
raise ReleaseRunConflict("Release event type is invalid.")
event: dict[str, Any] = {
"sequence": record["state"]["next_event_sequence"],
"at": _timestamp(),
"at": at if at is not None else _timestamp(),
"type": event_type,
}
if step_id is not None:
@@ -1355,6 +1371,216 @@ def _append_event(
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
def _ensure_start_recovery_byte_capacity(
record: dict[str, Any], step_id: str, *, mutating: bool
) -> None:
"""Prove that every required completion/recovery write still fits."""
started = deepcopy(record)
_project_persist(started)
projections = [started]
succeeded = _project_finish(started, step_id, succeeded=True)
failed = _project_finish(started, step_id, succeeded=False)
projections.extend((succeeded, failed, _project_retry(failed, step_id)))
interrupted = _project_resume(started)
projections.append(interrupted)
if mutating:
projections.extend(
_project_reconcile(interrupted, step_id, outcome=outcome)
for outcome in ("effect_absent", "effect_succeeded")
)
else:
projections.append(_project_retry(interrupted, step_id))
_ensure_projected_records_fit(
projections,
message=(
"Release run has insufficient serialized recovery capacity for a "
"new attempt; create a new run."
),
)
def _ensure_unresolved_recovery_byte_capacity(
record: dict[str, Any], step_id: str
) -> None:
"""Accept an advisory unresolved write only if a terminal write remains."""
unresolved = _project_reconcile(record, step_id, outcome="unresolved")
projections = [unresolved]
projections.extend(
_project_reconcile(unresolved, step_id, outcome=outcome)
for outcome in ("effect_absent", "effect_succeeded")
)
_ensure_projected_records_fit(
projections,
message=(
"The unresolved outcome would consume capacity reserved for a "
"terminal reconciliation."
),
)
def _project_finish(
record: dict[str, Any], step_id: str, *, succeeded: bool
) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt = _attempt_for_fingerprint(
projected, step["attempt_fingerprint"], step_id=step_id
)
target_state = "succeeded" if succeeded else "failed"
step.update(
{
"state": target_state,
"finished_at": PROJECTION_TIMESTAMP,
"result_code": PROJECTION_RESULT_CODE,
}
)
attempt["outcome"] = target_state
attempt["result_code"] = PROJECTION_RESULT_CODE
_append_event(
projected,
event_type="step_succeeded" if succeeded else "step_failed",
step_id=step_id,
result_code=PROJECTION_RESULT_CODE,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_resume(record: dict[str, Any]) -> dict[str, Any]:
projected = deepcopy(record)
running_steps = [
step for step in projected["state"]["steps"] if step["state"] == "running"
]
for step in running_steps:
attempt = _attempt_for_fingerprint(
projected, step["attempt_fingerprint"], step_id=step["id"]
)
attempt["outcome"] = "interrupted"
attempt["result_code"] = "process_interrupted"
step["state"] = "interrupted"
step["finished_at"] = PROJECTION_TIMESTAMP
step["result_code"] = "process_interrupted"
_append_event(
projected,
event_type="step_interrupted",
step_id=step["id"],
result_code="process_interrupted",
at=PROJECTION_TIMESTAMP,
)
_remember_request(
projected,
"0" * 64,
"resume",
None,
None,
attempt_fingerprint=running_steps[0]["attempt_fingerprint"],
)
_append_event(projected, event_type="run_resumed", at=PROJECTION_TIMESTAMP)
_project_persist(projected)
return projected
def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt_fingerprint = step["attempt_fingerprint"]
retry_reason = (
"known_failure"
if step["state"] == "failed"
else "read_only_interruption"
)
step.update(
{
"state": "pending",
"attempt_fingerprint": None,
"started_at": None,
"finished_at": None,
"result_code": None,
}
)
_remember_request(
projected,
"1" * 64,
"retry",
step_id,
None,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
projected,
event_type="step_retry_requested",
step_id=step_id,
result_code=retry_reason,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_reconcile(
record: dict[str, Any], step_id: str, *, outcome: str
) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt_fingerprint = step["attempt_fingerprint"]
if outcome == "effect_absent":
step.update(
{
"state": "pending",
"attempt_fingerprint": None,
"started_at": None,
"finished_at": None,
"result_code": None,
}
)
elif outcome == "effect_succeeded":
step.update(
{
"state": "succeeded",
"finished_at": PROJECTION_TIMESTAMP,
"result_code": "reconciled_effect_succeeded",
}
)
_remember_request(
projected,
"2" * 64 if outcome == "unresolved" else "3" * 64,
"reconcile",
step_id,
outcome,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
projected,
event_type="step_reconciled",
step_id=step_id,
result_code=outcome,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_persist(record: dict[str, Any]) -> None:
record["updated_at"] = PROJECTION_TIMESTAMP
_refresh_status(record)
record["record_digest"] = "f" * 64
def _ensure_projected_records_fit(
records: list[dict[str, Any]], *, message: str
) -> None:
if any(
len(_canonical_json(projected)) + 1 > MAX_RECORD_BYTES
for projected in records
):
raise ReleaseRunConflict(message)
def _ensure_command_capacity(
record: dict[str, Any], *, reserve_after: int = 0
) -> None:

View File

@@ -1006,10 +1006,12 @@
renderReleaseRunStoreUnavailable(runsResult.error);
} else {
renderReleaseRunList(runsResult.data?.runs || []);
const pendingRecoveryResult = await recoverPendingReleaseRun();
if (pendingRecoveryResult === null && releaseState.currentRun?.run_id) {
const pendingCreateResult = await recoverPendingReleaseRun();
const pendingCommandResult = await recoverPendingRunCommands();
const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null;
if (!recoveryAttempted && releaseState.currentRun?.run_id) {
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
} else if (pendingRecoveryResult === null) {
} else if (!recoveryAttempted) {
renderIdlePlan();
}
}
@@ -1262,6 +1264,7 @@
releaseState.currentRun = resumed;
renderReleaseRun(resumed);
} catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runStatus.textContent = "error";
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
} finally {
@@ -1280,6 +1283,7 @@
releaseState.currentRun = retried;
renderReleaseRun(retried);
} catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
}
}
@@ -1300,6 +1304,7 @@
releaseState.currentRun = reconciled;
renderReleaseRun(reconciled);
} catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
button.disabled = false;
}
@@ -1315,6 +1320,88 @@
}
}
function pendingRunCommandRequest(commandKey, requestIdToReplay) {
if (typeof commandKey !== "string" || typeof requestIdToReplay !== "string") return null;
if (commandKey.startsWith("resume:")) {
const runId = commandKey.slice("resume:".length);
if (!runId || runId.includes(":")) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/resume`,
body: { request_id: requestIdToReplay },
};
}
if (commandKey.startsWith("retry:")) {
const remainder = commandKey.slice("retry:".length);
const separator = remainder.indexOf(":");
const runId = remainder.slice(0, separator);
const stepId = remainder.slice(separator + 1);
if (separator < 1 || !stepId) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/retry`,
body: { request_id: requestIdToReplay },
};
}
if (commandKey.startsWith("reconcile:")) {
const remainder = commandKey.slice("reconcile:".length);
const runSeparator = remainder.indexOf(":");
const outcomeSeparator = remainder.lastIndexOf(":");
const runId = remainder.slice(0, runSeparator);
const stepId = remainder.slice(runSeparator + 1, outcomeSeparator);
const outcome = remainder.slice(outcomeSeparator + 1);
if (runSeparator < 1 || outcomeSeparator <= runSeparator + 1 || !["effect_absent", "effect_succeeded", "unresolved"].includes(outcome)) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/reconcile`,
body: { request_id: requestIdToReplay, outcome, confirm: "RECONCILE" },
};
}
return null;
}
async function recoverPendingRunCommands() {
const commands = readInFlightRunCommands();
const entries = Object.entries(commands);
if (!entries.length) return null;
let attempted = false;
let recoveredRun = null;
let pendingError = null;
let rejectedError = null;
for (const [commandKey, requestIdToReplay] of entries) {
const command = pendingRunCommandRequest(commandKey, requestIdToReplay);
if (!command) {
clearInFlightRunCommand(commandKey, requestIdToReplay);
continue;
}
attempted = true;
try {
recoveredRun = await postJson(command.path, command.body);
clearInFlightRunCommand(commandKey, requestIdToReplay);
} catch (error) {
if (deterministicRunCommandError(error)) {
clearInFlightRunCommand(commandKey, requestIdToReplay);
rejectedError = error;
} else {
pendingError = error;
}
}
}
if (recoveredRun) {
releaseState.currentRun = recoveredRun;
restoreReleaseRunSelection(recoveredRun);
renderReleaseRun(recoveredRun);
elements.releaseRun.value = recoveredRun.run_id;
}
if (pendingError) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("pending", "warn")} Saved command recovery is still pending</h3><p>${escapeHtml(pendingError.message)}</p><p class="action-meta">The same private request identifier will be replayed after refresh.</p></div>`);
} else if (rejectedError) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("rejected", "block")} Saved command was not accepted</h3><p>${escapeHtml(rejectedError.message)}</p><p class="action-meta">The server returned a deterministic client conflict; the stale request identifier was cleared.</p></div>`);
}
return attempted ? Boolean(recoveredRun) : null;
}
function deterministicRunCommandError(error) {
return error?.status >= 400 && error.status < 500;
}
function inFlightRunCommandId(commandKey, prefix) {
const commands = readInFlightRunCommands();
if (typeof commands[commandKey] === "string") return commands[commandKey];