fix(release): preserve durable recovery capacity
This commit is contained in:
@@ -108,13 +108,17 @@ commands, process output, confirmation text, credentials, bearer tokens, or
|
||||
signing material.
|
||||
|
||||
Before a step can enter `running`, the store reserves the two command-ledger
|
||||
slots needed for worst-case recovery. An explicit resume consumes one slot and
|
||||
is accepted only while a persisted attempt is actually running. A mutating
|
||||
attempt always retains its final `effect_absent` or `effect_succeeded` slot;
|
||||
slots needed for worst-case recovery. It also projects the serialized record
|
||||
through start, finish/failure, resume, and the required terminal reconciliation
|
||||
or read-only retry; the start is rejected unless every required atomic write
|
||||
fits the record-size bound. An explicit resume consumes one slot and is
|
||||
accepted only while a persisted attempt is actually running. A mutating attempt
|
||||
always retains its final `effect_absent` or `effect_succeeded` slot;
|
||||
`unresolved` may be recorded at most once for that attempt and only when an
|
||||
additional slot is available. Read-only interruption and known failure retain
|
||||
the slot needed to prepare a retry. Capacity exhaustion is therefore detected
|
||||
before starting an effect rather than stranding an ambiguous attempt.
|
||||
additional ledger slot and serialized terminal-write capacity are available.
|
||||
Read-only interruption and known failure retain the slot and byte capacity
|
||||
needed to prepare a retry. Capacity exhaustion is therefore detected before
|
||||
starting an effect rather than stranding an ambiguous attempt.
|
||||
|
||||
An explicit resume after a process restart converts every persisted `running`
|
||||
step to `interrupted`; it never guesses whether an external effect happened.
|
||||
@@ -128,16 +132,20 @@ code-only state event. A known failed attempt can likewise be prepared for
|
||||
retry. The UI keeps unavailable controls visible and disabled.
|
||||
|
||||
The browser likewise retains the request identifier for an uncertain
|
||||
resume/retry/reconciliation response and reuses it on the operator's retry.
|
||||
Identifiers are cleared only after the server returns the resulting run state.
|
||||
resume/retry/reconciliation response and replays it after reload. A successful
|
||||
replay selects the returned run state. Transport and server failures retain the
|
||||
identifier; a deterministic `4xx` rejection clears it so a stale command cannot
|
||||
poison a later attempt.
|
||||
|
||||
The run API is covered by the same local console token middleware as every
|
||||
other `/api/` route:
|
||||
|
||||
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
|
||||
freezes a selective plan only when that creation is not already known.
|
||||
- `GET /api/release-runs` lists bounded summaries; unreadable entries are
|
||||
reported as unavailable instead of being silently omitted.
|
||||
- `GET /api/release-runs` lists bounded summaries ordered by verified
|
||||
`updated_at`, then `created_at` and run identifier. Unreadable entries are
|
||||
appended deterministically when room remains instead of displacing newer
|
||||
verified runs.
|
||||
- `GET /api/release-runs/{run_id}` reads and verifies one exact record.
|
||||
- `POST /api/release-runs/{run_id}/resume` records explicit recovery.
|
||||
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or
|
||||
|
||||
@@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path:
|
||||
|
||||
from govoplan_release.release_run import ( # noqa: E402
|
||||
MAX_EVENTS,
|
||||
MAX_RECORD_BYTES,
|
||||
ReleaseRunConflict,
|
||||
ReleaseRunCorrupt,
|
||||
ReleaseRunNotFound,
|
||||
@@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual("completed", completed["state"]["status"])
|
||||
|
||||
def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None:
|
||||
rejected_plan = mutating_first_plan()
|
||||
rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)]
|
||||
rejected_run_id = create_run(
|
||||
self.store,
|
||||
request_id="byte-capacity-rejected-create-0001",
|
||||
plan_snapshot=rejected_plan,
|
||||
)["run_id"]
|
||||
rejected_path = self.root / f"{rejected_run_id}.json"
|
||||
self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000)
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseRunConflict, "serialized recovery capacity"
|
||||
):
|
||||
self.store.start_step(
|
||||
rejected_run_id,
|
||||
"core:tag",
|
||||
attempt_id="byte-capacity-rejected-attempt-0001",
|
||||
)
|
||||
untouched = self.store.get(rejected_run_id)
|
||||
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
|
||||
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
|
||||
|
||||
accepted_plan = mutating_first_plan()
|
||||
accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
|
||||
accepted_run_id = create_run(
|
||||
self.store,
|
||||
request_id="byte-capacity-accepted-create-0001",
|
||||
plan_snapshot=accepted_plan,
|
||||
)["run_id"]
|
||||
accepted_path = self.root / f"{accepted_run_id}.json"
|
||||
self.store.start_step(
|
||||
accepted_run_id,
|
||||
"core:tag",
|
||||
attempt_id="byte-capacity-accepted-attempt-0001",
|
||||
)
|
||||
self.store.resume(
|
||||
accepted_run_id,
|
||||
request_id="byte-capacity-accepted-resume-0001",
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseRunConflict, "reserved for a terminal reconciliation"
|
||||
):
|
||||
self.store.reconcile_step(
|
||||
accepted_run_id,
|
||||
"core:tag",
|
||||
request_id="byte-capacity-unresolved-request-0001",
|
||||
outcome="unresolved",
|
||||
confirmation="RECONCILE",
|
||||
)
|
||||
completed = self.store.reconcile_step(
|
||||
accepted_run_id,
|
||||
"core:tag",
|
||||
request_id="byte-capacity-terminal-request-0001",
|
||||
outcome="effect_succeeded",
|
||||
confirmation="RECONCILE",
|
||||
)
|
||||
self.assertEqual("completed", completed["state"]["status"])
|
||||
self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES)
|
||||
|
||||
def test_read_only_start_reserves_serialized_resume_and_retry_capacity(
|
||||
self,
|
||||
) -> None:
|
||||
plan = mutating_first_plan()
|
||||
plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index]
|
||||
plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
|
||||
run_id = create_run(
|
||||
self.store,
|
||||
request_id="byte-capacity-read-only-create-0001",
|
||||
plan_snapshot=plan,
|
||||
)["run_id"]
|
||||
self.store.start_step(
|
||||
run_id,
|
||||
"core:tag",
|
||||
attempt_id="byte-capacity-read-only-attempt-0001",
|
||||
)
|
||||
self.store.resume(
|
||||
run_id,
|
||||
request_id="byte-capacity-read-only-resume-0001",
|
||||
)
|
||||
retried = self.store.retry_step(
|
||||
run_id,
|
||||
"core:tag",
|
||||
request_id="byte-capacity-read-only-retry-0001",
|
||||
)
|
||||
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
|
||||
|
||||
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
|
||||
run_id = create_run(
|
||||
self.store,
|
||||
@@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase):
|
||||
listed = self.store.list(limit=1)
|
||||
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
|
||||
|
||||
def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable(
|
||||
self,
|
||||
) -> None:
|
||||
with patch(
|
||||
"govoplan_release.release_run._timestamp",
|
||||
return_value="2026-07-22T00:00:00Z",
|
||||
):
|
||||
older = create_run(
|
||||
self.store,
|
||||
request_id="timestamp-order-request-older-0001",
|
||||
)
|
||||
with patch(
|
||||
"govoplan_release.release_run._timestamp",
|
||||
return_value="2026-07-22T01:00:00Z",
|
||||
):
|
||||
newer = create_run(
|
||||
self.store,
|
||||
request_id="timestamp-order-request-newer-0001",
|
||||
)
|
||||
with patch(
|
||||
"govoplan_release.release_run._timestamp",
|
||||
return_value="2026-07-22T02:00:00Z",
|
||||
):
|
||||
self.store.start_step(
|
||||
older["run_id"],
|
||||
"core:preflight",
|
||||
attempt_id="timestamp-order-attempt-0001",
|
||||
)
|
||||
|
||||
corrupt = create_run(
|
||||
self.store,
|
||||
request_id="timestamp-order-request-corrupt-0001",
|
||||
)
|
||||
corrupt_path = self.root / f"{corrupt['run_id']}.json"
|
||||
corrupt_path.write_text("{}", encoding="utf-8")
|
||||
corrupt_path.chmod(0o600)
|
||||
|
||||
listed = self.store.list(limit=3)
|
||||
self.assertEqual(
|
||||
[older["run_id"], newer["run_id"], corrupt["run_id"]],
|
||||
[item["run_id"] for item in listed],
|
||||
)
|
||||
self.assertEqual("unavailable", listed[-1]["status"])
|
||||
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
|
||||
|
||||
def test_list_enumeration_oserror_is_normalized(self) -> None:
|
||||
self.store.list()
|
||||
with patch.object(
|
||||
Path,
|
||||
"iterdir",
|
||||
side_effect=PermissionError("denied"),
|
||||
):
|
||||
with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"):
|
||||
self.store.list()
|
||||
|
||||
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
|
||||
run_id = create_run(
|
||||
self.store,
|
||||
@@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase):
|
||||
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
|
||||
self.assertIn("pendingReleaseRunPayload", webui)
|
||||
self.assertIn("recoverPendingReleaseRun", webui)
|
||||
self.assertIn("recoverPendingRunCommands", webui)
|
||||
self.assertIn("inFlightRunCommandId", webui)
|
||||
self.assertIn("Run saved; list refresh unavailable", webui)
|
||||
self.assertIn("This foundation tracks state only.", runbook)
|
||||
@@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase):
|
||||
"submitReleaseRunCreate",
|
||||
"refreshReleaseRunListAfterCreate",
|
||||
"readInFlightRunCommands",
|
||||
"pendingRunCommandRequest",
|
||||
"recoverPendingRunCommands",
|
||||
"deterministicRunCommandError",
|
||||
"inFlightRunCommandId",
|
||||
"clearInFlightRunCommand",
|
||||
"requestId",
|
||||
@@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru
|
||||
const renderReleaseRunList = () => {};
|
||||
let failCreate = true;
|
||||
let failList = true;
|
||||
let failureStatus = null;
|
||||
const postedRequestIds = [];
|
||||
async function postJson(_path, payload) {
|
||||
const postedPaths = [];
|
||||
async function postJson(path, payload) {
|
||||
postedPaths.push(path);
|
||||
postedRequestIds.push(payload.request_id);
|
||||
if (failCreate) throw new Error("uncertain transport failure");
|
||||
if (failCreate) {
|
||||
const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure");
|
||||
if (failureStatus) error.status = failureStatus;
|
||||
throw error;
|
||||
}
|
||||
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
|
||||
}
|
||||
async function api(_path) {
|
||||
@@ -1194,11 +1348,40 @@ async function api(_path) {
|
||||
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
|
||||
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
|
||||
|
||||
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry");
|
||||
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry");
|
||||
const commandKey = "retry:rr-known:core:preflight";
|
||||
const commandOne = inFlightRunCommandId(commandKey, "retry");
|
||||
const commandReplay = inFlightRunCommandId(commandKey, "retry");
|
||||
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
|
||||
clearInFlightRunCommand("retry:rr-known:step", commandOne);
|
||||
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry");
|
||||
const retryRequest = pendingRunCommandRequest(commandKey, commandOne);
|
||||
assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable");
|
||||
const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request");
|
||||
assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable");
|
||||
assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed");
|
||||
|
||||
failCreate = true;
|
||||
const uncertainCommand = await recoverPendingRunCommands();
|
||||
assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded");
|
||||
assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared");
|
||||
failCreate = false;
|
||||
const recoveredCommand = await recoverPendingRunCommands();
|
||||
assert(recoveredCommand === true, "pending command did not recover after reload");
|
||||
assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared");
|
||||
assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID");
|
||||
assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint");
|
||||
assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run");
|
||||
|
||||
const conflictKey = "resume:rr-known";
|
||||
const conflictId = inFlightRunCommandId(conflictKey, "resume");
|
||||
failCreate = true;
|
||||
failureStatus = 409;
|
||||
const rejectedCommand = await recoverPendingRunCommands();
|
||||
assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded");
|
||||
assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID");
|
||||
assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained");
|
||||
failCreate = false;
|
||||
failureStatus = null;
|
||||
|
||||
const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry");
|
||||
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
|
||||
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
|
||||
"""
|
||||
|
||||
@@ -59,6 +59,8 @@ ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"}
|
||||
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
|
||||
RECONCILIATION_CONFIRMATION = "RECONCILE"
|
||||
START_RECOVERY_RESERVE = 2
|
||||
PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z"
|
||||
PROJECTION_RESULT_CODE = "r" * 64
|
||||
|
||||
|
||||
class ReleaseRunError(RuntimeError):
|
||||
@@ -200,32 +202,32 @@ class ReleaseRunStore:
|
||||
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
|
||||
with self._locked():
|
||||
self._ensure_root()
|
||||
paths = sorted(
|
||||
(
|
||||
try:
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.name.endswith(".json")
|
||||
and RUN_ID_PATTERN.fullmatch(path.stem)
|
||||
),
|
||||
key=lambda item: item.name,
|
||||
reverse=True,
|
||||
)
|
||||
]
|
||||
except OSError as exc:
|
||||
raise ReleaseRunCorrupt(
|
||||
"Release run storage could not be enumerated safely."
|
||||
) from exc
|
||||
summaries: list[dict[str, Any]] = []
|
||||
unavailable: list[dict[str, Any]] = []
|
||||
for path in paths:
|
||||
try:
|
||||
record = self._read(path.stem)
|
||||
except ReleaseRunWorkspaceMismatch:
|
||||
continue
|
||||
except (ReleaseRunCorrupt, ReleaseRunNotFound):
|
||||
summaries.append(
|
||||
unavailable.append(
|
||||
{
|
||||
"run_id": path.stem,
|
||||
"status": "unavailable",
|
||||
"error": "Run record is unavailable or failed integrity validation.",
|
||||
}
|
||||
)
|
||||
if len(summaries) >= limit:
|
||||
break
|
||||
continue
|
||||
view = _view(record)
|
||||
immutable_input = view["immutable"]["input"]
|
||||
@@ -240,9 +242,16 @@ class ReleaseRunStore:
|
||||
"recommended_next": view["recommended_next"],
|
||||
}
|
||||
)
|
||||
if len(summaries) >= limit:
|
||||
break
|
||||
return summaries
|
||||
summaries.sort(
|
||||
key=lambda item: (
|
||||
item["updated_at"],
|
||||
item["created_at"],
|
||||
item["run_id"],
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
|
||||
return (summaries + unavailable)[:limit]
|
||||
|
||||
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
|
||||
request_fingerprint = _request_fingerprint(request_id)
|
||||
@@ -386,6 +395,7 @@ class ReleaseRunStore:
|
||||
"An unresolved outcome is already recorded for this attempt."
|
||||
)
|
||||
_ensure_command_capacity(record, reserve_after=1)
|
||||
_ensure_unresolved_recovery_byte_capacity(record, step_id)
|
||||
else:
|
||||
_ensure_command_capacity(record)
|
||||
if outcome == "effect_absent":
|
||||
@@ -446,7 +456,7 @@ class ReleaseRunStore:
|
||||
"The attempt identifier was already used for another release step."
|
||||
)
|
||||
return _view(record)
|
||||
step, _plan_step = _step_pair(record, step_id)
|
||||
step, plan_step = _step_pair(record, step_id)
|
||||
available, reason = _step_available(record, step_id)
|
||||
if not available:
|
||||
raise ReleaseRunConflict(reason)
|
||||
@@ -471,6 +481,11 @@ class ReleaseRunStore:
|
||||
}
|
||||
)
|
||||
_append_event(record, event_type="step_started", step_id=step_id)
|
||||
_ensure_start_recovery_byte_capacity(
|
||||
record,
|
||||
step_id,
|
||||
mutating=bool(plan_step.get("mutating")),
|
||||
)
|
||||
self._persist_update(record)
|
||||
return _view(record)
|
||||
|
||||
@@ -1337,12 +1352,13 @@ def _append_event(
|
||||
event_type: str,
|
||||
step_id: str | None = None,
|
||||
result_code: str | None = None,
|
||||
at: str | None = None,
|
||||
) -> None:
|
||||
if event_type not in EVENT_TYPES:
|
||||
raise ReleaseRunConflict("Release event type is invalid.")
|
||||
event: dict[str, Any] = {
|
||||
"sequence": record["state"]["next_event_sequence"],
|
||||
"at": _timestamp(),
|
||||
"at": at if at is not None else _timestamp(),
|
||||
"type": event_type,
|
||||
}
|
||||
if step_id is not None:
|
||||
@@ -1355,6 +1371,216 @@ def _append_event(
|
||||
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
|
||||
|
||||
|
||||
def _ensure_start_recovery_byte_capacity(
|
||||
record: dict[str, Any], step_id: str, *, mutating: bool
|
||||
) -> None:
|
||||
"""Prove that every required completion/recovery write still fits."""
|
||||
|
||||
started = deepcopy(record)
|
||||
_project_persist(started)
|
||||
projections = [started]
|
||||
|
||||
succeeded = _project_finish(started, step_id, succeeded=True)
|
||||
failed = _project_finish(started, step_id, succeeded=False)
|
||||
projections.extend((succeeded, failed, _project_retry(failed, step_id)))
|
||||
|
||||
interrupted = _project_resume(started)
|
||||
projections.append(interrupted)
|
||||
if mutating:
|
||||
projections.extend(
|
||||
_project_reconcile(interrupted, step_id, outcome=outcome)
|
||||
for outcome in ("effect_absent", "effect_succeeded")
|
||||
)
|
||||
else:
|
||||
projections.append(_project_retry(interrupted, step_id))
|
||||
_ensure_projected_records_fit(
|
||||
projections,
|
||||
message=(
|
||||
"Release run has insufficient serialized recovery capacity for a "
|
||||
"new attempt; create a new run."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_unresolved_recovery_byte_capacity(
|
||||
record: dict[str, Any], step_id: str
|
||||
) -> None:
|
||||
"""Accept an advisory unresolved write only if a terminal write remains."""
|
||||
|
||||
unresolved = _project_reconcile(record, step_id, outcome="unresolved")
|
||||
projections = [unresolved]
|
||||
projections.extend(
|
||||
_project_reconcile(unresolved, step_id, outcome=outcome)
|
||||
for outcome in ("effect_absent", "effect_succeeded")
|
||||
)
|
||||
_ensure_projected_records_fit(
|
||||
projections,
|
||||
message=(
|
||||
"The unresolved outcome would consume capacity reserved for a "
|
||||
"terminal reconciliation."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _project_finish(
|
||||
record: dict[str, Any], step_id: str, *, succeeded: bool
|
||||
) -> dict[str, Any]:
|
||||
projected = deepcopy(record)
|
||||
step, _plan_step = _step_pair(projected, step_id)
|
||||
attempt = _attempt_for_fingerprint(
|
||||
projected, step["attempt_fingerprint"], step_id=step_id
|
||||
)
|
||||
target_state = "succeeded" if succeeded else "failed"
|
||||
step.update(
|
||||
{
|
||||
"state": target_state,
|
||||
"finished_at": PROJECTION_TIMESTAMP,
|
||||
"result_code": PROJECTION_RESULT_CODE,
|
||||
}
|
||||
)
|
||||
attempt["outcome"] = target_state
|
||||
attempt["result_code"] = PROJECTION_RESULT_CODE
|
||||
_append_event(
|
||||
projected,
|
||||
event_type="step_succeeded" if succeeded else "step_failed",
|
||||
step_id=step_id,
|
||||
result_code=PROJECTION_RESULT_CODE,
|
||||
at=PROJECTION_TIMESTAMP,
|
||||
)
|
||||
_project_persist(projected)
|
||||
return projected
|
||||
|
||||
|
||||
def _project_resume(record: dict[str, Any]) -> dict[str, Any]:
|
||||
projected = deepcopy(record)
|
||||
running_steps = [
|
||||
step for step in projected["state"]["steps"] if step["state"] == "running"
|
||||
]
|
||||
for step in running_steps:
|
||||
attempt = _attempt_for_fingerprint(
|
||||
projected, step["attempt_fingerprint"], step_id=step["id"]
|
||||
)
|
||||
attempt["outcome"] = "interrupted"
|
||||
attempt["result_code"] = "process_interrupted"
|
||||
step["state"] = "interrupted"
|
||||
step["finished_at"] = PROJECTION_TIMESTAMP
|
||||
step["result_code"] = "process_interrupted"
|
||||
_append_event(
|
||||
projected,
|
||||
event_type="step_interrupted",
|
||||
step_id=step["id"],
|
||||
result_code="process_interrupted",
|
||||
at=PROJECTION_TIMESTAMP,
|
||||
)
|
||||
_remember_request(
|
||||
projected,
|
||||
"0" * 64,
|
||||
"resume",
|
||||
None,
|
||||
None,
|
||||
attempt_fingerprint=running_steps[0]["attempt_fingerprint"],
|
||||
)
|
||||
_append_event(projected, event_type="run_resumed", at=PROJECTION_TIMESTAMP)
|
||||
_project_persist(projected)
|
||||
return projected
|
||||
|
||||
|
||||
def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]:
|
||||
projected = deepcopy(record)
|
||||
step, _plan_step = _step_pair(projected, step_id)
|
||||
attempt_fingerprint = step["attempt_fingerprint"]
|
||||
retry_reason = (
|
||||
"known_failure"
|
||||
if step["state"] == "failed"
|
||||
else "read_only_interruption"
|
||||
)
|
||||
step.update(
|
||||
{
|
||||
"state": "pending",
|
||||
"attempt_fingerprint": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"result_code": None,
|
||||
}
|
||||
)
|
||||
_remember_request(
|
||||
projected,
|
||||
"1" * 64,
|
||||
"retry",
|
||||
step_id,
|
||||
None,
|
||||
attempt_fingerprint=attempt_fingerprint,
|
||||
)
|
||||
_append_event(
|
||||
projected,
|
||||
event_type="step_retry_requested",
|
||||
step_id=step_id,
|
||||
result_code=retry_reason,
|
||||
at=PROJECTION_TIMESTAMP,
|
||||
)
|
||||
_project_persist(projected)
|
||||
return projected
|
||||
|
||||
|
||||
def _project_reconcile(
|
||||
record: dict[str, Any], step_id: str, *, outcome: str
|
||||
) -> dict[str, Any]:
|
||||
projected = deepcopy(record)
|
||||
step, _plan_step = _step_pair(projected, step_id)
|
||||
attempt_fingerprint = step["attempt_fingerprint"]
|
||||
if outcome == "effect_absent":
|
||||
step.update(
|
||||
{
|
||||
"state": "pending",
|
||||
"attempt_fingerprint": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"result_code": None,
|
||||
}
|
||||
)
|
||||
elif outcome == "effect_succeeded":
|
||||
step.update(
|
||||
{
|
||||
"state": "succeeded",
|
||||
"finished_at": PROJECTION_TIMESTAMP,
|
||||
"result_code": "reconciled_effect_succeeded",
|
||||
}
|
||||
)
|
||||
_remember_request(
|
||||
projected,
|
||||
"2" * 64 if outcome == "unresolved" else "3" * 64,
|
||||
"reconcile",
|
||||
step_id,
|
||||
outcome,
|
||||
attempt_fingerprint=attempt_fingerprint,
|
||||
)
|
||||
_append_event(
|
||||
projected,
|
||||
event_type="step_reconciled",
|
||||
step_id=step_id,
|
||||
result_code=outcome,
|
||||
at=PROJECTION_TIMESTAMP,
|
||||
)
|
||||
_project_persist(projected)
|
||||
return projected
|
||||
|
||||
|
||||
def _project_persist(record: dict[str, Any]) -> None:
|
||||
record["updated_at"] = PROJECTION_TIMESTAMP
|
||||
_refresh_status(record)
|
||||
record["record_digest"] = "f" * 64
|
||||
|
||||
|
||||
def _ensure_projected_records_fit(
|
||||
records: list[dict[str, Any]], *, message: str
|
||||
) -> None:
|
||||
if any(
|
||||
len(_canonical_json(projected)) + 1 > MAX_RECORD_BYTES
|
||||
for projected in records
|
||||
):
|
||||
raise ReleaseRunConflict(message)
|
||||
|
||||
|
||||
def _ensure_command_capacity(
|
||||
record: dict[str, Any], *, reserve_after: int = 0
|
||||
) -> None:
|
||||
|
||||
@@ -1006,10 +1006,12 @@
|
||||
renderReleaseRunStoreUnavailable(runsResult.error);
|
||||
} else {
|
||||
renderReleaseRunList(runsResult.data?.runs || []);
|
||||
const pendingRecoveryResult = await recoverPendingReleaseRun();
|
||||
if (pendingRecoveryResult === null && releaseState.currentRun?.run_id) {
|
||||
const pendingCreateResult = await recoverPendingReleaseRun();
|
||||
const pendingCommandResult = await recoverPendingRunCommands();
|
||||
const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null;
|
||||
if (!recoveryAttempted && releaseState.currentRun?.run_id) {
|
||||
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
|
||||
} else if (pendingRecoveryResult === null) {
|
||||
} else if (!recoveryAttempted) {
|
||||
renderIdlePlan();
|
||||
}
|
||||
}
|
||||
@@ -1262,6 +1264,7 @@
|
||||
releaseState.currentRun = resumed;
|
||||
renderReleaseRun(resumed);
|
||||
} catch (error) {
|
||||
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||
elements.runStatus.textContent = "error";
|
||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||
} finally {
|
||||
@@ -1280,6 +1283,7 @@
|
||||
releaseState.currentRun = retried;
|
||||
renderReleaseRun(retried);
|
||||
} catch (error) {
|
||||
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||
}
|
||||
}
|
||||
@@ -1300,6 +1304,7 @@
|
||||
releaseState.currentRun = reconciled;
|
||||
renderReleaseRun(reconciled);
|
||||
} catch (error) {
|
||||
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||
button.disabled = false;
|
||||
}
|
||||
@@ -1315,6 +1320,88 @@
|
||||
}
|
||||
}
|
||||
|
||||
function pendingRunCommandRequest(commandKey, requestIdToReplay) {
|
||||
if (typeof commandKey !== "string" || typeof requestIdToReplay !== "string") return null;
|
||||
if (commandKey.startsWith("resume:")) {
|
||||
const runId = commandKey.slice("resume:".length);
|
||||
if (!runId || runId.includes(":")) return null;
|
||||
return {
|
||||
path: `/api/release-runs/${encodeURIComponent(runId)}/resume`,
|
||||
body: { request_id: requestIdToReplay },
|
||||
};
|
||||
}
|
||||
if (commandKey.startsWith("retry:")) {
|
||||
const remainder = commandKey.slice("retry:".length);
|
||||
const separator = remainder.indexOf(":");
|
||||
const runId = remainder.slice(0, separator);
|
||||
const stepId = remainder.slice(separator + 1);
|
||||
if (separator < 1 || !stepId) return null;
|
||||
return {
|
||||
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/retry`,
|
||||
body: { request_id: requestIdToReplay },
|
||||
};
|
||||
}
|
||||
if (commandKey.startsWith("reconcile:")) {
|
||||
const remainder = commandKey.slice("reconcile:".length);
|
||||
const runSeparator = remainder.indexOf(":");
|
||||
const outcomeSeparator = remainder.lastIndexOf(":");
|
||||
const runId = remainder.slice(0, runSeparator);
|
||||
const stepId = remainder.slice(runSeparator + 1, outcomeSeparator);
|
||||
const outcome = remainder.slice(outcomeSeparator + 1);
|
||||
if (runSeparator < 1 || outcomeSeparator <= runSeparator + 1 || !["effect_absent", "effect_succeeded", "unresolved"].includes(outcome)) return null;
|
||||
return {
|
||||
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/reconcile`,
|
||||
body: { request_id: requestIdToReplay, outcome, confirm: "RECONCILE" },
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function recoverPendingRunCommands() {
|
||||
const commands = readInFlightRunCommands();
|
||||
const entries = Object.entries(commands);
|
||||
if (!entries.length) return null;
|
||||
let attempted = false;
|
||||
let recoveredRun = null;
|
||||
let pendingError = null;
|
||||
let rejectedError = null;
|
||||
for (const [commandKey, requestIdToReplay] of entries) {
|
||||
const command = pendingRunCommandRequest(commandKey, requestIdToReplay);
|
||||
if (!command) {
|
||||
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||
continue;
|
||||
}
|
||||
attempted = true;
|
||||
try {
|
||||
recoveredRun = await postJson(command.path, command.body);
|
||||
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||
} catch (error) {
|
||||
if (deterministicRunCommandError(error)) {
|
||||
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||
rejectedError = error;
|
||||
} else {
|
||||
pendingError = error;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (recoveredRun) {
|
||||
releaseState.currentRun = recoveredRun;
|
||||
restoreReleaseRunSelection(recoveredRun);
|
||||
renderReleaseRun(recoveredRun);
|
||||
elements.releaseRun.value = recoveredRun.run_id;
|
||||
}
|
||||
if (pendingError) {
|
||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("pending", "warn")} Saved command recovery is still pending</h3><p>${escapeHtml(pendingError.message)}</p><p class="action-meta">The same private request identifier will be replayed after refresh.</p></div>`);
|
||||
} else if (rejectedError) {
|
||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("rejected", "block")} Saved command was not accepted</h3><p>${escapeHtml(rejectedError.message)}</p><p class="action-meta">The server returned a deterministic client conflict; the stale request identifier was cleared.</p></div>`);
|
||||
}
|
||||
return attempted ? Boolean(recoveredRun) : null;
|
||||
}
|
||||
|
||||
function deterministicRunCommandError(error) {
|
||||
return error?.status >= 400 && error.status < 500;
|
||||
}
|
||||
|
||||
function inFlightRunCommandId(commandKey, prefix) {
|
||||
const commands = readInFlightRunCommands();
|
||||
if (typeof commands[commandKey] === "string") return commands[commandKey];
|
||||
|
||||
Reference in New Issue
Block a user