fix(release): preserve durable recovery capacity

This commit is contained in:
2026-07-22 19:48:55 +02:00
parent 22f5c2ff4e
commit 4c16069f88
4 changed files with 537 additions and 33 deletions

View File

@@ -108,13 +108,17 @@ commands, process output, confirmation text, credentials, bearer tokens, or
signing material. signing material.
Before a step can enter `running`, the store reserves the two command-ledger Before a step can enter `running`, the store reserves the two command-ledger
slots needed for worst-case recovery. An explicit resume consumes one slot and slots needed for worst-case recovery. It also projects the serialized record
is accepted only while a persisted attempt is actually running. A mutating through start, finish/failure, resume, and the required terminal reconciliation
attempt always retains its final `effect_absent` or `effect_succeeded` slot; or read-only retry; the start is rejected unless every required atomic write
fits the record-size bound. An explicit resume consumes one slot and is
accepted only while a persisted attempt is actually running. A mutating attempt
always retains its final `effect_absent` or `effect_succeeded` slot;
`unresolved` may be recorded at most once for that attempt and only when an `unresolved` may be recorded at most once for that attempt and only when an
additional slot is available. Read-only interruption and known failure retain additional ledger slot and serialized terminal-write capacity are available.
the slot needed to prepare a retry. Capacity exhaustion is therefore detected Read-only interruption and known failure retain the slot and byte capacity
before starting an effect rather than stranding an ambiguous attempt. needed to prepare a retry. Capacity exhaustion is therefore detected before
starting an effect rather than stranding an ambiguous attempt.
An explicit resume after a process restart converts every persisted `running` An explicit resume after a process restart converts every persisted `running`
step to `interrupted`; it never guesses whether an external effect happened. step to `interrupted`; it never guesses whether an external effect happened.
@@ -128,16 +132,20 @@ code-only state event. A known failed attempt can likewise be prepared for
retry. The UI keeps unavailable controls visible and disabled. retry. The UI keeps unavailable controls visible and disabled.
The browser likewise retains the request identifier for an uncertain The browser likewise retains the request identifier for an uncertain
resume/retry/reconciliation response and reuses it on the operator's retry. resume/retry/reconciliation response and replays it after reload. A successful
Identifiers are cleared only after the server returns the resulting run state. replay selects the returned run state. Transport and server failures retain the
identifier; a deterministic `4xx` rejection clears it so a stale command cannot
poison a later attempt.
The run API is covered by the same local console token middleware as every The run API is covered by the same local console token middleware as every
other `/api/` route: other `/api/` route:
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and - `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
freezes a selective plan only when that creation is not already known. freezes a selective plan only when that creation is not already known.
- `GET /api/release-runs` lists bounded summaries; unreadable entries are - `GET /api/release-runs` lists bounded summaries ordered by verified
reported as unavailable instead of being silently omitted. `updated_at`, then `created_at` and run identifier. Unreadable entries are
appended deterministically when room remains instead of displacing newer
verified runs.
- `GET /api/release-runs/{run_id}` reads and verifies one exact record. - `GET /api/release-runs/{run_id}` reads and verifies one exact record.
- `POST /api/release-runs/{run_id}/resume` records explicit recovery. - `POST /api/release-runs/{run_id}/resume` records explicit recovery.
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or - `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or

View File

@@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path:
from govoplan_release.release_run import ( # noqa: E402 from govoplan_release.release_run import ( # noqa: E402
MAX_EVENTS, MAX_EVENTS,
MAX_RECORD_BYTES,
ReleaseRunConflict, ReleaseRunConflict,
ReleaseRunCorrupt, ReleaseRunCorrupt,
ReleaseRunNotFound, ReleaseRunNotFound,
@@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase):
) )
self.assertEqual("completed", completed["state"]["status"]) self.assertEqual("completed", completed["state"]["status"])
def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None:
rejected_plan = mutating_first_plan()
rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)]
rejected_run_id = create_run(
self.store,
request_id="byte-capacity-rejected-create-0001",
plan_snapshot=rejected_plan,
)["run_id"]
rejected_path = self.root / f"{rejected_run_id}.json"
self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000)
with self.assertRaisesRegex(
ReleaseRunConflict, "serialized recovery capacity"
):
self.store.start_step(
rejected_run_id,
"core:tag",
attempt_id="byte-capacity-rejected-attempt-0001",
)
untouched = self.store.get(rejected_run_id)
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
accepted_plan = mutating_first_plan()
accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
accepted_run_id = create_run(
self.store,
request_id="byte-capacity-accepted-create-0001",
plan_snapshot=accepted_plan,
)["run_id"]
accepted_path = self.root / f"{accepted_run_id}.json"
self.store.start_step(
accepted_run_id,
"core:tag",
attempt_id="byte-capacity-accepted-attempt-0001",
)
self.store.resume(
accepted_run_id,
request_id="byte-capacity-accepted-resume-0001",
)
with self.assertRaisesRegex(
ReleaseRunConflict, "reserved for a terminal reconciliation"
):
self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
completed = self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-terminal-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("completed", completed["state"]["status"])
self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES)
def test_read_only_start_reserves_serialized_resume_and_retry_capacity(
self,
) -> None:
plan = mutating_first_plan()
plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index]
plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
run_id = create_run(
self.store,
request_id="byte-capacity-read-only-create-0001",
plan_snapshot=plan,
)["run_id"]
self.store.start_step(
run_id,
"core:tag",
attempt_id="byte-capacity-read-only-attempt-0001",
)
self.store.resume(
run_id,
request_id="byte-capacity-read-only-resume-0001",
)
retried = self.store.retry_step(
run_id,
"core:tag",
request_id="byte-capacity-read-only-retry-0001",
)
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None: def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
run_id = create_run( run_id = create_run(
self.store, self.store,
@@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase):
listed = self.store.list(limit=1) listed = self.store.list(limit=1)
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed]) self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable(
self,
) -> None:
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:00Z",
):
older = create_run(
self.store,
request_id="timestamp-order-request-older-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T01:00:00Z",
):
newer = create_run(
self.store,
request_id="timestamp-order-request-newer-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T02:00:00Z",
):
self.store.start_step(
older["run_id"],
"core:preflight",
attempt_id="timestamp-order-attempt-0001",
)
corrupt = create_run(
self.store,
request_id="timestamp-order-request-corrupt-0001",
)
corrupt_path = self.root / f"{corrupt['run_id']}.json"
corrupt_path.write_text("{}", encoding="utf-8")
corrupt_path.chmod(0o600)
listed = self.store.list(limit=3)
self.assertEqual(
[older["run_id"], newer["run_id"], corrupt["run_id"]],
[item["run_id"] for item in listed],
)
self.assertEqual("unavailable", listed[-1]["status"])
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
def test_list_enumeration_oserror_is_normalized(self) -> None:
self.store.list()
with patch.object(
Path,
"iterdir",
side_effect=PermissionError("denied"),
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"):
self.store.list()
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None: def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
run_id = create_run( run_id = create_run(
self.store, self.store,
@@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertIn('const runsRequest = api("/api/release-runs")', webui) self.assertIn('const runsRequest = api("/api/release-runs")', webui)
self.assertIn("pendingReleaseRunPayload", webui) self.assertIn("pendingReleaseRunPayload", webui)
self.assertIn("recoverPendingReleaseRun", webui) self.assertIn("recoverPendingReleaseRun", webui)
self.assertIn("recoverPendingRunCommands", webui)
self.assertIn("inFlightRunCommandId", webui) self.assertIn("inFlightRunCommandId", webui)
self.assertIn("Run saved; list refresh unavailable", webui) self.assertIn("Run saved; list refresh unavailable", webui)
self.assertIn("This foundation tracks state only.", runbook) self.assertIn("This foundation tracks state only.", runbook)
@@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase):
"submitReleaseRunCreate", "submitReleaseRunCreate",
"refreshReleaseRunListAfterCreate", "refreshReleaseRunListAfterCreate",
"readInFlightRunCommands", "readInFlightRunCommands",
"pendingRunCommandRequest",
"recoverPendingRunCommands",
"deterministicRunCommandError",
"inFlightRunCommandId", "inFlightRunCommandId",
"clearInFlightRunCommand", "clearInFlightRunCommand",
"requestId", "requestId",
@@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru
const renderReleaseRunList = () => {}; const renderReleaseRunList = () => {};
let failCreate = true; let failCreate = true;
let failList = true; let failList = true;
let failureStatus = null;
const postedRequestIds = []; const postedRequestIds = [];
async function postJson(_path, payload) { const postedPaths = [];
async function postJson(path, payload) {
postedPaths.push(path);
postedRequestIds.push(payload.request_id); postedRequestIds.push(payload.request_id);
if (failCreate) throw new Error("uncertain transport failure"); if (failCreate) {
const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure");
if (failureStatus) error.status = failureStatus;
throw error;
}
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } }; return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
} }
async function api(_path) { async function api(_path) {
@@ -1194,11 +1348,40 @@ async function api(_path) {
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure"); assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained"); assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry"); const commandKey = "retry:rr-known:core:preflight";
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry"); const commandOne = inFlightRunCommandId(commandKey, "retry");
const commandReplay = inFlightRunCommandId(commandKey, "retry");
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID"); assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
clearInFlightRunCommand("retry:rr-known:step", commandOne); const retryRequest = pendingRunCommandRequest(commandKey, commandOne);
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry"); assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable");
const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request");
assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable");
assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed");
failCreate = true;
const uncertainCommand = await recoverPendingRunCommands();
assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared");
failCreate = false;
const recoveredCommand = await recoverPendingRunCommands();
assert(recoveredCommand === true, "pending command did not recover after reload");
assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared");
assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID");
assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint");
assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run");
const conflictKey = "resume:rr-known";
const conflictId = inFlightRunCommandId(conflictKey, "resume");
failCreate = true;
failureStatus = 409;
const rejectedCommand = await recoverPendingRunCommands();
assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID");
assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained");
failCreate = false;
failureStatus = null;
const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry");
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID"); assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); }); })().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
""" """

View File

@@ -59,6 +59,8 @@ ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"}
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"} RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
RECONCILIATION_CONFIRMATION = "RECONCILE" RECONCILIATION_CONFIRMATION = "RECONCILE"
START_RECOVERY_RESERVE = 2 START_RECOVERY_RESERVE = 2
PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z"
PROJECTION_RESULT_CODE = "r" * 64
class ReleaseRunError(RuntimeError): class ReleaseRunError(RuntimeError):
@@ -200,32 +202,32 @@ class ReleaseRunStore:
limit = min(max(int(limit), 1), MAX_LIST_LIMIT) limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
with self._locked(): with self._locked():
self._ensure_root() self._ensure_root()
paths = sorted( try:
( paths = [
path path
for path in self.root.iterdir() for path in self.root.iterdir()
if path.name.endswith(".json") if path.name.endswith(".json")
and RUN_ID_PATTERN.fullmatch(path.stem) and RUN_ID_PATTERN.fullmatch(path.stem)
), ]
key=lambda item: item.name, except OSError as exc:
reverse=True, raise ReleaseRunCorrupt(
) "Release run storage could not be enumerated safely."
) from exc
summaries: list[dict[str, Any]] = [] summaries: list[dict[str, Any]] = []
unavailable: list[dict[str, Any]] = []
for path in paths: for path in paths:
try: try:
record = self._read(path.stem) record = self._read(path.stem)
except ReleaseRunWorkspaceMismatch: except ReleaseRunWorkspaceMismatch:
continue continue
except (ReleaseRunCorrupt, ReleaseRunNotFound): except (ReleaseRunCorrupt, ReleaseRunNotFound):
summaries.append( unavailable.append(
{ {
"run_id": path.stem, "run_id": path.stem,
"status": "unavailable", "status": "unavailable",
"error": "Run record is unavailable or failed integrity validation.", "error": "Run record is unavailable or failed integrity validation.",
} }
) )
if len(summaries) >= limit:
break
continue continue
view = _view(record) view = _view(record)
immutable_input = view["immutable"]["input"] immutable_input = view["immutable"]["input"]
@@ -240,9 +242,16 @@ class ReleaseRunStore:
"recommended_next": view["recommended_next"], "recommended_next": view["recommended_next"],
} }
) )
if len(summaries) >= limit: summaries.sort(
break key=lambda item: (
return summaries item["updated_at"],
item["created_at"],
item["run_id"],
),
reverse=True,
)
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
return (summaries + unavailable)[:limit]
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]: def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
request_fingerprint = _request_fingerprint(request_id) request_fingerprint = _request_fingerprint(request_id)
@@ -386,6 +395,7 @@ class ReleaseRunStore:
"An unresolved outcome is already recorded for this attempt." "An unresolved outcome is already recorded for this attempt."
) )
_ensure_command_capacity(record, reserve_after=1) _ensure_command_capacity(record, reserve_after=1)
_ensure_unresolved_recovery_byte_capacity(record, step_id)
else: else:
_ensure_command_capacity(record) _ensure_command_capacity(record)
if outcome == "effect_absent": if outcome == "effect_absent":
@@ -446,7 +456,7 @@ class ReleaseRunStore:
"The attempt identifier was already used for another release step." "The attempt identifier was already used for another release step."
) )
return _view(record) return _view(record)
step, _plan_step = _step_pair(record, step_id) step, plan_step = _step_pair(record, step_id)
available, reason = _step_available(record, step_id) available, reason = _step_available(record, step_id)
if not available: if not available:
raise ReleaseRunConflict(reason) raise ReleaseRunConflict(reason)
@@ -471,6 +481,11 @@ class ReleaseRunStore:
} }
) )
_append_event(record, event_type="step_started", step_id=step_id) _append_event(record, event_type="step_started", step_id=step_id)
_ensure_start_recovery_byte_capacity(
record,
step_id,
mutating=bool(plan_step.get("mutating")),
)
self._persist_update(record) self._persist_update(record)
return _view(record) return _view(record)
@@ -1337,12 +1352,13 @@ def _append_event(
event_type: str, event_type: str,
step_id: str | None = None, step_id: str | None = None,
result_code: str | None = None, result_code: str | None = None,
at: str | None = None,
) -> None: ) -> None:
if event_type not in EVENT_TYPES: if event_type not in EVENT_TYPES:
raise ReleaseRunConflict("Release event type is invalid.") raise ReleaseRunConflict("Release event type is invalid.")
event: dict[str, Any] = { event: dict[str, Any] = {
"sequence": record["state"]["next_event_sequence"], "sequence": record["state"]["next_event_sequence"],
"at": _timestamp(), "at": at if at is not None else _timestamp(),
"type": event_type, "type": event_type,
} }
if step_id is not None: if step_id is not None:
@@ -1355,6 +1371,216 @@ def _append_event(
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:] record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
def _ensure_start_recovery_byte_capacity(
record: dict[str, Any], step_id: str, *, mutating: bool
) -> None:
"""Prove that every required completion/recovery write still fits."""
started = deepcopy(record)
_project_persist(started)
projections = [started]
succeeded = _project_finish(started, step_id, succeeded=True)
failed = _project_finish(started, step_id, succeeded=False)
projections.extend((succeeded, failed, _project_retry(failed, step_id)))
interrupted = _project_resume(started)
projections.append(interrupted)
if mutating:
projections.extend(
_project_reconcile(interrupted, step_id, outcome=outcome)
for outcome in ("effect_absent", "effect_succeeded")
)
else:
projections.append(_project_retry(interrupted, step_id))
_ensure_projected_records_fit(
projections,
message=(
"Release run has insufficient serialized recovery capacity for a "
"new attempt; create a new run."
),
)
def _ensure_unresolved_recovery_byte_capacity(
record: dict[str, Any], step_id: str
) -> None:
"""Accept an advisory unresolved write only if a terminal write remains."""
unresolved = _project_reconcile(record, step_id, outcome="unresolved")
projections = [unresolved]
projections.extend(
_project_reconcile(unresolved, step_id, outcome=outcome)
for outcome in ("effect_absent", "effect_succeeded")
)
_ensure_projected_records_fit(
projections,
message=(
"The unresolved outcome would consume capacity reserved for a "
"terminal reconciliation."
),
)
def _project_finish(
record: dict[str, Any], step_id: str, *, succeeded: bool
) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt = _attempt_for_fingerprint(
projected, step["attempt_fingerprint"], step_id=step_id
)
target_state = "succeeded" if succeeded else "failed"
step.update(
{
"state": target_state,
"finished_at": PROJECTION_TIMESTAMP,
"result_code": PROJECTION_RESULT_CODE,
}
)
attempt["outcome"] = target_state
attempt["result_code"] = PROJECTION_RESULT_CODE
_append_event(
projected,
event_type="step_succeeded" if succeeded else "step_failed",
step_id=step_id,
result_code=PROJECTION_RESULT_CODE,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_resume(record: dict[str, Any]) -> dict[str, Any]:
projected = deepcopy(record)
running_steps = [
step for step in projected["state"]["steps"] if step["state"] == "running"
]
for step in running_steps:
attempt = _attempt_for_fingerprint(
projected, step["attempt_fingerprint"], step_id=step["id"]
)
attempt["outcome"] = "interrupted"
attempt["result_code"] = "process_interrupted"
step["state"] = "interrupted"
step["finished_at"] = PROJECTION_TIMESTAMP
step["result_code"] = "process_interrupted"
_append_event(
projected,
event_type="step_interrupted",
step_id=step["id"],
result_code="process_interrupted",
at=PROJECTION_TIMESTAMP,
)
_remember_request(
projected,
"0" * 64,
"resume",
None,
None,
attempt_fingerprint=running_steps[0]["attempt_fingerprint"],
)
_append_event(projected, event_type="run_resumed", at=PROJECTION_TIMESTAMP)
_project_persist(projected)
return projected
def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt_fingerprint = step["attempt_fingerprint"]
retry_reason = (
"known_failure"
if step["state"] == "failed"
else "read_only_interruption"
)
step.update(
{
"state": "pending",
"attempt_fingerprint": None,
"started_at": None,
"finished_at": None,
"result_code": None,
}
)
_remember_request(
projected,
"1" * 64,
"retry",
step_id,
None,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
projected,
event_type="step_retry_requested",
step_id=step_id,
result_code=retry_reason,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_reconcile(
record: dict[str, Any], step_id: str, *, outcome: str
) -> dict[str, Any]:
projected = deepcopy(record)
step, _plan_step = _step_pair(projected, step_id)
attempt_fingerprint = step["attempt_fingerprint"]
if outcome == "effect_absent":
step.update(
{
"state": "pending",
"attempt_fingerprint": None,
"started_at": None,
"finished_at": None,
"result_code": None,
}
)
elif outcome == "effect_succeeded":
step.update(
{
"state": "succeeded",
"finished_at": PROJECTION_TIMESTAMP,
"result_code": "reconciled_effect_succeeded",
}
)
_remember_request(
projected,
"2" * 64 if outcome == "unresolved" else "3" * 64,
"reconcile",
step_id,
outcome,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
projected,
event_type="step_reconciled",
step_id=step_id,
result_code=outcome,
at=PROJECTION_TIMESTAMP,
)
_project_persist(projected)
return projected
def _project_persist(record: dict[str, Any]) -> None:
record["updated_at"] = PROJECTION_TIMESTAMP
_refresh_status(record)
record["record_digest"] = "f" * 64
def _ensure_projected_records_fit(
records: list[dict[str, Any]], *, message: str
) -> None:
if any(
len(_canonical_json(projected)) + 1 > MAX_RECORD_BYTES
for projected in records
):
raise ReleaseRunConflict(message)
def _ensure_command_capacity( def _ensure_command_capacity(
record: dict[str, Any], *, reserve_after: int = 0 record: dict[str, Any], *, reserve_after: int = 0
) -> None: ) -> None:

View File

@@ -1006,10 +1006,12 @@
renderReleaseRunStoreUnavailable(runsResult.error); renderReleaseRunStoreUnavailable(runsResult.error);
} else { } else {
renderReleaseRunList(runsResult.data?.runs || []); renderReleaseRunList(runsResult.data?.runs || []);
const pendingRecoveryResult = await recoverPendingReleaseRun(); const pendingCreateResult = await recoverPendingReleaseRun();
if (pendingRecoveryResult === null && releaseState.currentRun?.run_id) { const pendingCommandResult = await recoverPendingRunCommands();
const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null;
if (!recoveryAttempted && releaseState.currentRun?.run_id) {
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false }); await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
} else if (pendingRecoveryResult === null) { } else if (!recoveryAttempted) {
renderIdlePlan(); renderIdlePlan();
} }
} }
@@ -1262,6 +1264,7 @@
releaseState.currentRun = resumed; releaseState.currentRun = resumed;
renderReleaseRun(resumed); renderReleaseRun(resumed);
} catch (error) { } catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runStatus.textContent = "error"; elements.runStatus.textContent = "error";
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`); elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
} finally { } finally {
@@ -1280,6 +1283,7 @@
releaseState.currentRun = retried; releaseState.currentRun = retried;
renderReleaseRun(retried); renderReleaseRun(retried);
} catch (error) { } catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`); elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
} }
} }
@@ -1300,6 +1304,7 @@
releaseState.currentRun = reconciled; releaseState.currentRun = reconciled;
renderReleaseRun(reconciled); renderReleaseRun(reconciled);
} catch (error) { } catch (error) {
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`); elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
button.disabled = false; button.disabled = false;
} }
@@ -1315,6 +1320,88 @@
} }
} }
function pendingRunCommandRequest(commandKey, requestIdToReplay) {
if (typeof commandKey !== "string" || typeof requestIdToReplay !== "string") return null;
if (commandKey.startsWith("resume:")) {
const runId = commandKey.slice("resume:".length);
if (!runId || runId.includes(":")) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/resume`,
body: { request_id: requestIdToReplay },
};
}
if (commandKey.startsWith("retry:")) {
const remainder = commandKey.slice("retry:".length);
const separator = remainder.indexOf(":");
const runId = remainder.slice(0, separator);
const stepId = remainder.slice(separator + 1);
if (separator < 1 || !stepId) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/retry`,
body: { request_id: requestIdToReplay },
};
}
if (commandKey.startsWith("reconcile:")) {
const remainder = commandKey.slice("reconcile:".length);
const runSeparator = remainder.indexOf(":");
const outcomeSeparator = remainder.lastIndexOf(":");
const runId = remainder.slice(0, runSeparator);
const stepId = remainder.slice(runSeparator + 1, outcomeSeparator);
const outcome = remainder.slice(outcomeSeparator + 1);
if (runSeparator < 1 || outcomeSeparator <= runSeparator + 1 || !["effect_absent", "effect_succeeded", "unresolved"].includes(outcome)) return null;
return {
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/reconcile`,
body: { request_id: requestIdToReplay, outcome, confirm: "RECONCILE" },
};
}
return null;
}
async function recoverPendingRunCommands() {
const commands = readInFlightRunCommands();
const entries = Object.entries(commands);
if (!entries.length) return null;
let attempted = false;
let recoveredRun = null;
let pendingError = null;
let rejectedError = null;
for (const [commandKey, requestIdToReplay] of entries) {
const command = pendingRunCommandRequest(commandKey, requestIdToReplay);
if (!command) {
clearInFlightRunCommand(commandKey, requestIdToReplay);
continue;
}
attempted = true;
try {
recoveredRun = await postJson(command.path, command.body);
clearInFlightRunCommand(commandKey, requestIdToReplay);
} catch (error) {
if (deterministicRunCommandError(error)) {
clearInFlightRunCommand(commandKey, requestIdToReplay);
rejectedError = error;
} else {
pendingError = error;
}
}
}
if (recoveredRun) {
releaseState.currentRun = recoveredRun;
restoreReleaseRunSelection(recoveredRun);
renderReleaseRun(recoveredRun);
elements.releaseRun.value = recoveredRun.run_id;
}
if (pendingError) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("pending", "warn")} Saved command recovery is still pending</h3><p>${escapeHtml(pendingError.message)}</p><p class="action-meta">The same private request identifier will be replayed after refresh.</p></div>`);
} else if (rejectedError) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("rejected", "block")} Saved command was not accepted</h3><p>${escapeHtml(rejectedError.message)}</p><p class="action-meta">The server returned a deterministic client conflict; the stale request identifier was cleared.</p></div>`);
}
return attempted ? Boolean(recoveredRun) : null;
}
function deterministicRunCommandError(error) {
return error?.status >= 400 && error.status < 500;
}
function inFlightRunCommandId(commandKey, prefix) { function inFlightRunCommandId(commandKey, prefix) {
const commands = readInFlightRunCommands(); const commands = readInFlightRunCommands();
if (typeof commands[commandKey] === "string") return commands[commandKey]; if (typeof commands[commandKey] === "string") return commands[commandKey];