diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index ca8c3a1..cb6a64a 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -108,13 +108,17 @@ commands, process output, confirmation text, credentials, bearer tokens, or signing material. Before a step can enter `running`, the store reserves the two command-ledger -slots needed for worst-case recovery. An explicit resume consumes one slot and -is accepted only while a persisted attempt is actually running. A mutating -attempt always retains its final `effect_absent` or `effect_succeeded` slot; +slots needed for worst-case recovery. It also projects the serialized record +through start, finish/failure, resume, and the required terminal reconciliation +or read-only retry; the start is rejected unless every required atomic write +fits the record-size bound. An explicit resume consumes one slot and is +accepted only while a persisted attempt is actually running. A mutating attempt +always retains its final `effect_absent` or `effect_succeeded` slot; `unresolved` may be recorded at most once for that attempt and only when an -additional slot is available. Read-only interruption and known failure retain -the slot needed to prepare a retry. Capacity exhaustion is therefore detected -before starting an effect rather than stranding an ambiguous attempt. +additional ledger slot and serialized terminal-write capacity are available. +Read-only interruption and known failure retain the slot and byte capacity +needed to prepare a retry. Capacity exhaustion is therefore detected before +starting an effect rather than stranding an ambiguous attempt. An explicit resume after a process restart converts every persisted `running` step to `interrupted`; it never guesses whether an external effect happened. @@ -128,16 +132,20 @@ code-only state event. A known failed attempt can likewise be prepared for retry. The UI keeps unavailable controls visible and disabled. The browser likewise retains the request identifier for an uncertain -resume/retry/reconciliation response and reuses it on the operator's retry. -Identifiers are cleared only after the server returns the resulting run state. +resume/retry/reconciliation response and replays it after reload. A successful +replay selects the returned run state. Transport and server failures retain the +identifier; a deterministic `4xx` rejection clears it so a stale command cannot +poison a later attempt. The run API is covered by the same local console token middleware as every other `/api/` route: - `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and freezes a selective plan only when that creation is not already known. -- `GET /api/release-runs` lists bounded summaries; unreadable entries are - reported as unavailable instead of being silently omitted. +- `GET /api/release-runs` lists bounded summaries ordered by verified + `updated_at`, then `created_at` and run identifier. Unreadable entries are + appended deterministically when room remains instead of displacing newer + verified runs. - `GET /api/release-runs/{run_id}` reads and verifies one exact record. - `POST /api/release-runs/{run_id}/resume` records explicit recovery. - `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or diff --git a/tests/test_release_run_state.py b/tests/test_release_run_state.py index 3cf646e..179401e 100644 --- a/tests/test_release_run_state.py +++ b/tests/test_release_run_state.py @@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path: from govoplan_release.release_run import ( # noqa: E402 MAX_EVENTS, + MAX_RECORD_BYTES, ReleaseRunConflict, ReleaseRunCorrupt, ReleaseRunNotFound, @@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase): ) self.assertEqual("completed", completed["state"]["status"]) + def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None: + rejected_plan = mutating_first_plan() + rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)] + rejected_run_id = create_run( + self.store, + request_id="byte-capacity-rejected-create-0001", + plan_snapshot=rejected_plan, + )["run_id"] + rejected_path = self.root / f"{rejected_run_id}.json" + self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000) + + with self.assertRaisesRegex( + ReleaseRunConflict, "serialized recovery capacity" + ): + self.store.start_step( + rejected_run_id, + "core:tag", + attempt_id="byte-capacity-rejected-attempt-0001", + ) + untouched = self.store.get(rejected_run_id) + self.assertEqual("pending", untouched["state"]["steps"][0]["state"]) + self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"]) + + accepted_plan = mutating_first_plan() + accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)] + accepted_run_id = create_run( + self.store, + request_id="byte-capacity-accepted-create-0001", + plan_snapshot=accepted_plan, + )["run_id"] + accepted_path = self.root / f"{accepted_run_id}.json" + self.store.start_step( + accepted_run_id, + "core:tag", + attempt_id="byte-capacity-accepted-attempt-0001", + ) + self.store.resume( + accepted_run_id, + request_id="byte-capacity-accepted-resume-0001", + ) + with self.assertRaisesRegex( + ReleaseRunConflict, "reserved for a terminal reconciliation" + ): + self.store.reconcile_step( + accepted_run_id, + "core:tag", + request_id="byte-capacity-unresolved-request-0001", + outcome="unresolved", + confirmation="RECONCILE", + ) + completed = self.store.reconcile_step( + accepted_run_id, + "core:tag", + request_id="byte-capacity-terminal-request-0001", + outcome="effect_succeeded", + confirmation="RECONCILE", + ) + self.assertEqual("completed", completed["state"]["status"]) + self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES) + + def test_read_only_start_reserves_serialized_resume_and_retry_capacity( + self, + ) -> None: + plan = mutating_first_plan() + plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index] + plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)] + run_id = create_run( + self.store, + request_id="byte-capacity-read-only-create-0001", + plan_snapshot=plan, + )["run_id"] + self.store.start_step( + run_id, + "core:tag", + attempt_id="byte-capacity-read-only-attempt-0001", + ) + self.store.resume( + run_id, + request_id="byte-capacity-read-only-resume-0001", + ) + retried = self.store.retry_step( + run_id, + "core:tag", + request_id="byte-capacity-read-only-retry-0001", + ) + self.assertEqual("pending", retried["state"]["steps"][0]["state"]) + def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None: run_id = create_run( self.store, @@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase): listed = self.store.list(limit=1) self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed]) + def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable( + self, + ) -> None: + with patch( + "govoplan_release.release_run._timestamp", + return_value="2026-07-22T00:00:00Z", + ): + older = create_run( + self.store, + request_id="timestamp-order-request-older-0001", + ) + with patch( + "govoplan_release.release_run._timestamp", + return_value="2026-07-22T01:00:00Z", + ): + newer = create_run( + self.store, + request_id="timestamp-order-request-newer-0001", + ) + with patch( + "govoplan_release.release_run._timestamp", + return_value="2026-07-22T02:00:00Z", + ): + self.store.start_step( + older["run_id"], + "core:preflight", + attempt_id="timestamp-order-attempt-0001", + ) + + corrupt = create_run( + self.store, + request_id="timestamp-order-request-corrupt-0001", + ) + corrupt_path = self.root / f"{corrupt['run_id']}.json" + corrupt_path.write_text("{}", encoding="utf-8") + corrupt_path.chmod(0o600) + + listed = self.store.list(limit=3) + self.assertEqual( + [older["run_id"], newer["run_id"], corrupt["run_id"]], + [item["run_id"] for item in listed], + ) + self.assertEqual("unavailable", listed[-1]["status"]) + self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"]) + + def test_list_enumeration_oserror_is_normalized(self) -> None: + self.store.list() + with patch.object( + Path, + "iterdir", + side_effect=PermissionError("denied"), + ): + with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"): + self.store.list() + def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None: run_id = create_run( self.store, @@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase): self.assertIn('const runsRequest = api("/api/release-runs")', webui) self.assertIn("pendingReleaseRunPayload", webui) self.assertIn("recoverPendingReleaseRun", webui) + self.assertIn("recoverPendingRunCommands", webui) self.assertIn("inFlightRunCommandId", webui) self.assertIn("Run saved; list refresh unavailable", webui) self.assertIn("This foundation tracks state only.", runbook) @@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase): "submitReleaseRunCreate", "refreshReleaseRunListAfterCreate", "readInFlightRunCommands", + "pendingRunCommandRequest", + "recoverPendingRunCommands", + "deterministicRunCommandError", "inFlightRunCommandId", "clearInFlightRunCommand", "requestId", @@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru const renderReleaseRunList = () => {}; let failCreate = true; let failList = true; +let failureStatus = null; const postedRequestIds = []; -async function postJson(_path, payload) { +const postedPaths = []; +async function postJson(path, payload) { + postedPaths.push(path); postedRequestIds.push(payload.request_id); - if (failCreate) throw new Error("uncertain transport failure"); + if (failCreate) { + const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure"); + if (failureStatus) error.status = failureStatus; + throw error; + } return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } }; } async function api(_path) { @@ -1194,11 +1348,40 @@ async function api(_path) { assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure"); assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained"); - const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry"); - const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry"); + const commandKey = "retry:rr-known:core:preflight"; + const commandOne = inFlightRunCommandId(commandKey, "retry"); + const commandReplay = inFlightRunCommandId(commandKey, "retry"); assert(commandOne === commandReplay, "uncertain command did not reuse its request ID"); - clearInFlightRunCommand("retry:rr-known:step", commandOne); - const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry"); + const retryRequest = pendingRunCommandRequest(commandKey, commandOne); + assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable"); + const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request"); + assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable"); + assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed"); + + failCreate = true; + const uncertainCommand = await recoverPendingRunCommands(); + assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded"); + assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared"); + failCreate = false; + const recoveredCommand = await recoverPendingRunCommands(); + assert(recoveredCommand === true, "pending command did not recover after reload"); + assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared"); + assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID"); + assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint"); + assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run"); + + const conflictKey = "resume:rr-known"; + const conflictId = inFlightRunCommandId(conflictKey, "resume"); + failCreate = true; + failureStatus = 409; + const rejectedCommand = await recoverPendingRunCommands(); + assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded"); + assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID"); + assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained"); + failCreate = false; + failureStatus = null; + + const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry"); assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID"); })().catch((error) => { console.error(error.stack || error.message); process.exit(1); }); """ diff --git a/tools/release/govoplan_release/release_run.py b/tools/release/govoplan_release/release_run.py index f85d0b2..42f34c0 100644 --- a/tools/release/govoplan_release/release_run.py +++ b/tools/release/govoplan_release/release_run.py @@ -59,6 +59,8 @@ ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"} RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"} RECONCILIATION_CONFIRMATION = "RECONCILE" START_RECOVERY_RESERVE = 2 +PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z" +PROJECTION_RESULT_CODE = "r" * 64 class ReleaseRunError(RuntimeError): @@ -200,32 +202,32 @@ class ReleaseRunStore: limit = min(max(int(limit), 1), MAX_LIST_LIMIT) with self._locked(): self._ensure_root() - paths = sorted( - ( + try: + paths = [ path for path in self.root.iterdir() if path.name.endswith(".json") and RUN_ID_PATTERN.fullmatch(path.stem) - ), - key=lambda item: item.name, - reverse=True, - ) + ] + except OSError as exc: + raise ReleaseRunCorrupt( + "Release run storage could not be enumerated safely." + ) from exc summaries: list[dict[str, Any]] = [] + unavailable: list[dict[str, Any]] = [] for path in paths: try: record = self._read(path.stem) except ReleaseRunWorkspaceMismatch: continue except (ReleaseRunCorrupt, ReleaseRunNotFound): - summaries.append( + unavailable.append( { "run_id": path.stem, "status": "unavailable", "error": "Run record is unavailable or failed integrity validation.", } ) - if len(summaries) >= limit: - break continue view = _view(record) immutable_input = view["immutable"]["input"] @@ -240,9 +242,16 @@ class ReleaseRunStore: "recommended_next": view["recommended_next"], } ) - if len(summaries) >= limit: - break - return summaries + summaries.sort( + key=lambda item: ( + item["updated_at"], + item["created_at"], + item["run_id"], + ), + reverse=True, + ) + unavailable.sort(key=lambda item: item["run_id"], reverse=True) + return (summaries + unavailable)[:limit] def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]: request_fingerprint = _request_fingerprint(request_id) @@ -386,6 +395,7 @@ class ReleaseRunStore: "An unresolved outcome is already recorded for this attempt." ) _ensure_command_capacity(record, reserve_after=1) + _ensure_unresolved_recovery_byte_capacity(record, step_id) else: _ensure_command_capacity(record) if outcome == "effect_absent": @@ -446,7 +456,7 @@ class ReleaseRunStore: "The attempt identifier was already used for another release step." ) return _view(record) - step, _plan_step = _step_pair(record, step_id) + step, plan_step = _step_pair(record, step_id) available, reason = _step_available(record, step_id) if not available: raise ReleaseRunConflict(reason) @@ -471,6 +481,11 @@ class ReleaseRunStore: } ) _append_event(record, event_type="step_started", step_id=step_id) + _ensure_start_recovery_byte_capacity( + record, + step_id, + mutating=bool(plan_step.get("mutating")), + ) self._persist_update(record) return _view(record) @@ -1337,12 +1352,13 @@ def _append_event( event_type: str, step_id: str | None = None, result_code: str | None = None, + at: str | None = None, ) -> None: if event_type not in EVENT_TYPES: raise ReleaseRunConflict("Release event type is invalid.") event: dict[str, Any] = { "sequence": record["state"]["next_event_sequence"], - "at": _timestamp(), + "at": at if at is not None else _timestamp(), "type": event_type, } if step_id is not None: @@ -1355,6 +1371,216 @@ def _append_event( record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:] +def _ensure_start_recovery_byte_capacity( + record: dict[str, Any], step_id: str, *, mutating: bool +) -> None: + """Prove that every required completion/recovery write still fits.""" + + started = deepcopy(record) + _project_persist(started) + projections = [started] + + succeeded = _project_finish(started, step_id, succeeded=True) + failed = _project_finish(started, step_id, succeeded=False) + projections.extend((succeeded, failed, _project_retry(failed, step_id))) + + interrupted = _project_resume(started) + projections.append(interrupted) + if mutating: + projections.extend( + _project_reconcile(interrupted, step_id, outcome=outcome) + for outcome in ("effect_absent", "effect_succeeded") + ) + else: + projections.append(_project_retry(interrupted, step_id)) + _ensure_projected_records_fit( + projections, + message=( + "Release run has insufficient serialized recovery capacity for a " + "new attempt; create a new run." + ), + ) + + +def _ensure_unresolved_recovery_byte_capacity( + record: dict[str, Any], step_id: str +) -> None: + """Accept an advisory unresolved write only if a terminal write remains.""" + + unresolved = _project_reconcile(record, step_id, outcome="unresolved") + projections = [unresolved] + projections.extend( + _project_reconcile(unresolved, step_id, outcome=outcome) + for outcome in ("effect_absent", "effect_succeeded") + ) + _ensure_projected_records_fit( + projections, + message=( + "The unresolved outcome would consume capacity reserved for a " + "terminal reconciliation." + ), + ) + + +def _project_finish( + record: dict[str, Any], step_id: str, *, succeeded: bool +) -> dict[str, Any]: + projected = deepcopy(record) + step, _plan_step = _step_pair(projected, step_id) + attempt = _attempt_for_fingerprint( + projected, step["attempt_fingerprint"], step_id=step_id + ) + target_state = "succeeded" if succeeded else "failed" + step.update( + { + "state": target_state, + "finished_at": PROJECTION_TIMESTAMP, + "result_code": PROJECTION_RESULT_CODE, + } + ) + attempt["outcome"] = target_state + attempt["result_code"] = PROJECTION_RESULT_CODE + _append_event( + projected, + event_type="step_succeeded" if succeeded else "step_failed", + step_id=step_id, + result_code=PROJECTION_RESULT_CODE, + at=PROJECTION_TIMESTAMP, + ) + _project_persist(projected) + return projected + + +def _project_resume(record: dict[str, Any]) -> dict[str, Any]: + projected = deepcopy(record) + running_steps = [ + step for step in projected["state"]["steps"] if step["state"] == "running" + ] + for step in running_steps: + attempt = _attempt_for_fingerprint( + projected, step["attempt_fingerprint"], step_id=step["id"] + ) + attempt["outcome"] = "interrupted" + attempt["result_code"] = "process_interrupted" + step["state"] = "interrupted" + step["finished_at"] = PROJECTION_TIMESTAMP + step["result_code"] = "process_interrupted" + _append_event( + projected, + event_type="step_interrupted", + step_id=step["id"], + result_code="process_interrupted", + at=PROJECTION_TIMESTAMP, + ) + _remember_request( + projected, + "0" * 64, + "resume", + None, + None, + attempt_fingerprint=running_steps[0]["attempt_fingerprint"], + ) + _append_event(projected, event_type="run_resumed", at=PROJECTION_TIMESTAMP) + _project_persist(projected) + return projected + + +def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]: + projected = deepcopy(record) + step, _plan_step = _step_pair(projected, step_id) + attempt_fingerprint = step["attempt_fingerprint"] + retry_reason = ( + "known_failure" + if step["state"] == "failed" + else "read_only_interruption" + ) + step.update( + { + "state": "pending", + "attempt_fingerprint": None, + "started_at": None, + "finished_at": None, + "result_code": None, + } + ) + _remember_request( + projected, + "1" * 64, + "retry", + step_id, + None, + attempt_fingerprint=attempt_fingerprint, + ) + _append_event( + projected, + event_type="step_retry_requested", + step_id=step_id, + result_code=retry_reason, + at=PROJECTION_TIMESTAMP, + ) + _project_persist(projected) + return projected + + +def _project_reconcile( + record: dict[str, Any], step_id: str, *, outcome: str +) -> dict[str, Any]: + projected = deepcopy(record) + step, _plan_step = _step_pair(projected, step_id) + attempt_fingerprint = step["attempt_fingerprint"] + if outcome == "effect_absent": + step.update( + { + "state": "pending", + "attempt_fingerprint": None, + "started_at": None, + "finished_at": None, + "result_code": None, + } + ) + elif outcome == "effect_succeeded": + step.update( + { + "state": "succeeded", + "finished_at": PROJECTION_TIMESTAMP, + "result_code": "reconciled_effect_succeeded", + } + ) + _remember_request( + projected, + "2" * 64 if outcome == "unresolved" else "3" * 64, + "reconcile", + step_id, + outcome, + attempt_fingerprint=attempt_fingerprint, + ) + _append_event( + projected, + event_type="step_reconciled", + step_id=step_id, + result_code=outcome, + at=PROJECTION_TIMESTAMP, + ) + _project_persist(projected) + return projected + + +def _project_persist(record: dict[str, Any]) -> None: + record["updated_at"] = PROJECTION_TIMESTAMP + _refresh_status(record) + record["record_digest"] = "f" * 64 + + +def _ensure_projected_records_fit( + records: list[dict[str, Any]], *, message: str +) -> None: + if any( + len(_canonical_json(projected)) + 1 > MAX_RECORD_BYTES + for projected in records + ): + raise ReleaseRunConflict(message) + + def _ensure_command_capacity( record: dict[str, Any], *, reserve_after: int = 0 ) -> None: diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html index f27b54a..9ebf2e1 100644 --- a/tools/release/webui/index.html +++ b/tools/release/webui/index.html @@ -1006,10 +1006,12 @@ renderReleaseRunStoreUnavailable(runsResult.error); } else { renderReleaseRunList(runsResult.data?.runs || []); - const pendingRecoveryResult = await recoverPendingReleaseRun(); - if (pendingRecoveryResult === null && releaseState.currentRun?.run_id) { + const pendingCreateResult = await recoverPendingReleaseRun(); + const pendingCommandResult = await recoverPendingRunCommands(); + const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null; + if (!recoveryAttempted && releaseState.currentRun?.run_id) { await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false }); - } else if (pendingRecoveryResult === null) { + } else if (!recoveryAttempted) { renderIdlePlan(); } } @@ -1262,6 +1264,7 @@ releaseState.currentRun = resumed; renderReleaseRun(resumed); } catch (error) { + if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId); elements.runStatus.textContent = "error"; elements.runOutput.insertAdjacentHTML("afterbegin", `
${escapeHtml(error.message)}
${escapeHtml(error.message)}
${escapeHtml(error.message)}
${escapeHtml(pendingError.message)}
${escapeHtml(rejectedError.message)}