fix(release): preserve durable recovery capacity
This commit is contained in:
@@ -108,13 +108,17 @@ commands, process output, confirmation text, credentials, bearer tokens, or
|
|||||||
signing material.
|
signing material.
|
||||||
|
|
||||||
Before a step can enter `running`, the store reserves the two command-ledger
|
Before a step can enter `running`, the store reserves the two command-ledger
|
||||||
slots needed for worst-case recovery. An explicit resume consumes one slot and
|
slots needed for worst-case recovery. It also projects the serialized record
|
||||||
is accepted only while a persisted attempt is actually running. A mutating
|
through start, finish/failure, resume, and the required terminal reconciliation
|
||||||
attempt always retains its final `effect_absent` or `effect_succeeded` slot;
|
or read-only retry; the start is rejected unless every required atomic write
|
||||||
|
fits the record-size bound. An explicit resume consumes one slot and is
|
||||||
|
accepted only while a persisted attempt is actually running. A mutating attempt
|
||||||
|
always retains its final `effect_absent` or `effect_succeeded` slot;
|
||||||
`unresolved` may be recorded at most once for that attempt and only when an
|
`unresolved` may be recorded at most once for that attempt and only when an
|
||||||
additional slot is available. Read-only interruption and known failure retain
|
additional ledger slot and serialized terminal-write capacity are available.
|
||||||
the slot needed to prepare a retry. Capacity exhaustion is therefore detected
|
Read-only interruption and known failure retain the slot and byte capacity
|
||||||
before starting an effect rather than stranding an ambiguous attempt.
|
needed to prepare a retry. Capacity exhaustion is therefore detected before
|
||||||
|
starting an effect rather than stranding an ambiguous attempt.
|
||||||
|
|
||||||
An explicit resume after a process restart converts every persisted `running`
|
An explicit resume after a process restart converts every persisted `running`
|
||||||
step to `interrupted`; it never guesses whether an external effect happened.
|
step to `interrupted`; it never guesses whether an external effect happened.
|
||||||
@@ -128,16 +132,20 @@ code-only state event. A known failed attempt can likewise be prepared for
|
|||||||
retry. The UI keeps unavailable controls visible and disabled.
|
retry. The UI keeps unavailable controls visible and disabled.
|
||||||
|
|
||||||
The browser likewise retains the request identifier for an uncertain
|
The browser likewise retains the request identifier for an uncertain
|
||||||
resume/retry/reconciliation response and reuses it on the operator's retry.
|
resume/retry/reconciliation response and replays it after reload. A successful
|
||||||
Identifiers are cleared only after the server returns the resulting run state.
|
replay selects the returned run state. Transport and server failures retain the
|
||||||
|
identifier; a deterministic `4xx` rejection clears it so a stale command cannot
|
||||||
|
poison a later attempt.
|
||||||
|
|
||||||
The run API is covered by the same local console token middleware as every
|
The run API is covered by the same local console token middleware as every
|
||||||
other `/api/` route:
|
other `/api/` route:
|
||||||
|
|
||||||
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
|
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
|
||||||
freezes a selective plan only when that creation is not already known.
|
freezes a selective plan only when that creation is not already known.
|
||||||
- `GET /api/release-runs` lists bounded summaries; unreadable entries are
|
- `GET /api/release-runs` lists bounded summaries ordered by verified
|
||||||
reported as unavailable instead of being silently omitted.
|
`updated_at`, then `created_at` and run identifier. Unreadable entries are
|
||||||
|
appended deterministically when room remains instead of displacing newer
|
||||||
|
verified runs.
|
||||||
- `GET /api/release-runs/{run_id}` reads and verifies one exact record.
|
- `GET /api/release-runs/{run_id}` reads and verifies one exact record.
|
||||||
- `POST /api/release-runs/{run_id}/resume` records explicit recovery.
|
- `POST /api/release-runs/{run_id}/resume` records explicit recovery.
|
||||||
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or
|
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path:
|
|||||||
|
|
||||||
from govoplan_release.release_run import ( # noqa: E402
|
from govoplan_release.release_run import ( # noqa: E402
|
||||||
MAX_EVENTS,
|
MAX_EVENTS,
|
||||||
|
MAX_RECORD_BYTES,
|
||||||
ReleaseRunConflict,
|
ReleaseRunConflict,
|
||||||
ReleaseRunCorrupt,
|
ReleaseRunCorrupt,
|
||||||
ReleaseRunNotFound,
|
ReleaseRunNotFound,
|
||||||
@@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual("completed", completed["state"]["status"])
|
self.assertEqual("completed", completed["state"]["status"])
|
||||||
|
|
||||||
|
def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None:
|
||||||
|
rejected_plan = mutating_first_plan()
|
||||||
|
rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)]
|
||||||
|
rejected_run_id = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="byte-capacity-rejected-create-0001",
|
||||||
|
plan_snapshot=rejected_plan,
|
||||||
|
)["run_id"]
|
||||||
|
rejected_path = self.root / f"{rejected_run_id}.json"
|
||||||
|
self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
ReleaseRunConflict, "serialized recovery capacity"
|
||||||
|
):
|
||||||
|
self.store.start_step(
|
||||||
|
rejected_run_id,
|
||||||
|
"core:tag",
|
||||||
|
attempt_id="byte-capacity-rejected-attempt-0001",
|
||||||
|
)
|
||||||
|
untouched = self.store.get(rejected_run_id)
|
||||||
|
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
|
||||||
|
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
|
||||||
|
|
||||||
|
accepted_plan = mutating_first_plan()
|
||||||
|
accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
|
||||||
|
accepted_run_id = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="byte-capacity-accepted-create-0001",
|
||||||
|
plan_snapshot=accepted_plan,
|
||||||
|
)["run_id"]
|
||||||
|
accepted_path = self.root / f"{accepted_run_id}.json"
|
||||||
|
self.store.start_step(
|
||||||
|
accepted_run_id,
|
||||||
|
"core:tag",
|
||||||
|
attempt_id="byte-capacity-accepted-attempt-0001",
|
||||||
|
)
|
||||||
|
self.store.resume(
|
||||||
|
accepted_run_id,
|
||||||
|
request_id="byte-capacity-accepted-resume-0001",
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
ReleaseRunConflict, "reserved for a terminal reconciliation"
|
||||||
|
):
|
||||||
|
self.store.reconcile_step(
|
||||||
|
accepted_run_id,
|
||||||
|
"core:tag",
|
||||||
|
request_id="byte-capacity-unresolved-request-0001",
|
||||||
|
outcome="unresolved",
|
||||||
|
confirmation="RECONCILE",
|
||||||
|
)
|
||||||
|
completed = self.store.reconcile_step(
|
||||||
|
accepted_run_id,
|
||||||
|
"core:tag",
|
||||||
|
request_id="byte-capacity-terminal-request-0001",
|
||||||
|
outcome="effect_succeeded",
|
||||||
|
confirmation="RECONCILE",
|
||||||
|
)
|
||||||
|
self.assertEqual("completed", completed["state"]["status"])
|
||||||
|
self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES)
|
||||||
|
|
||||||
|
def test_read_only_start_reserves_serialized_resume_and_retry_capacity(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
plan = mutating_first_plan()
|
||||||
|
plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index]
|
||||||
|
plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
|
||||||
|
run_id = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="byte-capacity-read-only-create-0001",
|
||||||
|
plan_snapshot=plan,
|
||||||
|
)["run_id"]
|
||||||
|
self.store.start_step(
|
||||||
|
run_id,
|
||||||
|
"core:tag",
|
||||||
|
attempt_id="byte-capacity-read-only-attempt-0001",
|
||||||
|
)
|
||||||
|
self.store.resume(
|
||||||
|
run_id,
|
||||||
|
request_id="byte-capacity-read-only-resume-0001",
|
||||||
|
)
|
||||||
|
retried = self.store.retry_step(
|
||||||
|
run_id,
|
||||||
|
"core:tag",
|
||||||
|
request_id="byte-capacity-read-only-retry-0001",
|
||||||
|
)
|
||||||
|
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
|
||||||
|
|
||||||
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
|
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
|
||||||
run_id = create_run(
|
run_id = create_run(
|
||||||
self.store,
|
self.store,
|
||||||
@@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase):
|
|||||||
listed = self.store.list(limit=1)
|
listed = self.store.list(limit=1)
|
||||||
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
|
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
|
||||||
|
|
||||||
|
def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_release.release_run._timestamp",
|
||||||
|
return_value="2026-07-22T00:00:00Z",
|
||||||
|
):
|
||||||
|
older = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="timestamp-order-request-older-0001",
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_release.release_run._timestamp",
|
||||||
|
return_value="2026-07-22T01:00:00Z",
|
||||||
|
):
|
||||||
|
newer = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="timestamp-order-request-newer-0001",
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_release.release_run._timestamp",
|
||||||
|
return_value="2026-07-22T02:00:00Z",
|
||||||
|
):
|
||||||
|
self.store.start_step(
|
||||||
|
older["run_id"],
|
||||||
|
"core:preflight",
|
||||||
|
attempt_id="timestamp-order-attempt-0001",
|
||||||
|
)
|
||||||
|
|
||||||
|
corrupt = create_run(
|
||||||
|
self.store,
|
||||||
|
request_id="timestamp-order-request-corrupt-0001",
|
||||||
|
)
|
||||||
|
corrupt_path = self.root / f"{corrupt['run_id']}.json"
|
||||||
|
corrupt_path.write_text("{}", encoding="utf-8")
|
||||||
|
corrupt_path.chmod(0o600)
|
||||||
|
|
||||||
|
listed = self.store.list(limit=3)
|
||||||
|
self.assertEqual(
|
||||||
|
[older["run_id"], newer["run_id"], corrupt["run_id"]],
|
||||||
|
[item["run_id"] for item in listed],
|
||||||
|
)
|
||||||
|
self.assertEqual("unavailable", listed[-1]["status"])
|
||||||
|
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
|
||||||
|
|
||||||
|
def test_list_enumeration_oserror_is_normalized(self) -> None:
|
||||||
|
self.store.list()
|
||||||
|
with patch.object(
|
||||||
|
Path,
|
||||||
|
"iterdir",
|
||||||
|
side_effect=PermissionError("denied"),
|
||||||
|
):
|
||||||
|
with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"):
|
||||||
|
self.store.list()
|
||||||
|
|
||||||
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
|
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
|
||||||
run_id = create_run(
|
run_id = create_run(
|
||||||
self.store,
|
self.store,
|
||||||
@@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase):
|
|||||||
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
|
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
|
||||||
self.assertIn("pendingReleaseRunPayload", webui)
|
self.assertIn("pendingReleaseRunPayload", webui)
|
||||||
self.assertIn("recoverPendingReleaseRun", webui)
|
self.assertIn("recoverPendingReleaseRun", webui)
|
||||||
|
self.assertIn("recoverPendingRunCommands", webui)
|
||||||
self.assertIn("inFlightRunCommandId", webui)
|
self.assertIn("inFlightRunCommandId", webui)
|
||||||
self.assertIn("Run saved; list refresh unavailable", webui)
|
self.assertIn("Run saved; list refresh unavailable", webui)
|
||||||
self.assertIn("This foundation tracks state only.", runbook)
|
self.assertIn("This foundation tracks state only.", runbook)
|
||||||
@@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase):
|
|||||||
"submitReleaseRunCreate",
|
"submitReleaseRunCreate",
|
||||||
"refreshReleaseRunListAfterCreate",
|
"refreshReleaseRunListAfterCreate",
|
||||||
"readInFlightRunCommands",
|
"readInFlightRunCommands",
|
||||||
|
"pendingRunCommandRequest",
|
||||||
|
"recoverPendingRunCommands",
|
||||||
|
"deterministicRunCommandError",
|
||||||
"inFlightRunCommandId",
|
"inFlightRunCommandId",
|
||||||
"clearInFlightRunCommand",
|
"clearInFlightRunCommand",
|
||||||
"requestId",
|
"requestId",
|
||||||
@@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru
|
|||||||
const renderReleaseRunList = () => {};
|
const renderReleaseRunList = () => {};
|
||||||
let failCreate = true;
|
let failCreate = true;
|
||||||
let failList = true;
|
let failList = true;
|
||||||
|
let failureStatus = null;
|
||||||
const postedRequestIds = [];
|
const postedRequestIds = [];
|
||||||
async function postJson(_path, payload) {
|
const postedPaths = [];
|
||||||
|
async function postJson(path, payload) {
|
||||||
|
postedPaths.push(path);
|
||||||
postedRequestIds.push(payload.request_id);
|
postedRequestIds.push(payload.request_id);
|
||||||
if (failCreate) throw new Error("uncertain transport failure");
|
if (failCreate) {
|
||||||
|
const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure");
|
||||||
|
if (failureStatus) error.status = failureStatus;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
|
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
|
||||||
}
|
}
|
||||||
async function api(_path) {
|
async function api(_path) {
|
||||||
@@ -1194,11 +1348,40 @@ async function api(_path) {
|
|||||||
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
|
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
|
||||||
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
|
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
|
||||||
|
|
||||||
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry");
|
const commandKey = "retry:rr-known:core:preflight";
|
||||||
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry");
|
const commandOne = inFlightRunCommandId(commandKey, "retry");
|
||||||
|
const commandReplay = inFlightRunCommandId(commandKey, "retry");
|
||||||
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
|
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
|
||||||
clearInFlightRunCommand("retry:rr-known:step", commandOne);
|
const retryRequest = pendingRunCommandRequest(commandKey, commandOne);
|
||||||
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry");
|
assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable");
|
||||||
|
const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request");
|
||||||
|
assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable");
|
||||||
|
assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed");
|
||||||
|
|
||||||
|
failCreate = true;
|
||||||
|
const uncertainCommand = await recoverPendingRunCommands();
|
||||||
|
assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded");
|
||||||
|
assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared");
|
||||||
|
failCreate = false;
|
||||||
|
const recoveredCommand = await recoverPendingRunCommands();
|
||||||
|
assert(recoveredCommand === true, "pending command did not recover after reload");
|
||||||
|
assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared");
|
||||||
|
assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID");
|
||||||
|
assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint");
|
||||||
|
assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run");
|
||||||
|
|
||||||
|
const conflictKey = "resume:rr-known";
|
||||||
|
const conflictId = inFlightRunCommandId(conflictKey, "resume");
|
||||||
|
failCreate = true;
|
||||||
|
failureStatus = 409;
|
||||||
|
const rejectedCommand = await recoverPendingRunCommands();
|
||||||
|
assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded");
|
||||||
|
assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID");
|
||||||
|
assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained");
|
||||||
|
failCreate = false;
|
||||||
|
failureStatus = null;
|
||||||
|
|
||||||
|
const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry");
|
||||||
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
|
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
|
||||||
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
|
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"}
|
|||||||
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
|
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
|
||||||
RECONCILIATION_CONFIRMATION = "RECONCILE"
|
RECONCILIATION_CONFIRMATION = "RECONCILE"
|
||||||
START_RECOVERY_RESERVE = 2
|
START_RECOVERY_RESERVE = 2
|
||||||
|
PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z"
|
||||||
|
PROJECTION_RESULT_CODE = "r" * 64
|
||||||
|
|
||||||
|
|
||||||
class ReleaseRunError(RuntimeError):
|
class ReleaseRunError(RuntimeError):
|
||||||
@@ -200,32 +202,32 @@ class ReleaseRunStore:
|
|||||||
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
|
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
|
||||||
with self._locked():
|
with self._locked():
|
||||||
self._ensure_root()
|
self._ensure_root()
|
||||||
paths = sorted(
|
try:
|
||||||
(
|
paths = [
|
||||||
path
|
path
|
||||||
for path in self.root.iterdir()
|
for path in self.root.iterdir()
|
||||||
if path.name.endswith(".json")
|
if path.name.endswith(".json")
|
||||||
and RUN_ID_PATTERN.fullmatch(path.stem)
|
and RUN_ID_PATTERN.fullmatch(path.stem)
|
||||||
),
|
]
|
||||||
key=lambda item: item.name,
|
except OSError as exc:
|
||||||
reverse=True,
|
raise ReleaseRunCorrupt(
|
||||||
)
|
"Release run storage could not be enumerated safely."
|
||||||
|
) from exc
|
||||||
summaries: list[dict[str, Any]] = []
|
summaries: list[dict[str, Any]] = []
|
||||||
|
unavailable: list[dict[str, Any]] = []
|
||||||
for path in paths:
|
for path in paths:
|
||||||
try:
|
try:
|
||||||
record = self._read(path.stem)
|
record = self._read(path.stem)
|
||||||
except ReleaseRunWorkspaceMismatch:
|
except ReleaseRunWorkspaceMismatch:
|
||||||
continue
|
continue
|
||||||
except (ReleaseRunCorrupt, ReleaseRunNotFound):
|
except (ReleaseRunCorrupt, ReleaseRunNotFound):
|
||||||
summaries.append(
|
unavailable.append(
|
||||||
{
|
{
|
||||||
"run_id": path.stem,
|
"run_id": path.stem,
|
||||||
"status": "unavailable",
|
"status": "unavailable",
|
||||||
"error": "Run record is unavailable or failed integrity validation.",
|
"error": "Run record is unavailable or failed integrity validation.",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if len(summaries) >= limit:
|
|
||||||
break
|
|
||||||
continue
|
continue
|
||||||
view = _view(record)
|
view = _view(record)
|
||||||
immutable_input = view["immutable"]["input"]
|
immutable_input = view["immutable"]["input"]
|
||||||
@@ -240,9 +242,16 @@ class ReleaseRunStore:
|
|||||||
"recommended_next": view["recommended_next"],
|
"recommended_next": view["recommended_next"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if len(summaries) >= limit:
|
summaries.sort(
|
||||||
break
|
key=lambda item: (
|
||||||
return summaries
|
item["updated_at"],
|
||||||
|
item["created_at"],
|
||||||
|
item["run_id"],
|
||||||
|
),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
|
||||||
|
return (summaries + unavailable)[:limit]
|
||||||
|
|
||||||
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
|
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
|
||||||
request_fingerprint = _request_fingerprint(request_id)
|
request_fingerprint = _request_fingerprint(request_id)
|
||||||
@@ -386,6 +395,7 @@ class ReleaseRunStore:
|
|||||||
"An unresolved outcome is already recorded for this attempt."
|
"An unresolved outcome is already recorded for this attempt."
|
||||||
)
|
)
|
||||||
_ensure_command_capacity(record, reserve_after=1)
|
_ensure_command_capacity(record, reserve_after=1)
|
||||||
|
_ensure_unresolved_recovery_byte_capacity(record, step_id)
|
||||||
else:
|
else:
|
||||||
_ensure_command_capacity(record)
|
_ensure_command_capacity(record)
|
||||||
if outcome == "effect_absent":
|
if outcome == "effect_absent":
|
||||||
@@ -446,7 +456,7 @@ class ReleaseRunStore:
|
|||||||
"The attempt identifier was already used for another release step."
|
"The attempt identifier was already used for another release step."
|
||||||
)
|
)
|
||||||
return _view(record)
|
return _view(record)
|
||||||
step, _plan_step = _step_pair(record, step_id)
|
step, plan_step = _step_pair(record, step_id)
|
||||||
available, reason = _step_available(record, step_id)
|
available, reason = _step_available(record, step_id)
|
||||||
if not available:
|
if not available:
|
||||||
raise ReleaseRunConflict(reason)
|
raise ReleaseRunConflict(reason)
|
||||||
@@ -471,6 +481,11 @@ class ReleaseRunStore:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
_append_event(record, event_type="step_started", step_id=step_id)
|
_append_event(record, event_type="step_started", step_id=step_id)
|
||||||
|
_ensure_start_recovery_byte_capacity(
|
||||||
|
record,
|
||||||
|
step_id,
|
||||||
|
mutating=bool(plan_step.get("mutating")),
|
||||||
|
)
|
||||||
self._persist_update(record)
|
self._persist_update(record)
|
||||||
return _view(record)
|
return _view(record)
|
||||||
|
|
||||||
@@ -1337,12 +1352,13 @@ def _append_event(
|
|||||||
event_type: str,
|
event_type: str,
|
||||||
step_id: str | None = None,
|
step_id: str | None = None,
|
||||||
result_code: str | None = None,
|
result_code: str | None = None,
|
||||||
|
at: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if event_type not in EVENT_TYPES:
|
if event_type not in EVENT_TYPES:
|
||||||
raise ReleaseRunConflict("Release event type is invalid.")
|
raise ReleaseRunConflict("Release event type is invalid.")
|
||||||
event: dict[str, Any] = {
|
event: dict[str, Any] = {
|
||||||
"sequence": record["state"]["next_event_sequence"],
|
"sequence": record["state"]["next_event_sequence"],
|
||||||
"at": _timestamp(),
|
"at": at if at is not None else _timestamp(),
|
||||||
"type": event_type,
|
"type": event_type,
|
||||||
}
|
}
|
||||||
if step_id is not None:
|
if step_id is not None:
|
||||||
@@ -1355,6 +1371,216 @@ def _append_event(
|
|||||||
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
|
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_start_recovery_byte_capacity(
|
||||||
|
record: dict[str, Any], step_id: str, *, mutating: bool
|
||||||
|
) -> None:
|
||||||
|
"""Prove that every required completion/recovery write still fits."""
|
||||||
|
|
||||||
|
started = deepcopy(record)
|
||||||
|
_project_persist(started)
|
||||||
|
projections = [started]
|
||||||
|
|
||||||
|
succeeded = _project_finish(started, step_id, succeeded=True)
|
||||||
|
failed = _project_finish(started, step_id, succeeded=False)
|
||||||
|
projections.extend((succeeded, failed, _project_retry(failed, step_id)))
|
||||||
|
|
||||||
|
interrupted = _project_resume(started)
|
||||||
|
projections.append(interrupted)
|
||||||
|
if mutating:
|
||||||
|
projections.extend(
|
||||||
|
_project_reconcile(interrupted, step_id, outcome=outcome)
|
||||||
|
for outcome in ("effect_absent", "effect_succeeded")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
projections.append(_project_retry(interrupted, step_id))
|
||||||
|
_ensure_projected_records_fit(
|
||||||
|
projections,
|
||||||
|
message=(
|
||||||
|
"Release run has insufficient serialized recovery capacity for a "
|
||||||
|
"new attempt; create a new run."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_unresolved_recovery_byte_capacity(
|
||||||
|
record: dict[str, Any], step_id: str
|
||||||
|
) -> None:
|
||||||
|
"""Accept an advisory unresolved write only if a terminal write remains."""
|
||||||
|
|
||||||
|
unresolved = _project_reconcile(record, step_id, outcome="unresolved")
|
||||||
|
projections = [unresolved]
|
||||||
|
projections.extend(
|
||||||
|
_project_reconcile(unresolved, step_id, outcome=outcome)
|
||||||
|
for outcome in ("effect_absent", "effect_succeeded")
|
||||||
|
)
|
||||||
|
_ensure_projected_records_fit(
|
||||||
|
projections,
|
||||||
|
message=(
|
||||||
|
"The unresolved outcome would consume capacity reserved for a "
|
||||||
|
"terminal reconciliation."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _project_finish(
|
||||||
|
record: dict[str, Any], step_id: str, *, succeeded: bool
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
projected = deepcopy(record)
|
||||||
|
step, _plan_step = _step_pair(projected, step_id)
|
||||||
|
attempt = _attempt_for_fingerprint(
|
||||||
|
projected, step["attempt_fingerprint"], step_id=step_id
|
||||||
|
)
|
||||||
|
target_state = "succeeded" if succeeded else "failed"
|
||||||
|
step.update(
|
||||||
|
{
|
||||||
|
"state": target_state,
|
||||||
|
"finished_at": PROJECTION_TIMESTAMP,
|
||||||
|
"result_code": PROJECTION_RESULT_CODE,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
attempt["outcome"] = target_state
|
||||||
|
attempt["result_code"] = PROJECTION_RESULT_CODE
|
||||||
|
_append_event(
|
||||||
|
projected,
|
||||||
|
event_type="step_succeeded" if succeeded else "step_failed",
|
||||||
|
step_id=step_id,
|
||||||
|
result_code=PROJECTION_RESULT_CODE,
|
||||||
|
at=PROJECTION_TIMESTAMP,
|
||||||
|
)
|
||||||
|
_project_persist(projected)
|
||||||
|
return projected
|
||||||
|
|
||||||
|
|
||||||
|
def _project_resume(record: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
projected = deepcopy(record)
|
||||||
|
running_steps = [
|
||||||
|
step for step in projected["state"]["steps"] if step["state"] == "running"
|
||||||
|
]
|
||||||
|
for step in running_steps:
|
||||||
|
attempt = _attempt_for_fingerprint(
|
||||||
|
projected, step["attempt_fingerprint"], step_id=step["id"]
|
||||||
|
)
|
||||||
|
attempt["outcome"] = "interrupted"
|
||||||
|
attempt["result_code"] = "process_interrupted"
|
||||||
|
step["state"] = "interrupted"
|
||||||
|
step["finished_at"] = PROJECTION_TIMESTAMP
|
||||||
|
step["result_code"] = "process_interrupted"
|
||||||
|
_append_event(
|
||||||
|
projected,
|
||||||
|
event_type="step_interrupted",
|
||||||
|
step_id=step["id"],
|
||||||
|
result_code="process_interrupted",
|
||||||
|
at=PROJECTION_TIMESTAMP,
|
||||||
|
)
|
||||||
|
_remember_request(
|
||||||
|
projected,
|
||||||
|
"0" * 64,
|
||||||
|
"resume",
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
attempt_fingerprint=running_steps[0]["attempt_fingerprint"],
|
||||||
|
)
|
||||||
|
_append_event(projected, event_type="run_resumed", at=PROJECTION_TIMESTAMP)
|
||||||
|
_project_persist(projected)
|
||||||
|
return projected
|
||||||
|
|
||||||
|
|
||||||
|
def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]:
|
||||||
|
projected = deepcopy(record)
|
||||||
|
step, _plan_step = _step_pair(projected, step_id)
|
||||||
|
attempt_fingerprint = step["attempt_fingerprint"]
|
||||||
|
retry_reason = (
|
||||||
|
"known_failure"
|
||||||
|
if step["state"] == "failed"
|
||||||
|
else "read_only_interruption"
|
||||||
|
)
|
||||||
|
step.update(
|
||||||
|
{
|
||||||
|
"state": "pending",
|
||||||
|
"attempt_fingerprint": None,
|
||||||
|
"started_at": None,
|
||||||
|
"finished_at": None,
|
||||||
|
"result_code": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_remember_request(
|
||||||
|
projected,
|
||||||
|
"1" * 64,
|
||||||
|
"retry",
|
||||||
|
step_id,
|
||||||
|
None,
|
||||||
|
attempt_fingerprint=attempt_fingerprint,
|
||||||
|
)
|
||||||
|
_append_event(
|
||||||
|
projected,
|
||||||
|
event_type="step_retry_requested",
|
||||||
|
step_id=step_id,
|
||||||
|
result_code=retry_reason,
|
||||||
|
at=PROJECTION_TIMESTAMP,
|
||||||
|
)
|
||||||
|
_project_persist(projected)
|
||||||
|
return projected
|
||||||
|
|
||||||
|
|
||||||
|
def _project_reconcile(
|
||||||
|
record: dict[str, Any], step_id: str, *, outcome: str
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
projected = deepcopy(record)
|
||||||
|
step, _plan_step = _step_pair(projected, step_id)
|
||||||
|
attempt_fingerprint = step["attempt_fingerprint"]
|
||||||
|
if outcome == "effect_absent":
|
||||||
|
step.update(
|
||||||
|
{
|
||||||
|
"state": "pending",
|
||||||
|
"attempt_fingerprint": None,
|
||||||
|
"started_at": None,
|
||||||
|
"finished_at": None,
|
||||||
|
"result_code": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
elif outcome == "effect_succeeded":
|
||||||
|
step.update(
|
||||||
|
{
|
||||||
|
"state": "succeeded",
|
||||||
|
"finished_at": PROJECTION_TIMESTAMP,
|
||||||
|
"result_code": "reconciled_effect_succeeded",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_remember_request(
|
||||||
|
projected,
|
||||||
|
"2" * 64 if outcome == "unresolved" else "3" * 64,
|
||||||
|
"reconcile",
|
||||||
|
step_id,
|
||||||
|
outcome,
|
||||||
|
attempt_fingerprint=attempt_fingerprint,
|
||||||
|
)
|
||||||
|
_append_event(
|
||||||
|
projected,
|
||||||
|
event_type="step_reconciled",
|
||||||
|
step_id=step_id,
|
||||||
|
result_code=outcome,
|
||||||
|
at=PROJECTION_TIMESTAMP,
|
||||||
|
)
|
||||||
|
_project_persist(projected)
|
||||||
|
return projected
|
||||||
|
|
||||||
|
|
||||||
|
def _project_persist(record: dict[str, Any]) -> None:
|
||||||
|
record["updated_at"] = PROJECTION_TIMESTAMP
|
||||||
|
_refresh_status(record)
|
||||||
|
record["record_digest"] = "f" * 64
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_projected_records_fit(
|
||||||
|
records: list[dict[str, Any]], *, message: str
|
||||||
|
) -> None:
|
||||||
|
if any(
|
||||||
|
len(_canonical_json(projected)) + 1 > MAX_RECORD_BYTES
|
||||||
|
for projected in records
|
||||||
|
):
|
||||||
|
raise ReleaseRunConflict(message)
|
||||||
|
|
||||||
|
|
||||||
def _ensure_command_capacity(
|
def _ensure_command_capacity(
|
||||||
record: dict[str, Any], *, reserve_after: int = 0
|
record: dict[str, Any], *, reserve_after: int = 0
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -1006,10 +1006,12 @@
|
|||||||
renderReleaseRunStoreUnavailable(runsResult.error);
|
renderReleaseRunStoreUnavailable(runsResult.error);
|
||||||
} else {
|
} else {
|
||||||
renderReleaseRunList(runsResult.data?.runs || []);
|
renderReleaseRunList(runsResult.data?.runs || []);
|
||||||
const pendingRecoveryResult = await recoverPendingReleaseRun();
|
const pendingCreateResult = await recoverPendingReleaseRun();
|
||||||
if (pendingRecoveryResult === null && releaseState.currentRun?.run_id) {
|
const pendingCommandResult = await recoverPendingRunCommands();
|
||||||
|
const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null;
|
||||||
|
if (!recoveryAttempted && releaseState.currentRun?.run_id) {
|
||||||
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
|
await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false });
|
||||||
} else if (pendingRecoveryResult === null) {
|
} else if (!recoveryAttempted) {
|
||||||
renderIdlePlan();
|
renderIdlePlan();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1262,6 +1264,7 @@
|
|||||||
releaseState.currentRun = resumed;
|
releaseState.currentRun = resumed;
|
||||||
renderReleaseRun(resumed);
|
renderReleaseRun(resumed);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||||
elements.runStatus.textContent = "error";
|
elements.runStatus.textContent = "error";
|
||||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Resume failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||||
} finally {
|
} finally {
|
||||||
@@ -1280,6 +1283,7 @@
|
|||||||
releaseState.currentRun = retried;
|
releaseState.currentRun = retried;
|
||||||
renderReleaseRun(retried);
|
renderReleaseRun(retried);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Retry preparation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1300,6 +1304,7 @@
|
|||||||
releaseState.currentRun = reconciled;
|
releaseState.currentRun = reconciled;
|
||||||
renderReleaseRun(reconciled);
|
renderReleaseRun(reconciled);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (deterministicRunCommandError(error)) clearInFlightRunCommand(commandKey, commandRequestId);
|
||||||
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("error", "block")} Reconciliation failed</h3><p>${escapeHtml(error.message)}</p></div>`);
|
||||||
button.disabled = false;
|
button.disabled = false;
|
||||||
}
|
}
|
||||||
@@ -1315,6 +1320,88 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function pendingRunCommandRequest(commandKey, requestIdToReplay) {
|
||||||
|
if (typeof commandKey !== "string" || typeof requestIdToReplay !== "string") return null;
|
||||||
|
if (commandKey.startsWith("resume:")) {
|
||||||
|
const runId = commandKey.slice("resume:".length);
|
||||||
|
if (!runId || runId.includes(":")) return null;
|
||||||
|
return {
|
||||||
|
path: `/api/release-runs/${encodeURIComponent(runId)}/resume`,
|
||||||
|
body: { request_id: requestIdToReplay },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (commandKey.startsWith("retry:")) {
|
||||||
|
const remainder = commandKey.slice("retry:".length);
|
||||||
|
const separator = remainder.indexOf(":");
|
||||||
|
const runId = remainder.slice(0, separator);
|
||||||
|
const stepId = remainder.slice(separator + 1);
|
||||||
|
if (separator < 1 || !stepId) return null;
|
||||||
|
return {
|
||||||
|
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/retry`,
|
||||||
|
body: { request_id: requestIdToReplay },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (commandKey.startsWith("reconcile:")) {
|
||||||
|
const remainder = commandKey.slice("reconcile:".length);
|
||||||
|
const runSeparator = remainder.indexOf(":");
|
||||||
|
const outcomeSeparator = remainder.lastIndexOf(":");
|
||||||
|
const runId = remainder.slice(0, runSeparator);
|
||||||
|
const stepId = remainder.slice(runSeparator + 1, outcomeSeparator);
|
||||||
|
const outcome = remainder.slice(outcomeSeparator + 1);
|
||||||
|
if (runSeparator < 1 || outcomeSeparator <= runSeparator + 1 || !["effect_absent", "effect_succeeded", "unresolved"].includes(outcome)) return null;
|
||||||
|
return {
|
||||||
|
path: `/api/release-runs/${encodeURIComponent(runId)}/steps/${encodeURIComponent(stepId)}/reconcile`,
|
||||||
|
body: { request_id: requestIdToReplay, outcome, confirm: "RECONCILE" },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function recoverPendingRunCommands() {
|
||||||
|
const commands = readInFlightRunCommands();
|
||||||
|
const entries = Object.entries(commands);
|
||||||
|
if (!entries.length) return null;
|
||||||
|
let attempted = false;
|
||||||
|
let recoveredRun = null;
|
||||||
|
let pendingError = null;
|
||||||
|
let rejectedError = null;
|
||||||
|
for (const [commandKey, requestIdToReplay] of entries) {
|
||||||
|
const command = pendingRunCommandRequest(commandKey, requestIdToReplay);
|
||||||
|
if (!command) {
|
||||||
|
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
attempted = true;
|
||||||
|
try {
|
||||||
|
recoveredRun = await postJson(command.path, command.body);
|
||||||
|
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||||
|
} catch (error) {
|
||||||
|
if (deterministicRunCommandError(error)) {
|
||||||
|
clearInFlightRunCommand(commandKey, requestIdToReplay);
|
||||||
|
rejectedError = error;
|
||||||
|
} else {
|
||||||
|
pendingError = error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (recoveredRun) {
|
||||||
|
releaseState.currentRun = recoveredRun;
|
||||||
|
restoreReleaseRunSelection(recoveredRun);
|
||||||
|
renderReleaseRun(recoveredRun);
|
||||||
|
elements.releaseRun.value = recoveredRun.run_id;
|
||||||
|
}
|
||||||
|
if (pendingError) {
|
||||||
|
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("pending", "warn")} Saved command recovery is still pending</h3><p>${escapeHtml(pendingError.message)}</p><p class="action-meta">The same private request identifier will be replayed after refresh.</p></div>`);
|
||||||
|
} else if (rejectedError) {
|
||||||
|
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("rejected", "block")} Saved command was not accepted</h3><p>${escapeHtml(rejectedError.message)}</p><p class="action-meta">The server returned a deterministic client conflict; the stale request identifier was cleared.</p></div>`);
|
||||||
|
}
|
||||||
|
return attempted ? Boolean(recoveredRun) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function deterministicRunCommandError(error) {
|
||||||
|
return error?.status >= 400 && error.status < 500;
|
||||||
|
}
|
||||||
|
|
||||||
function inFlightRunCommandId(commandKey, prefix) {
|
function inFlightRunCommandId(commandKey, prefix) {
|
||||||
const commands = readInFlightRunCommands();
|
const commands = readInFlightRunCommands();
|
||||||
if (typeof commands[commandKey] === "string") return commands[commandKey];
|
if (typeof commands[commandKey] === "string") return commands[commandKey];
|
||||||
|
|||||||
Reference in New Issue
Block a user