GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix(release): preserve durable recovery capacity

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 19:48:55 +02:00
parent 22f5c2ff4e
commit 4c16069f88
4 changed files with 537 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
tests/test_release_run_state.py
View File

@@ -25,6 +25,7 @@ if str(RELEASE_ROOT) not in sys.path:
from govoplan_release.release_run import ( # noqa: E402
MAX_EVENTS,
MAX_RECORD_BYTES,
ReleaseRunConflict,
ReleaseRunCorrupt,
ReleaseRunNotFound,
@@ -268,6 +269,93 @@ class ReleaseRunStoreTests(unittest.TestCase):
)
self.assertEqual("completed", completed["state"]["status"])
def test_attempt_start_reserves_serialized_recovery_capacity(self) -> None:
rejected_plan = mutating_first_plan()
rejected_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 2_500)]
rejected_run_id = create_run(
self.store,
request_id="byte-capacity-rejected-create-0001",
plan_snapshot=rejected_plan,
)["run_id"]
rejected_path = self.root / f"{rejected_run_id}.json"
self.assertGreater(rejected_path.stat().st_size, MAX_RECORD_BYTES - 1_000)
with self.assertRaisesRegex(
ReleaseRunConflict, "serialized recovery capacity"
):
self.store.start_step(
rejected_run_id,
"core:tag",
attempt_id="byte-capacity-rejected-attempt-0001",
)
untouched = self.store.get(rejected_run_id)
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
accepted_plan = mutating_first_plan()
accepted_plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
accepted_run_id = create_run(
self.store,
request_id="byte-capacity-accepted-create-0001",
plan_snapshot=accepted_plan,
)["run_id"]
accepted_path = self.root / f"{accepted_run_id}.json"
self.store.start_step(
accepted_run_id,
"core:tag",
attempt_id="byte-capacity-accepted-attempt-0001",
)
self.store.resume(
accepted_run_id,
request_id="byte-capacity-accepted-resume-0001",
)
with self.assertRaisesRegex(
ReleaseRunConflict, "reserved for a terminal reconciliation"
):
self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
completed = self.store.reconcile_step(
accepted_run_id,
"core:tag",
request_id="byte-capacity-terminal-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("completed", completed["state"]["status"])
self.assertLessEqual(accepted_path.stat().st_size, MAX_RECORD_BYTES)
def test_read_only_start_reserves_serialized_resume_and_retry_capacity(
self,
) -> None:
plan = mutating_first_plan()
plan["dry_run_steps"][0]["mutating"] = False # type: ignore[index]
plan["notes"] = ["x" * (MAX_RECORD_BYTES - 3_000)]
run_id = create_run(
self.store,
request_id="byte-capacity-read-only-create-0001",
plan_snapshot=plan,
)["run_id"]
self.store.start_step(
run_id,
"core:tag",
attempt_id="byte-capacity-read-only-attempt-0001",
)
self.store.resume(
run_id,
request_id="byte-capacity-read-only-resume-0001",
)
retried = self.store.retry_step(
run_id,
"core:tag",
request_id="byte-capacity-read-only-retry-0001",
)
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
run_id = create_run(
self.store,
@@ -659,6 +747,61 @@ class ReleaseRunStoreTests(unittest.TestCase):
listed = self.store.list(limit=1)
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
def test_list_orders_verified_runs_by_updated_timestamp_before_unavailable(
self,
) -> None:
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:00Z",
):
older = create_run(
self.store,
request_id="timestamp-order-request-older-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T01:00:00Z",
):
newer = create_run(
self.store,
request_id="timestamp-order-request-newer-0001",
)
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T02:00:00Z",
):
self.store.start_step(
older["run_id"],
"core:preflight",
attempt_id="timestamp-order-attempt-0001",
)
corrupt = create_run(
self.store,
request_id="timestamp-order-request-corrupt-0001",
)
corrupt_path = self.root / f"{corrupt['run_id']}.json"
corrupt_path.write_text("{}", encoding="utf-8")
corrupt_path.chmod(0o600)
listed = self.store.list(limit=3)
self.assertEqual(
[older["run_id"], newer["run_id"], corrupt["run_id"]],
[item["run_id"] for item in listed],
)
self.assertEqual("unavailable", listed[-1]["status"])
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
def test_list_enumeration_oserror_is_normalized(self) -> None:
self.store.list()
with patch.object(
Path,
"iterdir",
side_effect=PermissionError("denied"),
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "enumerated safely"):
self.store.list()
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
run_id = create_run(
self.store,
@@ -1103,6 +1246,7 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
self.assertIn("pendingReleaseRunPayload", webui)
self.assertIn("recoverPendingReleaseRun", webui)
self.assertIn("recoverPendingRunCommands", webui)
self.assertIn("inFlightRunCommandId", webui)
self.assertIn("Run saved; list refresh unavailable", webui)
self.assertIn("This foundation tracks state only.", runbook)
@@ -1127,6 +1271,9 @@ class ReleaseRunApiTests(unittest.TestCase):
"submitReleaseRunCreate",
"refreshReleaseRunListAfterCreate",
"readInFlightRunCommands",
"pendingRunCommandRequest",
"recoverPendingRunCommands",
"deterministicRunCommandError",
"inFlightRunCommandId",
"clearInFlightRunCommand",
"requestId",
@@ -1163,10 +1310,17 @@ const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.ru
const renderReleaseRunList = () => {};
let failCreate = true;
let failList = true;
let failureStatus = null;
const postedRequestIds = [];
async function postJson(_path, payload) {
const postedPaths = [];
async function postJson(path, payload) {
postedPaths.push(path);
postedRequestIds.push(payload.request_id);
if (failCreate) throw new Error("uncertain transport failure");
if (failCreate) {
const error = new Error(failureStatus ? `server rejected ${failureStatus}` : "uncertain transport failure");
if (failureStatus) error.status = failureStatus;
throw error;
}
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
}
async function api(_path) {
@@ -1194,11 +1348,40 @@ async function api(_path) {
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry");
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry");
const commandKey = "retry:rr-known:core:preflight";
const commandOne = inFlightRunCommandId(commandKey, "retry");
const commandReplay = inFlightRunCommandId(commandKey, "retry");
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
clearInFlightRunCommand("retry:rr-known:step", commandOne);
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry");
const retryRequest = pendingRunCommandRequest(commandKey, commandOne);
assert(retryRequest.path.endsWith("/steps/core%3Apreflight/retry"), "retry recovery path was not executable");
const reconcileRequest = pendingRunCommandRequest("reconcile:rr-known:core:tag:effect_succeeded", "reconcile-request");
assert(reconcileRequest.path.endsWith("/steps/core%3Atag/reconcile"), "reconciliation recovery path was not executable");
assert(reconcileRequest.body.confirm === "RECONCILE" && reconcileRequest.body.outcome === "effect_succeeded", "reconciliation recovery body changed");
failCreate = true;
const uncertainCommand = await recoverPendingRunCommands();
assert(uncertainCommand === false, "uncertain command recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[commandKey] === commandOne, "uncertain command ID was cleared");
failCreate = false;
const recoveredCommand = await recoverPendingRunCommands();
assert(recoveredCommand === true, "pending command did not recover after reload");
assert(readInFlightRunCommands()[commandKey] === undefined, "replayed command ID was not cleared");
assert(postedRequestIds.slice(-2).every((value) => value === commandOne), "command replay changed request ID");
assert(postedPaths.at(-1).endsWith("/steps/core%3Apreflight/retry"), "command replay used the wrong endpoint");
assert(releaseState.currentRun.run_id === "rr-known", "recovered command did not select its run");
const conflictKey = "resume:rr-known";
const conflictId = inFlightRunCommandId(conflictKey, "resume");
failCreate = true;
failureStatus = 409;
const rejectedCommand = await recoverPendingRunCommands();
assert(rejectedCommand === false, "state-conflict recovery unexpectedly succeeded");
assert(readInFlightRunCommands()[conflictKey] === undefined, "deterministic 409 retained a stale command ID");
assert(elements.runOutput.innerHTML.includes("Saved command was not accepted"), "deterministic conflict was not explained");
failCreate = false;
failureStatus = null;
const commandAfterSuccess = inFlightRunCommandId(commandKey, "retry");
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
"""