GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix(release): harden durable run recovery

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 18:58:54 +02:00
parent ead697d049
commit 1dc9148ec3
5 changed files with 1046 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

475
tests/test_release_run_state.py
View File

@@ -22,7 +22,9 @@ from govoplan_release.release_run import ( # noqa: E402
MAX_EVENTS,
ReleaseRunConflict,
ReleaseRunCorrupt,
ReleaseRunNotFound,
ReleaseRunStore,
_record_digest,
)
from server.app import ( # noqa: E402
create_app,
@@ -31,11 +33,17 @@ from server.app import ( # noqa: E402
)
WORKSPACE_FINGERPRINT = "a" * 64
class ReleaseRunStoreTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name) / "release-runs"
self.store = ReleaseRunStore(self.root)
self.store = ReleaseRunStore(
self.root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
)
def tearDown(self) -> None:
self.temporary.cleanup()
@@ -45,11 +53,14 @@ class ReleaseRunStoreTests(unittest.TestCase):
input_snapshot=run_input(), plan_snapshot=release_plan()
)
reopened = ReleaseRunStore(self.root).get(run["run_id"])
reopened = ReleaseRunStore(
self.root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
).get(run["run_id"])
record_path = self.root / f"{run['run_id']}.json"
self.assertEqual("govoplan.release-run", reopened["schema"])
self.assertEqual(1, reopened["schema_version"])
self.assertEqual(2, reopened["schema_version"])
self.assertEqual(
{"govoplan-core": "1.2.3", "govoplan-files": "1.2.4"},
reopened["immutable"]["input"]["repo_versions"],
@@ -63,6 +74,12 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertNotIn("processed_requests", reopened["state"])
self.assertNotIn("attempt_fingerprint", reopened["state"]["steps"][0])
def test_zero_step_plan_is_rejected(self) -> None:
plan = release_plan()
plan["dry_run_steps"] = []
with self.assertRaisesRegex(ReleaseRunConflict, "step collection"):
self.store.create(input_snapshot=run_input(), plan_snapshot=plan)
def test_attempt_transitions_are_idempotent_and_strictly_ordered(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
@@ -104,13 +121,119 @@ class ReleaseRunStoreTests(unittest.TestCase):
)
self.assertTrue(finished["state"]["steps"][1]["available"])
def test_failed_attempt_and_delayed_start_replays_are_idempotent(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
attempt_id = "failed-attempt-replay-0001"
self.store.start_step(run_id, "core:preflight", attempt_id=attempt_id)
failed = self.store.finish_step(
run_id,
"core:preflight",
attempt_id=attempt_id,
succeeded=False,
result_code="preflight_failed",
)
duplicate = self.store.start_step(
run_id, "core:preflight", attempt_id=attempt_id
)
self.assertEqual("failed", duplicate["state"]["steps"][0]["state"])
self.assertEqual(
len(failed["state"]["events"]), len(duplicate["state"]["events"])
)
retried = self.store.retry_step(
run_id,
"core:preflight",
request_id="failed-retry-request-0001",
)
delayed = self.store.start_step(
run_id, "core:preflight", attempt_id=attempt_id
)
self.assertEqual("pending", delayed["state"]["steps"][0]["state"])
self.assertEqual(
len(retried["state"]["events"]), len(delayed["state"]["events"])
)
def test_delayed_resume_request_is_idempotent_for_the_run_lifetime(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
original_request = "resume-original-request-0001"
self.store.resume(run_id, request_id=original_request)
for index in range(130):
self.store.resume(
run_id, request_id=f"resume-intervening-request-{index:04d}"
)
running = self.store.start_step(
run_id,
"core:preflight",
attempt_id="resume-delayed-attempt-0001",
)
duplicate = self.store.resume(run_id, request_id=original_request)
self.assertEqual("running", duplicate["state"]["steps"][0]["state"])
self.assertEqual(
len(running["state"]["events"]), len(duplicate["state"]["events"])
)
def test_command_and_attempt_capacity_fail_closed_without_eviction(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 2):
self.store.resume(run_id, request_id="capacity-resume-request-0001")
self.store.resume(run_id, request_id="capacity-resume-request-0002")
with self.assertRaisesRegex(ReleaseRunConflict, "safety limit"):
self.store.resume(run_id, request_id="capacity-resume-request-0003")
duplicate = self.store.resume(
run_id, request_id="capacity-resume-request-0001"
)
self.assertEqual("planned", duplicate["state"]["status"])
attempt_run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 1):
self.store.start_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
)
self.store.finish_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
succeeded=False,
result_code="preflight_failed",
)
self.store.retry_step(
attempt_run_id,
"core:preflight",
request_id="capacity-retry-request-0001",
)
with self.assertRaisesRegex(ReleaseRunConflict, "attempt history"):
self.store.start_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0002",
)
delayed_attempt = self.store.start_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
)
self.assertEqual("pending", delayed_attempt["state"]["steps"][0]["state"])
def test_reopen_and_resume_preserve_ambiguous_mutating_effect(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
)["run_id"]
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
reopened_store = ReleaseRunStore(self.root)
reopened_store = ReleaseRunStore(
self.root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
)
before_resume = reopened_store.get(run_id)
self.assertEqual("running", before_resume["state"]["steps"][0]["state"])
@@ -118,28 +241,38 @@ class ReleaseRunStoreTests(unittest.TestCase):
duplicate = reopened_store.resume(run_id, request_id="resume-request-0001")
self.assertEqual("interrupted", resumed["state"]["steps"][0]["state"])
self.assertEqual("reconcile_step", resumed["recommended_next"]["id"])
self.assertFalse(resumed["recommended_next"]["available"])
self.assertTrue(resumed["recommended_next"]["available"])
self.assertEqual(
len(resumed["state"]["events"]), len(duplicate["state"]["events"])
)
with self.assertRaisesRegex(ReleaseRunConflict, "must be reconciled"):
with self.assertRaisesRegex(ReleaseRunConflict, "observed effect"):
reopened_store.retry_step(
run_id,
"core:tag",
request_id="retry-request-0001",
)
retried = reopened_store.retry_step(
with self.assertRaisesRegex(ReleaseRunConflict, "Type RECONCILE"):
reopened_store.reconcile_step(
run_id,
"core:tag",
request_id="reconcile-request-0001",
outcome="effect_absent",
confirmation="",
)
retried = reopened_store.reconcile_step(
run_id,
"core:tag",
request_id="retry-request-0002",
reconciled=True,
request_id="reconcile-request-0002",
outcome="effect_absent",
confirmation="RECONCILE",
)
duplicate_retry = reopened_store.retry_step(
duplicate_retry = reopened_store.reconcile_step(
run_id,
"core:tag",
request_id="retry-request-0002",
reconciled=True,
request_id="reconcile-request-0002",
outcome="effect_absent",
confirmation="RECONCILE",
)
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
self.assertEqual(
@@ -147,6 +280,47 @@ class ReleaseRunStoreTests(unittest.TestCase):
len(duplicate_retry["state"]["events"]),
)
def test_reconciliation_can_record_unresolved_or_verified_success(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
)["run_id"]
self.store.start_step(run_id, "core:tag", attempt_id="reconcile-attempt-0001")
self.store.resume(run_id, request_id="reconcile-resume-request-0001")
unresolved = self.store.reconcile_step(
run_id,
"core:tag",
request_id="reconcile-outcome-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
self.assertEqual("interrupted", unresolved["state"]["steps"][0]["state"])
self.assertEqual(
"unresolved", unresolved["state"]["events"][-1]["result_code"]
)
with self.assertRaisesRegex(ReleaseRunConflict, "different run-state action"):
self.store.reconcile_step(
run_id,
"core:tag",
request_id="reconcile-outcome-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
succeeded = self.store.reconcile_step(
run_id,
"core:tag",
request_id="reconcile-outcome-request-0002",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("succeeded", succeeded["state"]["steps"][0]["state"])
self.assertEqual(
"reconciled_effect_succeeded",
succeeded["state"]["steps"][0]["result_code"],
)
self.assertEqual("completed", succeeded["state"]["status"])
def test_tampered_record_fails_closed_and_is_not_rewritten(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
@@ -178,7 +352,10 @@ class ReleaseRunStoreTests(unittest.TestCase):
linked = Path(self.temporary.name) / "linked"
linked.symlink_to(target, target_is_directory=True)
with self.assertRaisesRegex(ReleaseRunCorrupt, "symbolic-link"):
ReleaseRunStore(linked).list()
ReleaseRunStore(
linked,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
).list()
def test_state_specific_fields_and_boolean_counts_fail_closed(self) -> None:
run_id = self.store.create(
@@ -218,6 +395,110 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual(tampered, path.read_bytes())
def test_impossible_step_chronology_fails_even_with_recomputed_checksum(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.start_step(
run_id, "core:preflight", attempt_id="chronology-attempt-0001"
)
self.store.finish_step(
run_id,
"core:preflight",
attempt_id="chronology-attempt-0001",
succeeded=True,
result_code="preflight_valid",
)
path = self.root / f"{run_id}.json"
payload = json.loads(path.read_text(encoding="utf-8"))
payload["state"]["steps"][0]["started_at"] = "2026-07-22T20:00:00Z"
payload["state"]["steps"][0]["finished_at"] = "2026-07-22T19:00:00Z"
payload["record_digest"] = _record_digest(payload)
path.write_text(json.dumps(payload), encoding="utf-8")
path.chmod(0o600)
with self.assertRaises(ReleaseRunCorrupt):
self.store.get(run_id)
def test_record_and_retained_event_timestamps_are_strictly_ordered(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.resume(run_id, request_id="timestamp-resume-request-0001")
path = self.root / f"{run_id}.json"
payload = json.loads(path.read_text(encoding="utf-8"))
payload["created_at"] = "2026-07-22T00:00:00Z"
payload["updated_at"] = "2026-07-22T00:00:03Z"
payload["state"]["events"][0]["at"] = "2026-07-22T00:00:02Z"
payload["state"]["events"][1]["at"] = "2026-07-22T00:00:01Z"
payload["record_digest"] = _record_digest(payload)
path.write_text(json.dumps(payload), encoding="utf-8")
path.chmod(0o600)
with self.assertRaises(ReleaseRunCorrupt):
self.store.get(run_id)
def test_attempt_count_must_match_lifetime_attempt_history(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.start_step(
run_id, "core:preflight", attempt_id="counted-attempt-0001"
)
path = self.root / f"{run_id}.json"
payload = json.loads(path.read_text(encoding="utf-8"))
payload["state"]["steps"][0]["attempt_count"] = 2
payload["record_digest"] = _record_digest(payload)
path.write_text(json.dumps(payload), encoding="utf-8")
path.chmod(0o600)
with self.assertRaises(ReleaseRunCorrupt):
self.store.get(run_id)
def test_workspace_fingerprint_is_enforced_for_every_store_operation(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
foreign = ReleaseRunStore(
self.root,
expected_workspace_fingerprint="b" * 64,
)
self.assertEqual([], foreign.list())
with self.assertRaises(ReleaseRunNotFound):
foreign.get(run_id)
with self.assertRaises(ReleaseRunNotFound):
foreign.resume(run_id, request_id="foreign-resume-request-0001")
with self.assertRaisesRegex(ReleaseRunConflict, "different workspace"):
foreign.create(input_snapshot=run_input(), plan_snapshot=release_plan())
def test_list_limit_is_applied_after_foreign_workspace_records_are_skipped(
self,
) -> None:
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:00Z",
):
local_run = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)
foreign_store = ReleaseRunStore(
self.root,
expected_workspace_fingerprint="b" * 64,
)
foreign_input = run_input()
foreign_input["workspace_fingerprint"] = "b" * 64
with patch(
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:01Z",
):
foreign_store.create(
input_snapshot=foreign_input, plan_snapshot=release_plan()
)
listed = self.store.list(limit=1)
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
@@ -225,7 +506,10 @@ class ReleaseRunStoreTests(unittest.TestCase):
barrier = threading.Barrier(2)
def claim(attempt_id: str) -> str:
contender = ReleaseRunStore(self.root)
contender = ReleaseRunStore(
self.root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
)
barrier.wait(timeout=5)
try:
contender.start_step(run_id, "core:preflight", attempt_id=attempt_id)
@@ -250,7 +534,7 @@ class ReleaseRunStoreTests(unittest.TestCase):
),
)
def test_event_and_request_history_are_bounded_and_code_only(self) -> None:
def test_event_history_is_bounded_and_command_history_is_not_evicted(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
@@ -273,7 +557,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
view = self.store.get(run_id)
raw = json.loads((self.root / f"{run_id}.json").read_text(encoding="utf-8"))
self.assertEqual(MAX_EVENTS, len(view["state"]["events"]))
self.assertLessEqual(len(raw["state"]["processed_requests"]), 128)
self.assertEqual(90, len(raw["state"]["processed_requests"]))
self.assertEqual(90, len(raw["state"]["processed_attempts"]))
self.assertNotIn("bounded-retry-0089", json.dumps(raw))
for event in view["state"]["events"]:
self.assertLessEqual(
@@ -369,7 +654,10 @@ class ReleaseRunApiTests(unittest.TestCase):
def test_api_rejects_tampered_record_without_rewriting_it(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir) / "runs"
store = ReleaseRunStore(root)
store = ReleaseRunStore(
root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
)
run_id = store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
@@ -388,26 +676,119 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertEqual(409, response.status_code)
self.assertEqual(malformed, path.read_bytes())
def test_api_maps_unavailable_run_storage_without_server_exception(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
parent = Path(temp_dir)
target = parent / "target"
target.mkdir()
linked_root = parent / "linked-runs"
linked_root.symlink_to(target, target_is_directory=True)
app = create_app(
workspace_root=parent,
run_state_root=linked_root,
token="token",
)
with TestClient(app) as client:
response = client.get(
"/api/release-runs",
headers={"X-Release-Console-Token": "token"},
)
self.assertEqual(409, response.status_code)
self.assertIn("symbolic-link", response.json()["detail"])
def test_api_requires_confirmation_and_records_reconciliation_outcome(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir) / "runs"
app = create_app(
workspace_root=Path(temp_dir), run_state_root=root, token="token"
)
headers = {"X-Release-Console-Token": "token"}
with (
patch("server.app.build_dashboard", return_value=object()),
patch(
"server.app.build_selective_release_plan",
return_value=mutating_first_plan(),
),
TestClient(app) as client,
):
created = client.post(
"/api/release-runs", headers=headers, json=run_input()
).json()
run_id = created["run_id"]
app.state.release_runs.start_step(
run_id, "core:tag", attempt_id="api-reconcile-attempt-0001"
)
client.post(
f"/api/release-runs/{run_id}/resume",
headers=headers,
json={"request_id": "api-reconcile-resume-0001"},
)
missing_confirmation = client.post(
f"/api/release-runs/{run_id}/steps/core:tag/reconcile",
headers=headers,
json={
"request_id": "api-reconcile-request-0001",
"outcome": "effect_succeeded",
},
)
reconciled = client.post(
f"/api/release-runs/{run_id}/steps/core:tag/reconcile",
headers=headers,
json={
"request_id": "api-reconcile-request-0002",
"outcome": "effect_succeeded",
"confirm": "RECONCILE",
},
)
self.assertEqual(409, missing_confirmation.status_code)
self.assertEqual(200, reconciled.status_code)
self.assertEqual("completed", reconciled.json()["state"]["status"])
self.assertEqual(
"effect_succeeded",
reconciled.json()["state"]["events"][-1]["result_code"],
)
def test_default_storage_and_immutable_input_are_workspace_bound(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
parent = Path(temp_dir)
state_home = parent / "operator-state"
first = parent / "first"
second = parent / "second"
first.mkdir()
second.mkdir()
first_root = default_release_run_root(first)
second_root = default_release_run_root(second)
with patch.dict(
os.environ, {"XDG_STATE_HOME": str(state_home)}, clear=False
):
first_root = default_release_run_root(first)
second_root = default_release_run_root(second)
default_app = create_app(workspace_root=first, token="token")
with TestClient(default_app) as client:
default_list = client.get(
"/api/release-runs",
headers={"X-Release-Console-Token": "token"},
)
self.assertEqual(200, default_list.status_code)
self.assertEqual(0o700, first_root.stat().st_mode & 0o777)
self.assertNotEqual(first_root, second_root)
self.assertIn(release_workspace_fingerprint(first)[:16], first_root.name)
self.assertEqual("release-runs", first_root.name)
self.assertEqual(state_home, first_root.parents[3])
self.assertEqual(
f"workspace-{release_workspace_fingerprint(first)}",
first_root.parent.name,
)
self.assertFalse(first_root.is_relative_to(first))
local_workspace = parent / "local"
local_meta = local_workspace / "govoplan"
local_meta.mkdir(parents=True)
(local_meta / "repositories.json").write_text("{}", encoding="utf-8")
self.assertEqual(
local_meta / "runtime" / "release-runs",
default_release_run_root(local_workspace),
)
with patch.dict(
os.environ, {"XDG_STATE_HOME": str(state_home)}, clear=False
):
self.assertTrue(
default_release_run_root(local_workspace).is_relative_to(state_home)
)
explicit_root = parent / "explicit"
app = create_app(
@@ -435,6 +816,41 @@ class ReleaseRunApiTests(unittest.TestCase):
)
self.assertEqual(explicit_root.resolve(), app.state.release_runs.root)
def test_shared_explicit_root_does_not_cross_workspace_boundary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
parent = Path(temp_dir)
root = parent / "shared-runs"
first = parent / "first"
second = parent / "second"
first.mkdir()
second.mkdir()
first_app = create_app(
workspace_root=first, run_state_root=root, token="token"
)
headers = {"X-Release-Console-Token": "token"}
with (
patch("server.app.build_dashboard", return_value=object()),
patch(
"server.app.build_selective_release_plan",
return_value=release_plan(),
),
TestClient(first_app) as client,
):
created = client.post(
"/api/release-runs", headers=headers, json=run_input()
).json()
second_app = create_app(
workspace_root=second, run_state_root=root, token="token"
)
with TestClient(second_app) as client:
listed = client.get("/api/release-runs", headers=headers)
loaded = client.get(
f"/api/release-runs/{created['run_id']}", headers=headers
)
self.assertEqual([], listed.json()["runs"])
self.assertEqual(404, loaded.status_code)
def test_ui_and_runbook_state_tracking_boundary_are_explicit(self) -> None:
webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
runbook = (META_ROOT / "docs" / "RELEASE_CONSOLE.md").read_text(
@@ -446,9 +862,14 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertIn("Create Run from Selection", webui)
self.assertIn("Prepare Retry", webui)
self.assertIn("step.retry_available", webui)
self.assertIn("Record Reconciliation", webui)
self.assertIn("effect_succeeded", webui)
self.assertIn("renderReleaseRunStoreUnavailable", webui)
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
self.assertIn("This foundation tracks state only.", runbook)
self.assertIn("mutating step remains unavailable", runbook)
self.assertIn("same-directory temporary file", runbook)
self.assertIn("effect_absent", runbook)
self.assertIn("$XDG_STATE_HOME", runbook)
self.assertIn("same-directory", runbook)
def run_input() -> dict[str, object]:
@@ -462,7 +883,7 @@ def run_input() -> dict[str, object]:
"remote_tags": False,
"public_catalog": False,
"include_migrations": True,
"workspace_fingerprint": "a" * 64,
"workspace_fingerprint": WORKSPACE_FINGERPRINT,
}