GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix(release): make durable runs recovery safe

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 19:30:12 +02:00
parent 19c4b63ade
commit 22f5c2ff4e
5 changed files with 881 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

559
tests/test_release_run_state.py
View File

@@ -1,9 +1,13 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from itertools import count
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
import threading
@@ -11,6 +15,7 @@ import unittest
from unittest.mock import patch
from fastapi.testclient import TestClient
from filelock import FileLock
META_ROOT = Path(__file__).resolve().parents[1]
@@ -34,6 +39,7 @@ from server.app import ( # noqa: E402
WORKSPACE_FINGERPRINT = "a" * 64
CREATE_REQUESTS = count(1)
class ReleaseRunStoreTests(unittest.TestCase):
@@ -49,7 +55,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.temporary.cleanup()
def test_create_persists_private_immutable_plan_and_ordered_steps(self) -> None:
run = self.store.create(
run = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)
@@ -60,7 +67,7 @@ class ReleaseRunStoreTests(unittest.TestCase):
record_path = self.root / f"{run['run_id']}.json"
self.assertEqual("govoplan.release-run", reopened["schema"])
self.assertEqual(2, reopened["schema_version"])
self.assertEqual(3, reopened["schema_version"])
self.assertEqual(
{"govoplan-core": "1.2.3", "govoplan-files": "1.2.4"},
reopened["immutable"]["input"]["repo_versions"],
@@ -78,10 +85,43 @@ class ReleaseRunStoreTests(unittest.TestCase):
plan = release_plan()
plan["dry_run_steps"] = []
with self.assertRaisesRegex(ReleaseRunConflict, "step collection"):
self.store.create(input_snapshot=run_input(), plan_snapshot=plan)
create_run(self.store, input_snapshot=run_input(), plan_snapshot=plan)
def test_create_request_is_durable_idempotent_and_input_bound(self) -> None:
request_id = "durable-create-request-0001"
created = create_run(self.store, request_id=request_id)
changed_plan = release_plan()
changed_plan["dry_run_steps"][0]["title"] = "Dashboard drift" # type: ignore[index]
replayed = create_run(
self.store,
request_id=request_id,
plan_snapshot=changed_plan,
)
found = self.store.find_created_run(
request_id=request_id,
input_snapshot=run_input(),
)
self.assertEqual(created["run_id"], replayed["run_id"])
self.assertEqual(created["run_id"], found["run_id"] if found else None)
self.assertEqual(
"Inspect Core",
replayed["immutable"]["plan"]["dry_run_steps"][0]["title"],
)
self.assertEqual(1, len(list(self.root.glob("*.json"))))
changed_input = run_input()
changed_input["public_catalog"] = True
with self.assertRaisesRegex(ReleaseRunConflict, "different inputs"):
create_run(
self.store,
request_id=request_id,
input_snapshot=changed_input,
)
def test_attempt_transitions_are_idempotent_and_strictly_ordered(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
@@ -122,7 +162,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertTrue(finished["state"]["steps"][1]["available"])
def test_failed_attempt_and_delayed_start_replays_are_idempotent(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
attempt_id = "failed-attempt-replay-0001"
@@ -155,77 +196,129 @@ class ReleaseRunStoreTests(unittest.TestCase):
len(retried["state"]["events"]), len(delayed["state"]["events"])
)
def test_delayed_resume_request_is_idempotent_for_the_run_lifetime(self) -> None:
run_id = self.store.create(
def test_resume_requires_a_running_attempt_and_only_records_an_effect(self) -> None:
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
original_request = "resume-original-request-0001"
self.store.resume(run_id, request_id=original_request)
for index in range(130):
self.store.resume(
run_id, request_id=f"resume-intervening-request-{index:04d}"
)
running = self.store.start_step(
with self.assertRaisesRegex(ReleaseRunConflict, "only be resumed"):
self.store.resume(run_id, request_id=original_request)
before_start = self.store.get(run_id)
self.store.start_step(
run_id,
"core:preflight",
attempt_id="resume-delayed-attempt-0001",
)
resumed = self.store.resume(run_id, request_id=original_request)
duplicate = self.store.resume(run_id, request_id=original_request)
self.assertEqual("running", duplicate["state"]["steps"][0]["state"])
self.assertEqual("interrupted", duplicate["state"]["steps"][0]["state"])
self.assertEqual(
len(running["state"]["events"]), len(duplicate["state"]["events"])
len(resumed["state"]["events"]), len(duplicate["state"]["events"])
)
raw = json.loads(
(self.root / f"{run_id}.json").read_text(encoding="utf-8")
)
self.assertEqual(1, len(raw["state"]["processed_requests"]))
self.assertLess(len(before_start["state"]["events"]), len(resumed["state"]["events"]))
def test_command_and_attempt_capacity_fail_closed_without_eviction(self) -> None:
run_id = self.store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 2):
self.store.resume(run_id, request_id="capacity-resume-request-0001")
self.store.resume(run_id, request_id="capacity-resume-request-0002")
with self.assertRaisesRegex(ReleaseRunConflict, "safety limit"):
self.store.resume(run_id, request_id="capacity-resume-request-0003")
duplicate = self.store.resume(
run_id, request_id="capacity-resume-request-0001"
)
self.assertEqual("planned", duplicate["state"]["status"])
attempt_run_id = self.store.create(
def test_attempt_start_reserves_capacity_for_ambiguous_effect_recovery(self) -> None:
saturated_run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 1):
self.store.start_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
)
self.store.finish_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
succeeded=False,
result_code="preflight_failed",
)
self.store.retry_step(
attempt_run_id,
"core:preflight",
request_id="capacity-retry-request-0001",
)
with self.assertRaisesRegex(ReleaseRunConflict, "attempt history"):
with self.assertRaisesRegex(ReleaseRunConflict, "recovery capacity"):
self.store.start_step(
attempt_run_id,
saturated_run_id,
"core:preflight",
attempt_id="capacity-attempt-0002",
attempt_id="capacity-prestart-attempt-0001",
)
delayed_attempt = self.store.start_step(
attempt_run_id,
"core:preflight",
attempt_id="capacity-attempt-0001",
untouched = self.store.get(saturated_run_id)
self.assertEqual("pending", untouched["state"]["steps"][0]["state"])
self.assertEqual(0, untouched["state"]["steps"][0]["attempt_count"])
terminal_run_id = create_run(
self.store,
input_snapshot=run_input(),
plan_snapshot=mutating_first_plan(),
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 2):
self.store.start_step(
terminal_run_id, "core:tag", attempt_id="capacity-terminal-attempt"
)
self.assertEqual("pending", delayed_attempt["state"]["steps"][0]["state"])
self.store.resume(
terminal_run_id, request_id="capacity-terminal-resume-0001"
)
with self.assertRaisesRegex(ReleaseRunConflict, "safety limit"):
self.store.reconcile_step(
terminal_run_id,
"core:tag",
request_id="capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
completed = self.store.reconcile_step(
terminal_run_id,
"core:tag",
request_id="capacity-terminal-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("completed", completed["state"]["status"])
def test_unresolved_reconciliation_is_bounded_and_preserves_terminal_slot(self) -> None:
run_id = create_run(
self.store,
input_snapshot=run_input(),
plan_snapshot=mutating_first_plan(),
)["run_id"]
with patch("govoplan_release.release_run.MAX_COMMANDS", 3):
self.store.start_step(
run_id, "core:tag", attempt_id="capacity-unresolved-attempt"
)
self.store.resume(
run_id, request_id="capacity-unresolved-resume-0001"
)
unresolved = self.store.reconcile_step(
run_id,
"core:tag",
request_id="capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
duplicate = self.store.reconcile_step(
run_id,
"core:tag",
request_id="capacity-unresolved-request-0001",
outcome="unresolved",
confirmation="RECONCILE",
)
self.assertEqual(
len(unresolved["state"]["events"]),
len(duplicate["state"]["events"]),
)
with self.assertRaisesRegex(ReleaseRunConflict, "already recorded"):
self.store.reconcile_step(
run_id,
"core:tag",
request_id="capacity-unresolved-request-0002",
outcome="unresolved",
confirmation="RECONCILE",
)
completed = self.store.reconcile_step(
run_id,
"core:tag",
request_id="capacity-final-request-0001",
outcome="effect_succeeded",
confirmation="RECONCILE",
)
self.assertEqual("completed", completed["state"]["status"])
def test_reopen_and_resume_preserve_ambiguous_mutating_effect(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
)["run_id"]
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
@@ -281,7 +374,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
)
def test_reconciliation_can_record_unresolved_or_verified_success(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
)["run_id"]
self.store.start_step(run_id, "core:tag", attempt_id="reconcile-attempt-0001")
@@ -322,7 +416,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual("completed", succeeded["state"]["status"])
def test_tampered_record_fails_closed_and_is_not_rewritten(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
path = self.root / f"{run_id}.json"
@@ -338,7 +433,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual("unavailable", self.store.list()[0]["status"])
def test_broad_record_mode_and_symlinked_root_fail_closed(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
path = self.root / f"{run_id}.json"
@@ -357,8 +453,58 @@ class ReleaseRunStoreTests(unittest.TestCase):
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
).list()
def test_symlinked_lock_and_lock_io_errors_fail_closed(self) -> None:
self.store.list()
lock_path = self.root / ".release-runs.lock"
lock_path.unlink()
target = self.root / "lock-target"
target.write_text("do not follow", encoding="utf-8")
lock_path.symlink_to(target)
with self.assertRaisesRegex(ReleaseRunCorrupt, "lock path"):
self.store.list()
lock_path.unlink()
with patch(
"govoplan_release.release_run.FileLock.acquire",
side_effect=PermissionError("denied"),
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "acquired safely"):
self.store.list()
original_release = FileLock.release
def fail_operator_release(
lock: FileLock, *args: object, **kwargs: object
) -> None:
if kwargs.get("force") is True:
original_release(lock, *args, **kwargs) # type: ignore[arg-type]
return
raise PermissionError("denied")
with patch.object(
FileLock,
"release",
autospec=True,
side_effect=fail_operator_release,
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "released safely"):
self.store.list()
def test_atomic_write_oserror_is_normalized_and_leaves_no_record(self) -> None:
with patch(
"govoplan_release.release_run.os.replace",
side_effect=PermissionError("denied"),
):
with self.assertRaisesRegex(ReleaseRunCorrupt, "persisted safely"):
create_run(
self.store,
request_id="atomic-write-create-request-0001",
)
self.assertEqual([], list(self.root.glob("*.json")))
def test_state_specific_fields_and_boolean_counts_fail_closed(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
path = self.root / f"{run_id}.json"
@@ -372,7 +518,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.store.get(run_id)
def test_valid_looking_mutable_state_tampering_fails_record_checksum(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
)["run_id"]
self.store.start_step(run_id, "core:tag", attempt_id="checksum-attempt-0001")
@@ -396,7 +543,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual(tampered, path.read_bytes())
def test_impossible_step_chronology_fails_even_with_recomputed_checksum(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.start_step(
@@ -421,9 +569,13 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.store.get(run_id)
def test_record_and_retained_event_timestamps_are_strictly_ordered(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.start_step(
run_id, "core:preflight", attempt_id="timestamp-attempt-0001"
)
self.store.resume(run_id, request_id="timestamp-resume-request-0001")
path = self.root / f"{run_id}.json"
payload = json.loads(path.read_text(encoding="utf-8"))
@@ -439,7 +591,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.store.get(run_id)
def test_attempt_count_must_match_lifetime_attempt_history(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
self.store.start_step(
@@ -456,7 +609,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.store.get(run_id)
def test_workspace_fingerprint_is_enforced_for_every_store_operation(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
foreign = ReleaseRunStore(
@@ -470,7 +624,11 @@ class ReleaseRunStoreTests(unittest.TestCase):
with self.assertRaises(ReleaseRunNotFound):
foreign.resume(run_id, request_id="foreign-resume-request-0001")
with self.assertRaisesRegex(ReleaseRunConflict, "different workspace"):
foreign.create(input_snapshot=run_input(), plan_snapshot=release_plan())
create_run(
foreign,
input_snapshot=run_input(),
plan_snapshot=release_plan(),
)
def test_list_limit_is_applied_after_foreign_workspace_records_are_skipped(
self,
@@ -479,7 +637,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:00Z",
):
local_run = self.store.create(
local_run = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)
foreign_store = ReleaseRunStore(
@@ -492,7 +651,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
"govoplan_release.release_run._timestamp",
return_value="2026-07-22T00:00:01Z",
):
foreign_store.create(
create_run(
foreign_store,
input_snapshot=foreign_input, plan_snapshot=release_plan()
)
@@ -500,7 +660,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
barrier = threading.Barrier(2)
@@ -535,7 +696,8 @@ class ReleaseRunStoreTests(unittest.TestCase):
)
def test_event_history_is_bounded_and_command_history_is_not_evicted(self) -> None:
run_id = self.store.create(
run_id = create_run(
self.store,
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
for index in range(90):
@@ -588,7 +750,7 @@ class ReleaseRunApiTests(unittest.TestCase):
created_response = client.post(
"/api/release-runs",
headers=headers,
json=run_input(),
json=api_run_input(request_id="api-create-request-0001"),
)
self.assertEqual(200, created_response.status_code)
created = created_response.json()
@@ -623,6 +785,59 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertEqual(200, retried.status_code)
self.assertEqual("pending", retried.json()["state"]["steps"][0]["state"])
def test_create_requires_request_id_and_replays_before_plan_rebuild(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
app = create_app(
workspace_root=Path(temp_dir),
run_state_root=Path(temp_dir) / "runs",
token="token",
)
headers = {"X-Release-Console-Token": "token"}
dashboard = patch("server.app.build_dashboard", return_value=object())
planner = patch(
"server.app.build_selective_release_plan",
return_value=release_plan(),
)
with dashboard as build_dashboard, planner as build_plan, TestClient(
app
) as client:
missing = client.post(
"/api/release-runs",
headers=headers,
json={
key: value
for key, value in api_run_input().items()
if key != "request_id"
},
)
payload = api_run_input(request_id="api-idempotent-create-0001")
created = client.post(
"/api/release-runs", headers=headers, json=payload
)
self.assertEqual(200, created.status_code)
build_dashboard.reset_mock()
build_plan.reset_mock()
replayed = client.post(
"/api/release-runs", headers=headers, json=payload
)
mismatched = client.post(
"/api/release-runs",
headers=headers,
json=payload | {"public_catalog": True},
)
self.assertEqual(422, missing.status_code)
self.assertEqual(200, replayed.status_code)
self.assertEqual(created.json()["run_id"], replayed.json()["run_id"])
self.assertEqual(409, mismatched.status_code)
self.assertIn("different inputs", mismatched.json()["detail"])
build_dashboard.assert_not_called()
build_plan.assert_not_called()
self.assertEqual(
1, len(list(app.state.release_runs.root.glob("*.json")))
)
def test_create_rejects_unknown_or_unresolved_repository(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir) / "runs"
@@ -641,6 +856,7 @@ class ReleaseRunApiTests(unittest.TestCase):
"/api/release-runs",
headers={"X-Release-Console-Token": "token"},
json={
"request_id": "api-unknown-create-request-0001",
"channel": "stable",
"repo_versions": {"govoplan-unknown": "1.0.0"},
"public_catalog": False,
@@ -654,19 +870,20 @@ class ReleaseRunApiTests(unittest.TestCase):
def test_api_rejects_tampered_record_without_rewriting_it(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir) / "runs"
store = ReleaseRunStore(
root,
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
)
run_id = store.create(
input_snapshot=run_input(), plan_snapshot=release_plan()
)["run_id"]
path = root / f"{run_id}.json"
path.write_text("{}", encoding="utf-8")
malformed = path.read_bytes()
app = create_app(
workspace_root=Path(temp_dir), run_state_root=root, token="token"
)
run_id = create_run(
app.state.release_runs,
request_id="api-tamper-create-request-0001",
input_snapshot=run_input(
workspace_fingerprint=app.state.workspace_fingerprint
),
plan_snapshot=release_plan(),
)["run_id"]
path = app.state.release_runs.root / f"{run_id}.json"
path.write_text("{}", encoding="utf-8")
malformed = path.read_bytes()
with TestClient(app) as client:
response = client.get(
f"/api/release-runs/{run_id}",
@@ -712,7 +929,9 @@ class ReleaseRunApiTests(unittest.TestCase):
TestClient(app) as client,
):
created = client.post(
"/api/release-runs", headers=headers, json=run_input()
"/api/release-runs",
headers=headers,
json=api_run_input(request_id="api-reconcile-create-0001"),
).json()
run_id = created["run_id"]
app.state.release_runs.start_step(
@@ -807,14 +1026,19 @@ class ReleaseRunApiTests(unittest.TestCase):
response = client.post(
"/api/release-runs",
headers={"X-Release-Console-Token": "token"},
json=run_input(),
json=api_run_input(request_id="api-explicit-create-0001"),
)
self.assertEqual(200, response.status_code)
self.assertEqual(
release_workspace_fingerprint(first),
response.json()["immutable"]["input"]["workspace_fingerprint"],
)
self.assertEqual(explicit_root.resolve(), app.state.release_runs.root)
self.assertEqual(
explicit_root.resolve()
/ f"workspace-{release_workspace_fingerprint(first)}"
/ "release-runs",
app.state.release_runs.root,
)
def test_shared_explicit_root_does_not_cross_workspace_boundary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
@@ -837,9 +1061,16 @@ class ReleaseRunApiTests(unittest.TestCase):
TestClient(first_app) as client,
):
created = client.post(
"/api/release-runs", headers=headers, json=run_input()
"/api/release-runs",
headers=headers,
json=api_run_input(request_id="api-shared-create-0001"),
).json()
first_record = (
first_app.state.release_runs.root / f"{created['run_id']}.json"
)
first_record.write_text("{}", encoding="utf-8")
second_app = create_app(
workspace_root=second, run_state_root=root, token="token"
)
@@ -850,6 +1081,10 @@ class ReleaseRunApiTests(unittest.TestCase):
)
self.assertEqual([], listed.json()["runs"])
self.assertEqual(404, loaded.status_code)
self.assertNotEqual(
first_app.state.release_runs.root,
second_app.state.release_runs.root,
)
def test_ui_and_runbook_state_tracking_boundary_are_explicit(self) -> None:
webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
@@ -866,13 +1101,163 @@ class ReleaseRunApiTests(unittest.TestCase):
self.assertIn("effect_succeeded", webui)
self.assertIn("renderReleaseRunStoreUnavailable", webui)
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
self.assertIn("pendingReleaseRunPayload", webui)
self.assertIn("recoverPendingReleaseRun", webui)
self.assertIn("inFlightRunCommandId", webui)
self.assertIn("Run saved; list refresh unavailable", webui)
self.assertIn("This foundation tracks state only.", runbook)
self.assertIn("effect_absent", runbook)
self.assertIn("$XDG_STATE_HOME", runbook)
self.assertIn("same-directory", runbook)
self.assertIn("caller-generated `request_id`", runbook)
self.assertIn("reserves the two command-ledger", runbook)
def test_ui_reuses_uncertain_ids_and_separates_saved_run_list_failure(
self,
) -> None:
node = shutil.which("node")
if node is None:
self.skipTest("Node.js is required for the Release Console UI check")
webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
function_names = (
"pendingReleaseRunPayload",
"readPendingReleaseRun",
"clearPendingReleaseRun",
"recoverPendingReleaseRun",
"submitReleaseRunCreate",
"refreshReleaseRunListAfterCreate",
"readInFlightRunCommands",
"inFlightRunCommandId",
"clearInFlightRunCommand",
"requestId",
)
functions = "\n".join(
javascript_function(webui, name) for name in function_names
)
script = (
"""
const assert = (condition, message) => { if (!condition) throw new Error(message); };
const values = new Map();
const sessionStorage = {
getItem: (key) => values.has(key) ? values.get(key) : null,
setItem: (key, value) => values.set(key, String(value)),
removeItem: (key) => values.delete(key),
};
let uuidSequence = 0;
const window = { crypto: { randomUUID: () => `uuid-${++uuidSequence}` } };
const pendingReleaseRunKey = "test-pending-create";
const pendingRunCommandsKey = "test-pending-commands";
const releaseState = { currentRun: null, runStoreAvailable: true };
const elements = {
runStatus: { textContent: "" },
runOutput: {
innerHTML: "",
insertAdjacentHTML: (_position, html) => { elements.runOutput.innerHTML = html + elements.runOutput.innerHTML; },
},
releaseRun: { value: "" },
};
const pill = (label) => label;
const escapeHtml = (value) => String(value);
const restoreReleaseRunSelection = () => {};
const renderReleaseRun = (run) => { elements.runOutput.innerHTML = `RUN:${run.run_id}`; };
const renderReleaseRunList = () => {};
let failCreate = true;
let failList = true;
const postedRequestIds = [];
async function postJson(_path, payload) {
postedRequestIds.push(payload.request_id);
if (failCreate) throw new Error("uncertain transport failure");
return { run_id: "rr-known", immutable: { input: {} }, state: { steps: [] } };
}
async function api(_path) {
if (failList) throw new Error("list refresh failed");
return { runs: [] };
}
"""
+ functions
+ """
(async () => {
const input = { repo_versions: { "govoplan-files": "1.2.4", "govoplan-core": "1.2.3" }, channel: "stable" };
const first = pendingReleaseRunPayload(input);
const same = pendingReleaseRunPayload({ ...input, repo_versions: { "govoplan-core": "1.2.3", "govoplan-files": "1.2.4" } });
assert(first.request_id === same.request_id, "same create input did not reuse its request ID");
const uncertain = await submitReleaseRunCreate(first, { automatic: true });
assert(uncertain === false, "uncertain create unexpectedly succeeded");
assert(readPendingReleaseRun().payload.request_id === first.request_id, "uncertain create ID was cleared");
failCreate = false;
const recovered = await recoverPendingReleaseRun();
assert(recovered === true, "pending create did not recover");
assert(postedRequestIds.length === 2 && postedRequestIds.every((value) => value === first.request_id), "create replay changed request ID");
assert(readPendingReleaseRun() === null, "successful replay did not clear pending create");
assert(elements.runOutput.innerHTML.includes("Run saved; list refresh unavailable"), "list failure did not preserve saved state");
assert(!elements.runOutput.innerHTML.includes("Run creation failed"), "list failure was labeled as create failure");
assert(releaseState.currentRun.run_id === "rr-known", "saved run was not retained");
const commandOne = inFlightRunCommandId("retry:rr-known:step", "retry");
const commandReplay = inFlightRunCommandId("retry:rr-known:step", "retry");
assert(commandOne === commandReplay, "uncertain command did not reuse its request ID");
clearInFlightRunCommand("retry:rr-known:step", commandOne);
const commandAfterSuccess = inFlightRunCommandId("retry:rr-known:step", "retry");
assert(commandAfterSuccess !== commandOne, "successful command did not clear its request ID");
})().catch((error) => { console.error(error.stack || error.message); process.exit(1); });
"""
)
result = subprocess.run( # noqa: S603
[node, "-e", script],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(0, result.returncode, result.stderr)
def run_input() -> dict[str, object]:
def create_run(
store: ReleaseRunStore,
*,
request_id: str | None = None,
input_snapshot: dict[str, object] | None = None,
plan_snapshot: dict[str, object] | None = None,
) -> dict[str, object]:
return store.create(
request_id=request_id
or f"test-create-request-{next(CREATE_REQUESTS):06d}",
input_snapshot=input_snapshot if input_snapshot is not None else run_input(),
plan_snapshot=plan_snapshot if plan_snapshot is not None else release_plan(),
)
def api_run_input(
*, request_id: str | None = None, **overrides: object
) -> dict[str, object]:
payload = run_input()
payload.pop("workspace_fingerprint")
payload["request_id"] = request_id or (
f"api-create-request-{next(CREATE_REQUESTS):06d}"
)
payload.update(overrides)
return payload
def javascript_function(source: str, name: str) -> str:
match = re.search(
rf"\n\s+(?:async\s+)?function\s+{re.escape(name)}\s*\(", source
)
if match is None:
raise AssertionError(f"JavaScript function {name} was not found")
following = re.search(
r"\n\s+(?:async\s+)?function\s+[A-Za-z_$]",
source[match.end() :],
)
if following is None:
raise AssertionError(f"JavaScript function after {name} was not found")
end = match.end() + following.start()
return source[match.start() : end].strip()
def run_input(
*, workspace_fingerprint: str = WORKSPACE_FINGERPRINT
) -> dict[str, object]:
return {
"channel": "stable",
"repo_versions": {
@@ -883,7 +1268,7 @@ def run_input() -> dict[str, object]:
"remote_tags": False,
"public_catalog": False,
"include_migrations": True,
"workspace_fingerprint": WORKSPACE_FINGERPRINT,
"workspace_fingerprint": workspace_fingerprint,
}