GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat(release): bound durable run retention

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 20:16:19 +02:00
parent ddee5c00dc
commit 4edbc11fe5
5 changed files with 409 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
docs/RELEASE_CONSOLE.md
View File

@@ -66,9 +66,10 @@ channel, or gate input requires a new run.
Creation requires a caller-generated `request_id`. Its SHA-256 fingerprint is
the private, workspace-scoped durable mapping to exactly one run; the raw
identifier is never persisted. Repeating the same identifier with the same
immutable inputs returns that run without rebuilding the plan, even if the
live dashboard has since drifted. Reusing it with different inputs fails
closed. The browser keeps an uncertain create identifier in session storage,
immutable inputs returns that run without rebuilding the plan while it remains
inside the bounded local retention window, even if the live dashboard has
since drifted. Reusing it with different inputs fails closed. The browser keeps
an uncertain create identifier in session storage,
replays it after reload, and selects the known run returned by the server. A
successful create remains shown as saved if only the subsequent list refresh
fails.
@@ -86,6 +87,14 @@ modes, malformed schemas, unknown fields, invalid state combinations,
oversized files, and digest mismatches fail closed. A bad record is neither
rewritten nor automatically quarantined.
The store retains at most 512 workspace-scoped run records created through the
console. On creation at that limit it removes only the oldest fully completed,
integrity-verified record. Running, planned, attention, blocked, foreign, and
unreadable records are never deleted implicitly; if no completed record is
available, creation fails closed with a retention remediation. Unavailable
records still consume the bound and remain visible in cursor-paginated lists
so corruption cannot be hidden by normal turnover.
The full resolved-workspace SHA-256 is part of both the private storage
namespace and immutable input snapshot. Every list, read, and state transition
checks it. An alternate workspace therefore cannot list or resume another
@@ -99,7 +108,7 @@ Each frozen plan step has an explicit `pending`, `running`, `succeeded`,
`failed`, or `interrupted` state. Plan order remains a prerequisite: a later
step is unavailable until earlier steps have succeeded. Exact attempt and
resume/retry/reconciliation request identifiers are fingerprinted so delayed
repetitions remain idempotent for the run's lifetime. These fingerprints are
repetitions remain idempotent for the retained run's lifetime. These fingerprints are
never evicted: the store fails closed before accepting more than 2,048 commands
or 2,048 attempts and asks the operator to create a fresh run. The display
record keeps at most 256 server-generated state events. Events contain only an
@@ -143,9 +152,10 @@ other `/api/` route:
- `POST /api/release-runs` requires `request_id`, then idempotently rebuilds and
freezes a selective plan only when that creation is not already known.
- `GET /api/release-runs` lists bounded summaries ordered by verified
`updated_at`, then `created_at` and run identifier. Unreadable entries are
appended deterministically when room remains instead of displacing newer
verified runs.
`updated_at`, then `created_at` and run identifier. `next_cursor` advances a
stable descending traversal without offset duplicates. Unreadable entries
remain a deterministic final section and are therefore reachable through
pagination instead of displacing newer verified runs.
- `GET /api/release-runs/{run_id}` reads and verifies one exact record.
- `POST /api/release-runs/{run_id}/resume` records explicit recovery.
- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or

136
tests/test_release_run_state.py
View File

@@ -792,6 +792,97 @@ class ReleaseRunStoreTests(unittest.TestCase):
self.assertEqual("unavailable", listed[-1]["status"])
self.assertEqual(older["run_id"], self.store.list(limit=1)[0]["run_id"])
def test_cursor_pagination_is_stable_and_keeps_unavailable_records_visible(
self,
) -> None:
created: list[dict[str, object]] = []
for index in range(3):
with patch(
"govoplan_release.release_run._timestamp",
return_value=f"2026-07-22T0{index}:00:00Z",
):
created.append(
create_run(
self.store,
request_id=f"cursor-page-create-request-{index:04d}",
)
)
corrupt_path = self.root / f"{created[0]['run_id']}.json"
corrupt_path.write_text("{}", encoding="utf-8")
corrupt_path.chmod(0o600)
first = self.store.list_page(limit=1)
second = self.store.list_page(limit=1, cursor=first["next_cursor"])
third = self.store.list_page(limit=1, cursor=second["next_cursor"])
self.assertEqual(created[2]["run_id"], first["runs"][0]["run_id"])
self.assertEqual(created[1]["run_id"], second["runs"][0]["run_id"])
self.assertEqual("unavailable", third["runs"][0]["status"])
self.assertIsNone(third["next_cursor"])
with self.assertRaisesRegex(ReleaseRunConflict, "cursor is invalid"):
self.store.list_page(limit=1, cursor=f"{first['next_cursor']}tampered")
def test_retention_prunes_only_oldest_completed_records_and_fails_closed(
self,
) -> None:
with patch("govoplan_release.release_run.MAX_RUN_RECORDS", 3):
completed: list[dict[str, object]] = []
for index in range(2):
with patch(
"govoplan_release.release_run._timestamp",
return_value=f"2026-07-22T0{index}:00:00Z",
):
run = create_run(
self.store,
request_id=f"retention-completed-create-{index:04d}",
plan_snapshot=mutating_first_plan(),
)
attempt_id = f"retention-completed-attempt-{index:04d}"
self.store.start_step(
run["run_id"], "core:tag", attempt_id=attempt_id
)
self.store.finish_step(
run["run_id"],
"core:tag",
attempt_id=attempt_id,
succeeded=True,
result_code="tag_created",
)
completed.append(run)
active = create_run(
self.store,
request_id="retention-active-create-0001",
)
replacement = create_run(
self.store,
request_id="retention-replacement-create-0001",
)
self.assertFalse(
(self.root / f"{completed[0]['run_id']}.json").exists()
)
self.assertTrue((self.root / f"{completed[1]['run_id']}.json").exists())
self.assertTrue((self.root / f"{active['run_id']}.json").exists())
self.assertTrue((self.root / f"{replacement['run_id']}.json").exists())
corrupt_path = self.root / f"{completed[1]['run_id']}.json"
corrupt_path.write_text("{}", encoding="utf-8")
corrupt_path.chmod(0o600)
with self.assertRaisesRegex(ReleaseRunConflict, "retention limit"):
create_run(
self.store,
request_id="retention-refused-create-0001",
)
self.assertEqual(3, len(list(self.root.glob("*.json"))))
self.assertIn(
completed[1]["run_id"],
{
item["run_id"]
for item in self.store.list_page(limit=3)["runs"]
if item["status"] == "unavailable"
},
)
def test_list_enumeration_oserror_is_normalized(self) -> None:
self.store.list()
with patch.object(
@@ -872,6 +963,51 @@ class ReleaseRunStoreTests(unittest.TestCase):
class ReleaseRunApiTests(unittest.TestCase):
def test_api_exposes_and_validates_cursor_pagination(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
app = create_app(
workspace_root=Path(temp_dir),
run_state_root=Path(temp_dir) / "runs",
token="token",
)
headers = {"X-Release-Console-Token": "token"}
with (
patch("server.app.build_dashboard", return_value=object()),
patch(
"server.app.build_selective_release_plan",
return_value=release_plan(),
),
TestClient(app) as client,
):
for index in range(2):
response = client.post(
"/api/release-runs",
headers=headers,
json=api_run_input(
request_id=f"api-page-create-request-{index:04d}"
),
)
self.assertEqual(200, response.status_code)
first = client.get(
"/api/release-runs?limit=1", headers=headers
).json()
second = client.get(
"/api/release-runs",
headers=headers,
params={"limit": 1, "cursor": first["next_cursor"]},
).json()
invalid = client.get(
"/api/release-runs?cursor=not-a-cursor", headers=headers
)
self.assertEqual(1, len(first["runs"]))
self.assertEqual(1, len(second["runs"]))
self.assertNotEqual(
first["runs"][0]["run_id"], second["runs"][0]["run_id"]
)
self.assertIsNone(second["next_cursor"])
self.assertEqual(409, invalid.status_code)
def test_token_guarded_create_list_read_resume_and_retry(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir) / "runs"

229
tools/release/govoplan_release/release_run.py
View File

@@ -8,6 +8,8 @@ to this state in a later slice.
from __future__ import annotations
import base64
import binascii
from copy import deepcopy
from datetime import UTC, datetime
import hashlib
@@ -31,6 +33,8 @@ MAX_REPOSITORY_VERSIONS = 128
MAX_EVENTS = 256
MAX_COMMANDS = 2_048
MAX_LIST_LIMIT = 100
MAX_RUN_RECORDS = 512
MAX_CURSOR_BYTES = 1_024
RUN_ID_PATTERN = re.compile(
r"^(?:rr-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}|rr-request-[0-9a-f]{64})$"
)
@@ -165,7 +169,16 @@ class ReleaseRunStore:
input_snapshot=immutable_input,
)
return _view(existing)
retirement = self._retirement_candidate()
self._write_new(record)
if retirement is not None:
try:
self._unlink_retained_record(retirement)
except ReleaseRunError:
# Do not leave the store beyond its hard record bound when
# retirement fails before it can make room.
self._unlink_new_record(self._path(run_id))
raise
return _view(record)
def find_created_run(
@@ -199,39 +212,49 @@ class ReleaseRunStore:
return _view(record)
def list(self, *, limit: int = 20) -> list[dict[str, Any]]:
"""Return the first page for backwards-compatible local callers."""
return self.list_page(limit=limit)["runs"]
def list_page(
self, *, limit: int = 20, cursor: str | None = None
) -> dict[str, Any]:
"""Return a deterministic descending page, including corrupt entries."""
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
cursor_key = _decode_list_cursor(
cursor,
expected_workspace_fingerprint=self.expected_workspace_fingerprint,
)
with self._locked():
self._ensure_root()
try:
paths = [
path
for path in self.root.iterdir()
if path.name.endswith(".json")
and RUN_ID_PATTERN.fullmatch(path.stem)
]
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run storage could not be enumerated safely."
) from exc
summaries: list[dict[str, Any]] = []
unavailable: list[dict[str, Any]] = []
for path in paths:
entries: list[tuple[tuple[int, str, str, str], dict[str, Any]]] = []
for path in self._record_paths():
try:
record = self._read(path.stem)
except ReleaseRunWorkspaceMismatch:
continue
except (ReleaseRunCorrupt, ReleaseRunNotFound):
unavailable.append(
entries.append(
(
(0, "", "", path.stem),
{
"run_id": path.stem,
"status": "unavailable",
"error": "Run record is unavailable or failed integrity validation.",
}
},
)
)
continue
view = _view(record)
immutable_input = view["immutable"]["input"]
summaries.append(
entries.append(
(
(
1,
view["updated_at"],
view["created_at"],
view["run_id"],
),
{
"run_id": view["run_id"],
"created_at": view["created_at"],
@@ -240,18 +263,27 @@ class ReleaseRunStore:
"channel": immutable_input["channel"],
"repo_versions": immutable_input["repo_versions"],
"recommended_next": view["recommended_next"],
},
)
)
entries.sort(key=lambda item: item[0], reverse=True)
if cursor_key is not None:
entries = [entry for entry in entries if entry[0] < cursor_key]
page = entries[: limit + 1]
has_more = len(page) > limit
visible = page[:limit]
next_cursor = (
_encode_list_cursor(
visible[-1][0],
workspace_fingerprint=self.expected_workspace_fingerprint,
)
if has_more and visible
else None
)
return {
"runs": [summary for _key, summary in visible],
"next_cursor": next_cursor,
}
)
summaries.sort(
key=lambda item: (
item["updated_at"],
item["created_at"],
item["run_id"],
),
reverse=True,
)
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
return (summaries + unavailable)[:limit]
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
request_fingerprint = _request_fingerprint(request_id)
@@ -614,6 +646,88 @@ class ReleaseRunStore:
except ReleaseRunNotFound:
return None
def _record_paths(self) -> list[Path]:
self._ensure_root()
try:
return [
path
for path in self.root.iterdir()
if path.name.endswith(".json")
and RUN_ID_PATTERN.fullmatch(path.stem)
]
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run storage could not be enumerated safely."
) from exc
def _retirement_candidate(self) -> Path | None:
"""Select one verified terminal record without deleting it yet."""
paths = self._record_paths()
if len(paths) < MAX_RUN_RECORDS:
return None
if len(paths) > MAX_RUN_RECORDS:
raise ReleaseRunConflict(
"Release run storage exceeds its retention limit; inspect and "
"repair the private store before creating another run."
)
completed: list[tuple[tuple[str, str, str], Path]] = []
for path in paths:
try:
record = self._read(path.stem)
except (ReleaseRunCorrupt, ReleaseRunNotFound, ReleaseRunWorkspaceMismatch):
# Unavailable evidence is deliberately never deleted implicitly.
continue
if record["state"]["status"] == "completed":
completed.append(
(
(
record["updated_at"],
record["created_at"],
record["run_id"],
),
path,
)
)
completed.sort(key=lambda item: item[0])
if not completed:
raise ReleaseRunConflict(
"Release run retention limit is reached; complete or explicitly "
"remove retained runs before creating another run."
)
return completed[0][1]
def _unlink_retained_record(self, path: Path) -> None:
try:
metadata = path.lstat()
if (
not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink != 1
or (hasattr(os, "geteuid") and metadata.st_uid != os.geteuid())
or metadata.st_mode & 0o077
):
raise ReleaseRunCorrupt(
"Completed release run could not be retired safely."
)
path.unlink()
_fsync_directory(self.root)
except ReleaseRunError:
raise
except OSError as exc:
raise ReleaseRunCorrupt(
"Completed release run could not be retired safely."
) from exc
def _unlink_new_record(self, path: Path) -> None:
try:
path.unlink()
_fsync_directory(self.root)
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run creation could not be rolled back safely after a "
"retention failure. Inspect the private store before continuing."
) from exc
def _assert_workspace(self, record: dict[str, Any]) -> None:
if (
record["immutable"]["input"]["workspace_fingerprint"]
@@ -1736,6 +1850,63 @@ def _creation_run_id(request_fingerprint: str) -> str:
return f"rr-request-{request_fingerprint}"
def _encode_list_cursor(
key: tuple[int, str, str, str], *, workspace_fingerprint: str
) -> str:
payload = {
"v": 1,
"workspace": workspace_fingerprint,
"key": list(key),
}
encoded = base64.urlsafe_b64encode(_canonical_json(payload)).decode("ascii")
return encoded.rstrip("=")
def _decode_list_cursor(
cursor: str | None, *, expected_workspace_fingerprint: str
) -> tuple[int, str, str, str] | None:
if cursor is None or cursor == "":
return None
if not isinstance(cursor, str) or len(cursor) > MAX_CURSOR_BYTES:
raise ReleaseRunConflict("Release run list cursor is invalid.")
try:
padding = "=" * (-len(cursor) % 4)
encoded = base64.b64decode(
cursor + padding,
altchars=b"-_",
validate=True,
)
payload = json.loads(encoded.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError
key = payload.get("key")
if (
set(payload) != {"v", "workspace", "key"}
or payload.get("v") != 1
or payload.get("workspace") != expected_workspace_fingerprint
or not isinstance(key, list)
or len(key) != 4
or type(key[0]) is not int
or key[0] not in {0, 1}
or not all(isinstance(item, str) for item in key[1:])
or not RUN_ID_PATTERN.fullmatch(key[3])
):
raise ValueError
if key[0] == 1:
if not _is_timestamp(key[1]) or not _is_timestamp(key[2]):
raise ValueError
elif key[1] or key[2]:
raise ValueError
except (
binascii.Error,
UnicodeDecodeError,
ValueError,
json.JSONDecodeError,
) as exc:
raise ReleaseRunConflict("Release run list cursor is invalid.") from exc
return key[0], key[1], key[2], key[3]
def _validate_create_replay(
record: dict[str, Any],
*,

6
tools/release/server/app.py
View File

@@ -249,9 +249,11 @@ def create_app(
return to_jsonable(release_plan)
@app.get("/api/release-runs")
def release_runs(limit: int = 20) -> dict[str, object]:
def release_runs(
limit: int = 20, cursor: str | None = None
) -> dict[str, object]:
try:
return {"runs": app.state.release_runs.list(limit=limit)}
return app.state.release_runs.list_page(limit=limit, cursor=cursor)
except (ReleaseRunConflict, ReleaseRunCorrupt) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc

49
tools/release/webui/index.html
View File

@@ -650,6 +650,9 @@
<option value="">No saved run selected</option>
</select>
</div>
<div class="button-row">
<button class="secondary" id="loadOlderReleaseRuns" disabled>Load older</button>
</div>
</div>
<div class="button-row">
<button id="createReleaseRun">Create Run from Selection</button>
@@ -884,6 +887,7 @@
commitPrepare: document.getElementById("commitPrepare"),
buildSelectedPlan: document.getElementById("buildSelectedPlan"),
releaseRun: document.getElementById("releaseRun"),
loadOlderReleaseRuns: document.getElementById("loadOlderReleaseRuns"),
runStatus: document.getElementById("runStatus"),
runOutput: document.getElementById("runOutput"),
createReleaseRun: document.getElementById("createReleaseRun"),
@@ -907,6 +911,7 @@
manualRepo: "",
currentRun: null,
runs: [],
runNextCursor: null,
runStoreAvailable: true,
};
const pendingReleaseRunKey = "govoplanReleaseConsolePendingRunCreate";
@@ -928,6 +933,7 @@
elements.createReleaseRun.addEventListener("click", createReleaseRun);
elements.resumeReleaseRun.addEventListener("click", resumeReleaseRun);
elements.releaseRun.addEventListener("change", () => loadReleaseRun(elements.releaseRun.value));
elements.loadOlderReleaseRuns.addEventListener("click", loadOlderReleaseRuns);
elements.previewReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: false, push: true }));
elements.createReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: true, push: false }));
elements.publishReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: true, push: true }));
@@ -943,9 +949,11 @@
});
function api(path) {
const query = new URLSearchParams();
const separator = path.indexOf("?");
const basePath = separator === -1 ? path : path.slice(0, separator);
const query = new URLSearchParams(separator === -1 ? "" : path.slice(separator + 1));
if (elements.channel.value.trim()) query.set("channel", elements.channel.value.trim());
if (path === "/api/selective-plan") {
if (basePath === "/api/selective-plan") {
const selected = selectedRepoVersions();
const repoNames = Object.keys(selected);
if (repoNames.length) {
@@ -957,9 +965,9 @@
else query.set("public_catalog", "false");
if (elements.online.checked) query.set("remote_tags", "true");
if (elements.migrations.checked) query.set("include_migrations", "true");
if (elements.website.checked && path === "/api/dashboard") query.set("include_website", "true");
if (elements.website.checked && basePath === "/api/dashboard") query.set("include_website", "true");
const suffix = query.toString() ? `?${query}` : "";
return fetch(`${path}${suffix}`, {
return fetch(`${basePath}${suffix}`, {
headers: token ? { "X-Release-Console-Token": token } : {},
}).then((response) => {
if (!response.ok) {
@@ -1005,7 +1013,7 @@
if (runsResult.error) {
renderReleaseRunStoreUnavailable(runsResult.error);
} else {
renderReleaseRunList(runsResult.data?.runs || []);
renderReleaseRunList(runsResult.data?.runs || [], { nextCursor: runsResult.data?.next_cursor || null });
const pendingCreateResult = await recoverPendingReleaseRun();
const pendingCommandResult = await recoverPendingRunCommands();
const recoveryAttempted = pendingCreateResult !== null || pendingCommandResult !== null;
@@ -1039,11 +1047,16 @@
renderCatalog(data.catalog, data.target_version);
}
function renderReleaseRunList(runs) {
function renderReleaseRunList(runs, options = {}) {
releaseState.runStoreAvailable = true;
releaseState.runs = Array.isArray(runs) ? runs : [];
const incoming = Array.isArray(runs) ? runs : [];
releaseState.runs = options.append
? [...new Map([...releaseState.runs, ...incoming].map((run) => [run.run_id, run])).values()]
: incoming;
releaseState.runNextCursor = options.nextCursor || null;
elements.releaseRun.disabled = false;
elements.createReleaseRun.disabled = false;
elements.loadOlderReleaseRuns.disabled = !releaseState.runNextCursor;
const selectedId = releaseState.currentRun?.run_id || "";
const options = releaseState.runs.map((run) => {
const repositories = run.repo_versions ? Object.keys(run.repo_versions).length : 0;
@@ -1060,9 +1073,11 @@
releaseState.runStoreAvailable = false;
releaseState.runs = [];
releaseState.currentRun = null;
releaseState.runNextCursor = null;
elements.releaseRun.innerHTML = `<option value="">Run storage unavailable</option>`;
elements.releaseRun.disabled = true;
elements.createReleaseRun.disabled = true;
elements.loadOlderReleaseRuns.disabled = true;
elements.resumeReleaseRun.disabled = true;
elements.runStatus.textContent = "unavailable";
elements.runOutput.innerHTML = `<div class="action"><h3>${pill("unavailable", "block")} Durable run storage cannot be read</h3><p>${escapeHtml(error.message)}</p><p class="action-meta">Dashboard and release previews remain available. Correct the private state-directory problem, then refresh.</p></div>`;
@@ -1157,13 +1172,31 @@
async function refreshReleaseRunListAfterCreate(run) {
try {
const runs = await api("/api/release-runs");
renderReleaseRunList(runs.runs || []);
renderReleaseRunList(runs.runs || [], { nextCursor: runs.next_cursor || null });
elements.releaseRun.value = run.run_id;
} catch (error) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("saved", "warn")} Run saved; list refresh unavailable</h3><p>${escapeHtml(error.message)}</p><p class="action-meta">The persisted run remains selected and can be reloaded after refresh.</p></div>`);
}
}
async function loadOlderReleaseRuns() {
const cursor = releaseState.runNextCursor;
if (!cursor) return;
setBusy([elements.loadOlderReleaseRuns], true);
try {
const page = await api(`/api/release-runs?cursor=${encodeURIComponent(cursor)}`);
renderReleaseRunList(page.runs || [], {
append: true,
nextCursor: page.next_cursor || null,
});
} catch (error) {
elements.runOutput.insertAdjacentHTML("afterbegin", `<div class="action"><h3>${pill("unavailable", "warn")} Older runs could not be loaded</h3><p>${escapeHtml(error.message)}</p></div>`);
} finally {
setBusy([elements.loadOlderReleaseRuns], false);
elements.loadOlderReleaseRuns.disabled = !releaseState.runNextCursor;
}
}
async function loadReleaseRun(runId, options = {}) {
if (!runId) {
releaseState.currentRun = null;