From d0dc91683723f13fc7dc63161d6c5ce95a058dbf Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Wed, 22 Jul 2026 18:30:40 +0200 Subject: [PATCH] feat(release): persist resumable run state --- docs/RELEASE_CONSOLE.md | 65 + tests/test_release_run_state.py | 520 ++++++++ tools/release/govoplan_release/release_run.py | 1154 +++++++++++++++++ tools/release/server/app.py | 149 ++- tools/release/webui/index.html | 199 ++- 5 files changed, 2082 insertions(+), 5 deletions(-) create mode 100644 tests/test_release_run_state.py create mode 100644 tools/release/govoplan_release/release_run.py diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index 4bbdbc5..6ddef4d 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -18,6 +18,7 @@ candidate/publish actions: - configure target versions per release unit in the web UI - select the repositories that should advance through checkboxes - generate dry-run selective release plans for independently versioned packages +- freeze a selective plan as a durable, resumable local release run - create annotated source release tags for selected clean, version-aligned repositories and publish each branch/tag pair atomically - generate signed catalog candidates for the selected release units @@ -49,6 +50,70 @@ tag. `source_preflight_ready` means that the plan-visible source gates pass; the non-mutating `Preview Tag + Publish` remains mandatory for remote, manifest, and immutable-tag checks. +## Durable release runs + +The **Durable Run State** card turns the current repository/version selection +into a versioned local run record. The server rebuilds the selective plan and +requires the plan to resolve exactly the requested repositories and target +versions; the browser cannot submit or replace the plan snapshot. The input and +plan are then immutable and covered by a canonical SHA-256 integrity digest. +The complete record also has a checksum so a valid-looking manual edit to its +mutable state fails closed. File permissions remain the authority boundary; +these digests detect accidental or manual corruption, not an attacker who can +replace the private record and recompute its checksums. Changing a target, +channel, or gate input requires a new run. + +Run records live below `runtime/release-runs/`. They survive console restarts, +but remain local operator state rather than a signed release artifact or the +system audit log. The directory is restricted to mode `0700` and records to +`0600`. Writes use a cross-process lock, a same-directory temporary file, +`fsync`, atomic replacement, and directory `fsync`. Symbolic-link storage paths, +overly broad record modes, malformed schemas, unknown fields, invalid state +combinations, oversized files, and digest mismatches fail closed. A bad record +is neither rewritten nor automatically quarantined. + +The default store belongs to the selected workspace: the console uses that +workspace's meta checkout when it is available, or a stable workspace-digest +namespace below this checkout's runtime directory otherwise. The same digest is +part of the immutable input snapshot. An alternate workspace therefore cannot +silently list or resume another workspace's runs. `run_state_root` remains an +explicit embedding/test override. + +Each frozen plan step has an explicit `pending`, `running`, `succeeded`, +`failed`, or `interrupted` state. Plan order remains a prerequisite: a later +step is unavailable until earlier steps have succeeded. Exact attempt and +resume/retry request identifiers are fingerprinted so repeated state commands +are idempotent. The record keeps at most 256 server-generated state events and +128 request fingerprints. Events contain only an enum event type, timestamp, +step identifier, and bounded result code—never commands, process output, +confirmation text, credentials, bearer tokens, or signing material. + +An explicit resume after a process restart converts every persisted `running` +step to `interrupted`; it never guesses whether an external effect happened. +An interrupted read-only step can be prepared for retry. An interrupted +mutating step remains unavailable until the operator has reconciled local and +remote state and explicitly records that reconciliation when preparing the +retry. A known failed attempt can likewise be prepared for retry. The UI keeps +the retry control visible and explains why it is disabled when retry is unsafe. + +The run API is covered by the same local console token middleware as every +other `/api/` route: + +- `POST /api/release-runs` rebuilds and freezes a selective plan. +- `GET /api/release-runs` lists bounded summaries; unreadable entries are + reported as unavailable instead of being silently omitted. +- `GET /api/release-runs/{run_id}` reads and verifies one exact record. +- `POST /api/release-runs/{run_id}/resume` records explicit recovery. +- `POST /api/release-runs/{run_id}/steps/{step_id}/retry` prepares a failed or + safely reconciled interrupted step for another attempt. + +This foundation tracks state only. It deliberately does not run a plan step or +infer success from a button click. Existing preview and executor controls stay +separate and continue to require `COMMIT`, `TAG`, `PUBLISH`, `APPLY`, or `PUSH` +at their existing narrow boundaries. Wiring confirmed executors to claim and +finish run steps, plus package/install verification, remains follow-up work; +until then the console must not present a run as execution evidence. + Dashboard collection is also fail closed. Unreadable Core version metadata or a malformed module contract is returned as a bounded `collection_errors` entry with a remediation, marks the dashboard blocked, and becomes a structured diff --git a/tests/test_release_run_state.py b/tests/test_release_run_state.py new file mode 100644 index 0000000..9b91c98 --- /dev/null +++ b/tests/test_release_run_state.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + + +META_ROOT = Path(__file__).resolve().parents[1] +RELEASE_ROOT = META_ROOT / "tools" / "release" +if str(RELEASE_ROOT) not in sys.path: + sys.path.insert(0, str(RELEASE_ROOT)) + +from govoplan_release.release_run import ( # noqa: E402 + MAX_EVENTS, + ReleaseRunConflict, + ReleaseRunCorrupt, + ReleaseRunStore, +) +from server.app import ( # noqa: E402 + create_app, + default_release_run_root, + release_workspace_fingerprint, +) + + +class ReleaseRunStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) / "release-runs" + self.store = ReleaseRunStore(self.root) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_create_persists_private_immutable_plan_and_ordered_steps(self) -> None: + run = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + ) + + reopened = ReleaseRunStore(self.root).get(run["run_id"]) + record_path = self.root / f"{run['run_id']}.json" + + self.assertEqual("govoplan.release-run", reopened["schema"]) + self.assertEqual(1, reopened["schema_version"]) + self.assertEqual( + {"govoplan-core": "1.2.3", "govoplan-files": "1.2.4"}, + reopened["immutable"]["input"]["repo_versions"], + ) + self.assertEqual("planned", reopened["state"]["status"]) + self.assertTrue(reopened["state"]["steps"][0]["available"]) + self.assertFalse(reopened["state"]["steps"][1]["available"]) + self.assertEqual("core:preflight", reopened["recommended_next"]["step_id"]) + self.assertEqual(0o700, os.stat(self.root).st_mode & 0o777) + self.assertEqual(0o600, os.stat(record_path).st_mode & 0o777) + self.assertNotIn("processed_requests", reopened["state"]) + self.assertNotIn("attempt_fingerprint", reopened["state"]["steps"][0]) + + def test_attempt_transitions_are_idempotent_and_strictly_ordered(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + + with self.assertRaisesRegex(ReleaseRunConflict, "Complete core:preflight"): + self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001") + + started = self.store.start_step( + run_id, "core:preflight", attempt_id="attempt-preflight-0001" + ) + duplicate_start = self.store.start_step( + run_id, "core:preflight", attempt_id="attempt-preflight-0001" + ) + self.assertEqual(1, started["state"]["steps"][0]["attempt_count"]) + self.assertEqual( + len(started["state"]["events"]), + len(duplicate_start["state"]["events"]), + ) + + finished = self.store.finish_step( + run_id, + "core:preflight", + attempt_id="attempt-preflight-0001", + succeeded=True, + result_code="preflight_valid", + ) + duplicate_finish = self.store.finish_step( + run_id, + "core:preflight", + attempt_id="attempt-preflight-0001", + succeeded=True, + result_code="preflight_valid", + ) + self.assertEqual("succeeded", finished["state"]["steps"][0]["state"]) + self.assertEqual( + len(finished["state"]["events"]), + len(duplicate_finish["state"]["events"]), + ) + self.assertTrue(finished["state"]["steps"][1]["available"]) + + def test_reopen_and_resume_preserve_ambiguous_mutating_effect(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=mutating_first_plan() + )["run_id"] + self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001") + + reopened_store = ReleaseRunStore(self.root) + before_resume = reopened_store.get(run_id) + self.assertEqual("running", before_resume["state"]["steps"][0]["state"]) + + resumed = reopened_store.resume(run_id, request_id="resume-request-0001") + duplicate = reopened_store.resume(run_id, request_id="resume-request-0001") + self.assertEqual("interrupted", resumed["state"]["steps"][0]["state"]) + self.assertEqual("reconcile_step", resumed["recommended_next"]["id"]) + self.assertFalse(resumed["recommended_next"]["available"]) + self.assertEqual( + len(resumed["state"]["events"]), len(duplicate["state"]["events"]) + ) + + with self.assertRaisesRegex(ReleaseRunConflict, "must be reconciled"): + reopened_store.retry_step( + run_id, + "core:tag", + request_id="retry-request-0001", + ) + retried = reopened_store.retry_step( + run_id, + "core:tag", + request_id="retry-request-0002", + reconciled=True, + ) + duplicate_retry = reopened_store.retry_step( + run_id, + "core:tag", + request_id="retry-request-0002", + reconciled=True, + ) + self.assertEqual("pending", retried["state"]["steps"][0]["state"]) + self.assertEqual( + len(retried["state"]["events"]), + len(duplicate_retry["state"]["events"]), + ) + + def test_tampered_record_fails_closed_and_is_not_rewritten(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + path = self.root / f"{run_id}.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["immutable"]["input"]["channel"] = "testing" + path.write_text(json.dumps(payload), encoding="utf-8") + tampered = path.read_bytes() + + with self.assertRaises(ReleaseRunCorrupt): + self.store.get(run_id) + + self.assertEqual(tampered, path.read_bytes()) + self.assertEqual("unavailable", self.store.list()[0]["status"]) + + def test_broad_record_mode_and_symlinked_root_fail_closed(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + path = self.root / f"{run_id}.json" + path.chmod(0o640) + with self.assertRaisesRegex(ReleaseRunCorrupt, "broader than 0600"): + self.store.get(run_id) + self.assertEqual(0o640, os.stat(path).st_mode & 0o777) + + target = Path(self.temporary.name) / "target" + target.mkdir() + linked = Path(self.temporary.name) / "linked" + linked.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(ReleaseRunCorrupt, "symbolic-link"): + ReleaseRunStore(linked).list() + + def test_state_specific_fields_and_boolean_counts_fail_closed(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + path = self.root / f"{run_id}.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["state"]["steps"][0]["attempt_count"] = True + payload["state"]["steps"][0]["started_at"] = "2026-07-22T00:00:00Z" + path.write_text(json.dumps(payload), encoding="utf-8") + path.chmod(0o600) + + with self.assertRaises(ReleaseRunCorrupt): + self.store.get(run_id) + + def test_valid_looking_mutable_state_tampering_fails_record_checksum(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=mutating_first_plan() + )["run_id"] + self.store.start_step(run_id, "core:tag", attempt_id="checksum-attempt-0001") + self.store.finish_step( + run_id, + "core:tag", + attempt_id="checksum-attempt-0001", + succeeded=True, + result_code="tag_created", + ) + path = self.root / f"{run_id}.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["state"]["steps"][0]["result_code"] = "tag_published" + path.write_text(json.dumps(payload), encoding="utf-8") + path.chmod(0o600) + tampered = path.read_bytes() + + with self.assertRaises(ReleaseRunCorrupt): + self.store.get(run_id) + + self.assertEqual(tampered, path.read_bytes()) + + def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + barrier = threading.Barrier(2) + + def claim(attempt_id: str) -> str: + contender = ReleaseRunStore(self.root) + barrier.wait(timeout=5) + try: + contender.start_step(run_id, "core:preflight", attempt_id=attempt_id) + except ReleaseRunConflict: + return "conflict" + return "started" + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list( + executor.map( + claim, ("concurrent-attempt-0001", "concurrent-attempt-0002") + ) + ) + + reopened = self.store.get(run_id) + self.assertEqual(["conflict", "started"], sorted(results)) + self.assertEqual(1, reopened["state"]["steps"][0]["attempt_count"]) + self.assertEqual( + 1, + sum( + event["type"] == "step_started" for event in reopened["state"]["events"] + ), + ) + + def test_event_and_request_history_are_bounded_and_code_only(self) -> None: + run_id = self.store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + for index in range(90): + attempt_id = f"bounded-attempt-{index:04d}" + self.store.start_step(run_id, "core:preflight", attempt_id=attempt_id) + self.store.finish_step( + run_id, + "core:preflight", + attempt_id=attempt_id, + succeeded=False, + result_code="preflight_failed", + ) + self.store.retry_step( + run_id, + "core:preflight", + request_id=f"bounded-retry-{index:04d}", + ) + + view = self.store.get(run_id) + raw = json.loads((self.root / f"{run_id}.json").read_text(encoding="utf-8")) + self.assertEqual(MAX_EVENTS, len(view["state"]["events"])) + self.assertLessEqual(len(raw["state"]["processed_requests"]), 128) + self.assertNotIn("bounded-retry-0089", json.dumps(raw)) + for event in view["state"]["events"]: + self.assertLessEqual( + set(event), {"sequence", "at", "type", "step_id", "result_code"} + ) + + +class ReleaseRunApiTests(unittest.TestCase): + def test_token_guarded_create_list_read_resume_and_retry(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / "runs" + app = create_app( + workspace_root=Path(temp_dir), + run_state_root=root, + token="release-token", + ) + headers = {"X-Release-Console-Token": "release-token"} + with ( + patch("server.app.build_dashboard", return_value=object()), + patch( + "server.app.build_selective_release_plan", + return_value=release_plan(), + ), + TestClient(app) as client, + ): + unauthorized = client.get("/api/release-runs") + created_response = client.post( + "/api/release-runs", + headers=headers, + json=run_input(), + ) + self.assertEqual(200, created_response.status_code) + created = created_response.json() + run_id = created["run_id"] + listed = client.get("/api/release-runs", headers=headers).json() + loaded = client.get( + f"/api/release-runs/{run_id}", headers=headers + ).json() + + app.state.release_runs.start_step( + run_id, + "core:preflight", + attempt_id="api-attempt-0001", + ) + resumed = client.post( + f"/api/release-runs/{run_id}/resume", + headers=headers, + json={"request_id": "api-resume-0001"}, + ) + retried = client.post( + f"/api/release-runs/{run_id}/steps/core:preflight/retry", + headers=headers, + json={"request_id": "api-retry-0001"}, + ) + + self.assertEqual(401, unauthorized.status_code) + self.assertEqual(run_id, listed["runs"][0]["run_id"]) + self.assertEqual(run_id, loaded["run_id"]) + self.assertEqual( + "interrupted", resumed.json()["state"]["steps"][0]["state"] + ) + self.assertEqual(200, retried.status_code) + self.assertEqual("pending", retried.json()["state"]["steps"][0]["state"]) + + def test_create_rejects_unknown_or_unresolved_repository(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / "runs" + app = create_app( + workspace_root=Path(temp_dir), run_state_root=root, token="token" + ) + with ( + patch("server.app.build_dashboard", return_value=object()), + patch( + "server.app.build_selective_release_plan", + return_value=release_plan(), + ), + TestClient(app) as client, + ): + response = client.post( + "/api/release-runs", + headers={"X-Release-Console-Token": "token"}, + json={ + "channel": "stable", + "repo_versions": {"govoplan-unknown": "1.0.0"}, + "public_catalog": False, + }, + ) + + self.assertEqual(409, response.status_code) + self.assertIn("did not resolve exactly", response.json()["detail"]) + self.assertEqual([], list(root.glob("*.json"))) + + def test_api_rejects_tampered_record_without_rewriting_it(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / "runs" + store = ReleaseRunStore(root) + run_id = store.create( + input_snapshot=run_input(), plan_snapshot=release_plan() + )["run_id"] + path = root / f"{run_id}.json" + path.write_text("{}", encoding="utf-8") + malformed = path.read_bytes() + app = create_app( + workspace_root=Path(temp_dir), run_state_root=root, token="token" + ) + with TestClient(app) as client: + response = client.get( + f"/api/release-runs/{run_id}", + headers={"X-Release-Console-Token": "token"}, + ) + + self.assertEqual(409, response.status_code) + self.assertEqual(malformed, path.read_bytes()) + + def test_default_storage_and_immutable_input_are_workspace_bound(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + parent = Path(temp_dir) + first = parent / "first" + second = parent / "second" + first.mkdir() + second.mkdir() + first_root = default_release_run_root(first) + second_root = default_release_run_root(second) + self.assertNotEqual(first_root, second_root) + self.assertIn(release_workspace_fingerprint(first)[:16], first_root.name) + + local_workspace = parent / "local" + local_meta = local_workspace / "govoplan" + local_meta.mkdir(parents=True) + (local_meta / "repositories.json").write_text("{}", encoding="utf-8") + self.assertEqual( + local_meta / "runtime" / "release-runs", + default_release_run_root(local_workspace), + ) + + explicit_root = parent / "explicit" + app = create_app( + workspace_root=first, + run_state_root=explicit_root, + token="token", + ) + with ( + patch("server.app.build_dashboard", return_value=object()), + patch( + "server.app.build_selective_release_plan", + return_value=release_plan(), + ), + TestClient(app) as client, + ): + response = client.post( + "/api/release-runs", + headers={"X-Release-Console-Token": "token"}, + json=run_input(), + ) + self.assertEqual(200, response.status_code) + self.assertEqual( + release_workspace_fingerprint(first), + response.json()["immutable"]["input"]["workspace_fingerprint"], + ) + self.assertEqual(explicit_root.resolve(), app.state.release_runs.root) + + def test_ui_and_runbook_state_tracking_boundary_are_explicit(self) -> None: + webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8") + runbook = (META_ROOT / "docs" / "RELEASE_CONSOLE.md").read_text( + encoding="utf-8" + ) + + self.assertIn("Durable Run State", webui) + self.assertIn("State tracking only.", webui) + self.assertIn("Create Run from Selection", webui) + self.assertIn("Prepare Retry", webui) + self.assertIn("step.retry_available", webui) + self.assertIn("This foundation tracks state only.", runbook) + self.assertIn("mutating step remains unavailable", runbook) + self.assertIn("same-directory temporary file", runbook) + + +def run_input() -> dict[str, object]: + return { + "channel": "stable", + "repo_versions": { + "govoplan-files": "1.2.4", + "govoplan-core": "1.2.3", + }, + "online": False, + "remote_tags": False, + "public_catalog": False, + "include_migrations": True, + "workspace_fingerprint": "a" * 64, + } + + +def release_plan() -> dict[str, object]: + return { + "generated_at": "2026-07-22T00:00:00Z", + "target_channel": "stable", + "status": "attention", + "units": [ + {"repo": "govoplan-core", "target_version": "1.2.3"}, + {"repo": "govoplan-files", "target_version": "1.2.4"}, + ], + "compatibility": [], + "gate_findings": [], + "recommended_action": { + "id": "preview_source_release", + "title": "Preview source release tags", + "detail": "Plan-visible gates pass.", + "remediation": "Preview before mutation.", + }, + "source_preflight_ready": True, + "dry_run_steps": [ + { + "id": "core:preflight", + "title": "Inspect Core", + "detail": "Run a read-only preflight.", + "command": "git status --short --branch", + "cwd": "/workspace/govoplan-core", + "mutating": False, + "repo": "govoplan-core", + "status": "planned", + }, + { + "id": "core:tag", + "title": "Create Core tag", + "detail": "Create an annotated tag after confirmation.", + "command": "git tag -a v1.2.3", + "cwd": "/workspace/govoplan-core", + "mutating": True, + "repo": "govoplan-core", + "status": "planned", + }, + ], + "notes": [], + } + + +def mutating_first_plan() -> dict[str, object]: + plan = release_plan() + plan["dry_run_steps"] = [plan["dry_run_steps"][1]] # type: ignore[index] + return plan + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/govoplan_release/release_run.py b/tools/release/govoplan_release/release_run.py new file mode 100644 index 0000000..434d7d4 --- /dev/null +++ b/tools/release/govoplan_release/release_run.py @@ -0,0 +1,1154 @@ +"""Durable state for guided, explicitly executed release runs. + +The run store records an immutable selective-plan snapshot and a small mutable +state machine. It does not execute release commands. Existing release +executors remain responsible for their narrow confirmations and will be wired +to this state in a later slice. +""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import tempfile +import threading +from typing import Any +import uuid + +from filelock import FileLock + + +SCHEMA = "govoplan.release-run" +SCHEMA_VERSION = 1 +MAX_RECORD_BYTES = 2 * 1024 * 1024 +MAX_PLAN_STEPS = 512 +MAX_REPOSITORY_VERSIONS = 128 +MAX_EVENTS = 256 +MAX_REQUESTS = 128 +MAX_LIST_LIMIT = 100 +RUN_ID_PATTERN = re.compile(r"^rr-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$") +STEP_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,159}$") +REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{7,127}$") +CHANNEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +WORKSPACE_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$") +VERSION_PATTERN = re.compile( + r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z][0-9A-Za-z.-]*)?$" +) +STEP_STATES = {"pending", "running", "succeeded", "failed", "interrupted"} +PLAN_STATUSES = {"ready", "attention", "blocked"} +PLANNED_STEP_STATUSES = {"planned", "needs-executor", "needs-version-metadata"} +EVENT_TYPES = { + "run_created", + "run_resumed", + "step_started", + "step_succeeded", + "step_failed", + "step_interrupted", + "step_retry_requested", +} + + +class ReleaseRunError(RuntimeError): + """Base error for durable release run state.""" + + +class ReleaseRunNotFound(ReleaseRunError): + """Raised when a run does not exist.""" + + +class ReleaseRunConflict(ReleaseRunError): + """Raised when a state transition is unsafe or no longer applicable.""" + + +class ReleaseRunCorrupt(ReleaseRunError): + """Raised when a persisted record fails its integrity or schema checks.""" + + +class ReleaseRunStore: + """Atomic, bounded local JSON store for release-run state.""" + + def __init__(self, root: Path) -> None: + # Keep the configured path spelling so a symlink root (or symlinked + # parent) remains detectable instead of being erased by resolve(). + self.root = Path(os.path.abspath(os.fspath(root.expanduser()))) + self._thread_lock = threading.RLock() + self._lock_path = self.root / ".release-runs.lock" + + def create( + self, + *, + input_snapshot: dict[str, Any], + plan_snapshot: dict[str, Any], + ) -> dict[str, Any]: + immutable_input = _validated_input(input_snapshot) + immutable_plan = _validated_plan(plan_snapshot) + now = _timestamp() + run_id = ( + f"rr-{now.replace('-', '').replace(':', '')[:15]}Z-{uuid.uuid4().hex[:12]}" + ) + immutable = { + "input": immutable_input, + "plan": immutable_plan, + } + immutable["digest"] = _immutable_digest(immutable) + steps = [ + { + "id": item["id"], + "state": "pending", + "attempt_count": 0, + "attempt_fingerprint": None, + "started_at": None, + "finished_at": None, + "result_code": None, + } + for item in immutable_plan["dry_run_steps"] + ] + record: dict[str, Any] = { + "schema": SCHEMA, + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "created_at": now, + "updated_at": now, + "immutable": immutable, + "state": { + "status": "planned", + "steps": steps, + "events": [], + "next_event_sequence": 1, + "processed_requests": [], + }, + } + _append_event(record, event_type="run_created") + _refresh_status(record) + record["record_digest"] = _record_digest(record) + with self._locked(): + self._ensure_root() + self._write_new(record) + return _view(record) + + def get(self, run_id: str) -> dict[str, Any]: + with self._locked(): + record = self._read(run_id) + return _view(record) + + def list(self, *, limit: int = 20) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), MAX_LIST_LIMIT) + with self._locked(): + self._ensure_root() + paths = sorted( + ( + path + for path in self.root.iterdir() + if path.name.endswith(".json") + and RUN_ID_PATTERN.fullmatch(path.stem) + ), + key=lambda item: item.name, + reverse=True, + )[:limit] + summaries: list[dict[str, Any]] = [] + for path in paths: + try: + record = self._read(path.stem) + except (ReleaseRunCorrupt, ReleaseRunNotFound): + summaries.append( + { + "run_id": path.stem, + "status": "unavailable", + "error": "Run record is unavailable or failed integrity validation.", + } + ) + continue + view = _view(record) + immutable_input = view["immutable"]["input"] + summaries.append( + { + "run_id": view["run_id"], + "created_at": view["created_at"], + "updated_at": view["updated_at"], + "status": view["state"]["status"], + "channel": immutable_input["channel"], + "repo_versions": immutable_input["repo_versions"], + "recommended_next": view["recommended_next"], + } + ) + return summaries + + def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]: + request_fingerprint = _request_fingerprint(request_id) + with self._locked(): + record = self._read(run_id) + if _request_seen(record, request_fingerprint, "resume", None): + return _view(record) + for step in record["state"]["steps"]: + if step["state"] != "running": + continue + step["state"] = "interrupted" + step["finished_at"] = _timestamp() + step["result_code"] = "process_interrupted" + _append_event( + record, + event_type="step_interrupted", + step_id=step["id"], + result_code="process_interrupted", + ) + _remember_request(record, request_fingerprint, "resume", None) + _append_event(record, event_type="run_resumed") + self._persist_update(record) + return _view(record) + + def retry_step( + self, + run_id: str, + step_id: str, + *, + request_id: str, + reconciled: bool = False, + ) -> dict[str, Any]: + _validate_step_id(step_id) + request_fingerprint = _request_fingerprint(request_id) + with self._locked(): + record = self._read(run_id) + if _request_seen(record, request_fingerprint, "retry", step_id): + return _view(record) + step, plan_step = _step_pair(record, step_id) + if step["state"] not in {"failed", "interrupted"}: + raise ReleaseRunConflict( + "Only a failed or interrupted release step can be prepared for retry." + ) + if ( + step["state"] == "interrupted" + and bool(plan_step.get("mutating")) + and not reconciled + ): + raise ReleaseRunConflict( + "An interrupted mutating step must be reconciled before retry." + ) + step.update( + { + "state": "pending", + "attempt_fingerprint": None, + "started_at": None, + "finished_at": None, + "result_code": None, + } + ) + _remember_request(record, request_fingerprint, "retry", step_id) + _append_event( + record, + event_type="step_retry_requested", + step_id=step_id, + result_code="reconciled" if reconciled else "known_failure", + ) + self._persist_update(record) + return _view(record) + + def start_step( + self, + run_id: str, + step_id: str, + *, + attempt_id: str, + ) -> dict[str, Any]: + """Claim one step for a future confirmed executor. + + This is deliberately not an HTTP endpoint in the foundation slice. + """ + + _validate_step_id(step_id) + attempt_fingerprint = _request_fingerprint(attempt_id) + with self._locked(): + record = self._read(run_id) + step, _plan_step = _step_pair(record, step_id) + if step["attempt_fingerprint"] == attempt_fingerprint and step["state"] in { + "running", + "succeeded", + }: + return _view(record) + available, reason = _step_available(record, step_id) + if not available: + raise ReleaseRunConflict(reason) + step.update( + { + "state": "running", + "attempt_count": step["attempt_count"] + 1, + "attempt_fingerprint": attempt_fingerprint, + "started_at": _timestamp(), + "finished_at": None, + "result_code": None, + } + ) + _append_event(record, event_type="step_started", step_id=step_id) + self._persist_update(record) + return _view(record) + + def finish_step( + self, + run_id: str, + step_id: str, + *, + attempt_id: str, + succeeded: bool, + result_code: str, + ) -> dict[str, Any]: + """Record a future executor outcome without persisting its output.""" + + _validate_step_id(step_id) + attempt_fingerprint = _request_fingerprint(attempt_id) + result_code = _safe_code(result_code) + target_state = "succeeded" if succeeded else "failed" + with self._locked(): + record = self._read(run_id) + step, _plan_step = _step_pair(record, step_id) + if ( + step["attempt_fingerprint"] == attempt_fingerprint + and step["state"] == target_state + and step["result_code"] == result_code + ): + return _view(record) + if ( + step["state"] != "running" + or step["attempt_fingerprint"] != attempt_fingerprint + ): + raise ReleaseRunConflict( + "The release step is not running with this attempt identifier." + ) + step.update( + { + "state": target_state, + "finished_at": _timestamp(), + "result_code": result_code, + } + ) + _append_event( + record, + event_type="step_succeeded" if succeeded else "step_failed", + step_id=step_id, + result_code=result_code, + ) + self._persist_update(record) + return _view(record) + + def _persist_update(self, record: dict[str, Any]) -> None: + record["updated_at"] = _timestamp() + _refresh_status(record) + record["record_digest"] = _record_digest(record) + _validate_record(record) + self._atomic_write(self._path(record["run_id"]), record) + + def _write_new(self, record: dict[str, Any]) -> None: + _validate_record(record) + path = self._path(record["run_id"]) + if path.exists() or path.is_symlink(): + raise ReleaseRunConflict("The generated release run already exists.") + self._atomic_write(path, record) + + def _read(self, run_id: str) -> dict[str, Any]: + path = self._path(run_id) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except FileNotFoundError as exc: + raise ReleaseRunNotFound("Release run was not found.") from exc + except OSError as exc: + raise ReleaseRunCorrupt( + "Release run record could not be opened safely." + ) from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ReleaseRunCorrupt("Release run record is not a regular file.") + if metadata.st_mode & 0o077: + raise ReleaseRunCorrupt( + "Release run record permissions are broader than 0600." + ) + if metadata.st_size > MAX_RECORD_BYTES: + raise ReleaseRunCorrupt("Release run record exceeds its size limit.") + with os.fdopen(descriptor, "rb") as handle: + descriptor = -1 + encoded = handle.read(MAX_RECORD_BYTES + 1) + if len(encoded) > MAX_RECORD_BYTES: + raise ReleaseRunCorrupt("Release run record exceeds its size limit.") + payload = json.loads(encoded.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ReleaseRunCorrupt("Release run record is unreadable.") from exc + finally: + if descriptor >= 0: + os.close(descriptor) + _validate_record(payload, expected_run_id=run_id) + return payload + + def _path(self, run_id: str) -> Path: + if not RUN_ID_PATTERN.fullmatch(run_id): + raise ReleaseRunNotFound("Release run was not found.") + return self.root / f"{run_id}.json" + + def _ensure_root(self) -> None: + _reject_symlink_components(self.root) + self.root.mkdir(mode=0o700, parents=True, exist_ok=True) + _reject_symlink_components(self.root) + if self.root.is_symlink() or not self.root.is_dir(): + raise ReleaseRunCorrupt("Release run storage root is not a directory.") + try: + self.root.chmod(0o700) + except OSError as exc: + raise ReleaseRunCorrupt( + "Release run storage permissions could not be secured." + ) from exc + if self.root.stat().st_mode & 0o077: + raise ReleaseRunCorrupt( + "Release run storage permissions are broader than 0700." + ) + + def _atomic_write(self, path: Path, record: dict[str, Any]) -> None: + encoded = _canonical_json(record) + b"\n" + if len(encoded) > MAX_RECORD_BYTES: + raise ReleaseRunConflict("Release run record exceeds its size limit.") + descriptor = -1 + temporary_path: str | None = None + try: + descriptor, temporary_path = tempfile.mkstemp( + dir=self.root, prefix=".release-run-", suffix=".tmp" + ) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + descriptor = -1 + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + temporary_path = None + try: + path.chmod(0o600) + except OSError as exc: + raise ReleaseRunCorrupt( + "Release run record permissions could not be secured." + ) from exc + directory_flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + directory_flags |= os.O_DIRECTORY + directory_descriptor = os.open(self.root, directory_flags) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + finally: + if descriptor >= 0: + os.close(descriptor) + if temporary_path is not None: + try: + os.unlink(temporary_path) + except FileNotFoundError: + pass + + def _locked(self): # type: ignore[no-untyped-def] + self._ensure_root() + return _CombinedLock(self._thread_lock, FileLock(str(self._lock_path))) + + +class _CombinedLock: + def __init__(self, thread_lock: threading.RLock, file_lock: FileLock) -> None: + self.thread_lock = thread_lock + self.file_lock = file_lock + + def __enter__(self) -> None: + self.thread_lock.acquire() + try: + self.file_lock.acquire(timeout=10) + except BaseException: + self.thread_lock.release() + raise + + def __exit__(self, *exc_info: object) -> None: + try: + self.file_lock.release() + finally: + self.thread_lock.release() + + +def _validated_input(value: dict[str, Any]) -> dict[str, Any]: + if not isinstance(value, dict): + raise ReleaseRunConflict("Release run input must be an object.") + allowed = { + "channel", + "repo_versions", + "online", + "remote_tags", + "public_catalog", + "include_migrations", + "workspace_fingerprint", + } + if set(value) != allowed: + raise ReleaseRunConflict("Release run input fields are invalid.") + channel = value.get("channel") + repo_versions = value.get("repo_versions") + if not isinstance(channel, str) or not CHANNEL_PATTERN.fullmatch(channel): + raise ReleaseRunConflict("Release channel is invalid.") + if ( + not isinstance(repo_versions, dict) + or not repo_versions + or len(repo_versions) > MAX_REPOSITORY_VERSIONS + ): + raise ReleaseRunConflict("At least one repository version is required.") + normalized_versions: dict[str, str] = {} + for repo, version in sorted(repo_versions.items()): + if not isinstance(repo, str) or not REPOSITORY_PATTERN.fullmatch(repo): + raise ReleaseRunConflict("Release repository name is invalid.") + if not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version): + raise ReleaseRunConflict("Release target version is invalid.") + normalized_versions[repo] = version + result: dict[str, Any] = { + "channel": channel, + "repo_versions": normalized_versions, + } + for field in ("online", "remote_tags", "public_catalog", "include_migrations"): + if not isinstance(value.get(field), bool): + raise ReleaseRunConflict(f"Release run input {field} must be boolean.") + result[field] = value[field] + workspace_fingerprint = value.get("workspace_fingerprint") + if not isinstance( + workspace_fingerprint, str + ) or not WORKSPACE_FINGERPRINT_PATTERN.fullmatch(workspace_fingerprint): + raise ReleaseRunConflict("Release workspace fingerprint is invalid.") + result["workspace_fingerprint"] = workspace_fingerprint + return result + + +def _validated_plan(value: dict[str, Any]) -> dict[str, Any]: + if not isinstance(value, dict): + raise ReleaseRunConflict("Release plan snapshot must be an object.") + copied = deepcopy(value) + if set(copied) != { + "generated_at", + "target_channel", + "status", + "units", + "compatibility", + "dry_run_steps", + "notes", + "gate_findings", + "recommended_action", + "source_preflight_ready", + }: + raise ReleaseRunConflict("Release plan snapshot fields are invalid.") + if not _is_timestamp(copied.get("generated_at")): + raise ReleaseRunConflict("Release plan timestamp is invalid.") + if copied.get("status") not in PLAN_STATUSES: + raise ReleaseRunConflict("Release plan status is invalid.") + if not isinstance( + copied.get("target_channel"), str + ) or not CHANNEL_PATTERN.fullmatch(copied["target_channel"]): + raise ReleaseRunConflict("Release plan channel is invalid.") + if not isinstance(copied.get("units"), list): + raise ReleaseRunConflict("Release plan units are invalid.") + if not isinstance(copied.get("compatibility"), list) or not isinstance( + copied.get("gate_findings"), list + ): + raise ReleaseRunConflict("Release plan findings are invalid.") + if not isinstance(copied.get("notes"), list) or not all( + isinstance(note, str) for note in copied["notes"] + ): + raise ReleaseRunConflict("Release plan notes are invalid.") + if not isinstance(copied.get("source_preflight_ready"), bool): + raise ReleaseRunConflict("Release plan preflight state is invalid.") + recommendation = copied.get("recommended_action") + if recommendation is not None and not isinstance(recommendation, dict): + raise ReleaseRunConflict("Release plan recommendation is invalid.") + steps = copied.get("dry_run_steps") + if not isinstance(steps, list) or len(steps) > MAX_PLAN_STEPS: + raise ReleaseRunConflict("Release plan step collection is invalid.") + seen: set[str] = set() + for step in steps: + if not isinstance(step, dict) or set(step) != { + "id", + "title", + "detail", + "command", + "cwd", + "mutating", + "repo", + "status", + }: + raise ReleaseRunConflict("Release plan step is invalid.") + step_id = step.get("id") + _validate_step_id(step_id) + if step_id in seen: + raise ReleaseRunConflict("Release plan step identifiers must be unique.") + seen.add(step_id) + if not isinstance(step.get("mutating", False), bool): + raise ReleaseRunConflict("Release plan step mutation flag is invalid.") + if step.get("status") not in PLANNED_STEP_STATUSES: + raise ReleaseRunConflict("Release plan step status is invalid.") + for field, maximum in (("title", 500), ("detail", 2_000)): + if not isinstance(step.get(field), str) or len(step[field]) > maximum: + raise ReleaseRunConflict("Release plan step text is invalid.") + for field in ("command", "cwd", "repo"): + if step.get(field) is not None and ( + not isinstance(step[field], str) or len(step[field]) > 8_192 + ): + raise ReleaseRunConflict("Release plan step metadata is invalid.") + encoded = _canonical_json(copied) + if len(encoded) > MAX_RECORD_BYTES: + raise ReleaseRunConflict("Release plan snapshot exceeds its size limit.") + return copied + + +def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None: + try: + if not isinstance(value, dict): + raise ValueError + if set(value) != { + "schema", + "schema_version", + "run_id", + "created_at", + "updated_at", + "immutable", + "state", + "record_digest", + }: + raise ValueError + if ( + value.get("schema") != SCHEMA + or type(value.get("schema_version")) is not int + or value["schema_version"] != SCHEMA_VERSION + ): + raise ValueError + run_id = value.get("run_id") + if not isinstance(run_id, str) or not RUN_ID_PATTERN.fullmatch(run_id): + raise ValueError + if expected_run_id is not None and run_id != expected_run_id: + raise ValueError + for key in ("created_at", "updated_at"): + if not _is_timestamp(value.get(key)): + raise ValueError + immutable = value.get("immutable") + if not isinstance(immutable, dict) or set(immutable) != { + "input", + "plan", + "digest", + }: + raise ValueError + validated_input = _validated_input(immutable["input"]) + validated_plan = _validated_plan(immutable["plan"]) + if immutable["input"] != validated_input or immutable["plan"] != validated_plan: + raise ValueError + if immutable.get("digest") != _immutable_digest(immutable): + raise ValueError + if validated_plan["target_channel"] != validated_input["channel"]: + raise ValueError + planned_units = validated_plan["units"] + planned_versions = { + unit.get("repo"): unit.get("target_version") + for unit in planned_units + if isinstance(unit, dict) + and isinstance(unit.get("repo"), str) + and isinstance(unit.get("target_version"), str) + } + if planned_versions != validated_input["repo_versions"] or len( + planned_versions + ) != len(planned_units): + raise ValueError + state = value.get("state") + if not isinstance(state, dict) or set(state) != { + "status", + "steps", + "events", + "next_event_sequence", + "processed_requests", + }: + raise ValueError + steps = state.get("steps") + plan_steps = validated_plan["dry_run_steps"] + plan_step_ids = {plan_step["id"] for plan_step in plan_steps} + if not isinstance(steps, list) or len(steps) != len(plan_steps): + raise ValueError + for step, plan_step in zip(steps, plan_steps, strict=True): + if ( + not isinstance(step, dict) + or set(step) + != { + "id", + "state", + "attempt_count", + "attempt_fingerprint", + "started_at", + "finished_at", + "result_code", + } + or step.get("id") != plan_step["id"] + ): + raise ValueError + if step.get("state") not in STEP_STATES: + raise ValueError + if type(step.get("attempt_count")) is not int or step["attempt_count"] < 0: + raise ValueError + fingerprint = step.get("attempt_fingerprint") + if fingerprint is not None and not _is_sha256(fingerprint): + raise ValueError + for timestamp in (step.get("started_at"), step.get("finished_at")): + if timestamp is not None and ( + not isinstance(timestamp, str) or len(timestamp) > 40 + ): + raise ValueError + result_code = step.get("result_code") + if result_code is not None: + _safe_code(result_code) + _validate_step_state_fields(step) + events = state.get("events") + if not isinstance(events, list) or len(events) > MAX_EVENTS: + raise ValueError + previous_sequence = 0 + for event in events: + if not isinstance(event, dict) or event.get("type") not in EVENT_TYPES: + raise ValueError + expected_event_keys = {"sequence", "at", "type"} + if event.get("step_id") is not None: + expected_event_keys.add("step_id") + if event.get("result_code") is not None: + expected_event_keys.add("result_code") + if set(event) != expected_event_keys: + raise ValueError + sequence = event.get("sequence") + if type(sequence) is not int or sequence <= previous_sequence: + raise ValueError + previous_sequence = sequence + if not _is_timestamp(event.get("at")): + raise ValueError + if event.get("step_id") is not None: + _validate_step_id(event["step_id"]) + if event["step_id"] not in plan_step_ids: + raise ValueError + if event.get("result_code") is not None: + _safe_code(event["result_code"]) + _validate_event_fields(event) + next_sequence = state.get("next_event_sequence") + if type(next_sequence) is not int or next_sequence <= previous_sequence: + raise ValueError + requests = state.get("processed_requests") + if not isinstance(requests, list) or len(requests) > MAX_REQUESTS: + raise ValueError + for request in requests: + if ( + not isinstance(request, dict) + or set(request) != {"fingerprint", "operation", "step_id"} + or not _is_sha256(request.get("fingerprint")) + or request.get("operation") not in {"resume", "retry"} + ): + raise ValueError + if request.get("step_id") is not None: + _validate_step_id(request["step_id"]) + if request["step_id"] not in plan_step_ids: + raise ValueError + if (request["operation"] == "resume") != (request["step_id"] is None): + raise ValueError + if len({request["fingerprint"] for request in requests}) != len(requests): + raise ValueError + if state.get("status") != _status(value): + raise ValueError + if value.get("record_digest") != _record_digest(value): + raise ValueError + except ( + KeyError, + StopIteration, + TypeError, + ValueError, + ReleaseRunConflict, + ) as exc: + raise ReleaseRunCorrupt( + "Release run record failed schema or integrity validation." + ) from exc + + +def _view(record: dict[str, Any]) -> dict[str, Any]: + view = deepcopy(record) + plan_steps = view["immutable"]["plan"]["dry_run_steps"] + mutable_steps = view["state"]["steps"] + enriched_steps: list[dict[str, Any]] = [] + for index, (step, plan_step) in enumerate( + zip(mutable_steps, plan_steps, strict=True) + ): + available, disabled_reason = _step_available(record, step["id"]) + retry_available = step["state"] == "failed" or ( + step["state"] == "interrupted" and not bool(plan_step.get("mutating")) + ) + enriched_steps.append( + { + **plan_step, + **{ + key: value + for key, value in step.items() + if key != "attempt_fingerprint" + }, + "order": index + 1, + "available": available, + "retry_available": retry_available, + "disabled_reason": disabled_reason, + } + ) + view["state"]["steps"] = enriched_steps + view["state"].pop("processed_requests", None) + view["recommended_next"] = _recommended_next(record) + return view + + +def _step_available(record: dict[str, Any], step_id: str) -> tuple[bool, str]: + step, plan_step = _step_pair(record, step_id) + if record["immutable"]["plan"].get("status") == "blocked": + return False, "The immutable plan contains release-gate blockers." + if step["state"] != "pending": + descriptions = { + "running": "This step is already running.", + "succeeded": "This step is complete.", + "failed": "Request an explicit retry before running this step again.", + "interrupted": "Reconcile the interrupted attempt before retry.", + } + return False, descriptions.get(step["state"], "This step is unavailable.") + planned_status = plan_step.get("status", "planned") + if planned_status != "planned": + return ( + False, + f"The plan marks this step {planned_status}; no executor is available.", + ) + steps = record["state"]["steps"] + index = next(index for index, item in enumerate(steps) if item["id"] == step_id) + incomplete = next( + (item for item in steps[:index] if item["state"] != "succeeded"), None + ) + if incomplete is not None: + return False, f"Complete {incomplete['id']} first." + return True, "" + + +def _recommended_next(record: dict[str, Any]) -> dict[str, Any]: + plan = record["immutable"]["plan"] + if plan.get("status") == "blocked": + recommendation = plan.get("recommended_action") + if isinstance(recommendation, dict): + return { + "id": recommendation.get("id") or "resolve_plan_blocker", + "step_id": None, + "title": recommendation.get("title") or "Resolve release gate", + "detail": recommendation.get("detail") or "The frozen plan is blocked.", + "remediation": recommendation.get("remediation") + or "Resolve the blocker and create a new run from a fresh plan.", + "available": False, + } + return { + "id": "resolve_plan_blocker", + "step_id": None, + "title": "Resolve release gate", + "detail": "The frozen plan is blocked.", + "remediation": "Resolve the blocker and create a new run from a fresh plan.", + "available": False, + } + for step, plan_step in zip( + record["state"]["steps"], plan["dry_run_steps"], strict=True + ): + if step["state"] == "running": + return _step_recommendation( + "wait_for_step", + step, + plan_step, + "Wait for the active executor or resume the run after an interruption.", + False, + ) + if step["state"] == "failed": + return _step_recommendation( + "retry_step", + step, + plan_step, + "Review the bounded result code, then explicitly prepare this known failed step for retry.", + True, + ) + if step["state"] == "interrupted": + if bool(plan_step.get("mutating")): + return _step_recommendation( + "reconcile_step", + step, + plan_step, + "Reconcile the local and remote effect before explicitly preparing a retry.", + False, + ) + return _step_recommendation( + "retry_step", + step, + plan_step, + "The read-only attempt was interrupted and can be explicitly prepared for retry.", + True, + ) + if step["state"] == "pending": + available, reason = _step_available(record, step["id"]) + return _step_recommendation( + "execute_step" if available else "unavailable_step", + step, + plan_step, + "Use the existing preview and narrowly confirmed executor controls." + if available + else reason, + available, + ) + return { + "id": "run_complete", + "step_id": None, + "title": "Release run complete", + "detail": "Every recorded step succeeded.", + "remediation": "Review publication and installation evidence before closing the release.", + "available": False, + } + + +def _step_recommendation( + recommendation_id: str, + step: dict[str, Any], + plan_step: dict[str, Any], + remediation: str, + available: bool, +) -> dict[str, Any]: + return { + "id": recommendation_id, + "step_id": step["id"], + "title": plan_step.get("title") or step["id"], + "detail": plan_step.get("detail") or "Release plan step.", + "remediation": remediation, + "available": available, + } + + +def _step_pair( + record: dict[str, Any], step_id: str +) -> tuple[dict[str, Any], dict[str, Any]]: + for step, plan_step in zip( + record["state"]["steps"], + record["immutable"]["plan"]["dry_run_steps"], + strict=True, + ): + if step["id"] == step_id: + return step, plan_step + raise ReleaseRunNotFound("Release run step was not found.") + + +def _append_event( + record: dict[str, Any], + *, + event_type: str, + step_id: str | None = None, + result_code: str | None = None, +) -> None: + if event_type not in EVENT_TYPES: + raise ReleaseRunConflict("Release event type is invalid.") + event: dict[str, Any] = { + "sequence": record["state"]["next_event_sequence"], + "at": _timestamp(), + "type": event_type, + } + if step_id is not None: + _validate_step_id(step_id) + event["step_id"] = step_id + if result_code is not None: + event["result_code"] = _safe_code(result_code) + record["state"]["next_event_sequence"] += 1 + record["state"]["events"].append(event) + record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:] + + +def _remember_request( + record: dict[str, Any], fingerprint: str, operation: str, step_id: str | None +) -> None: + record["state"]["processed_requests"].append( + { + "fingerprint": fingerprint, + "operation": operation, + "step_id": step_id, + } + ) + record["state"]["processed_requests"] = record["state"]["processed_requests"][ + -MAX_REQUESTS: + ] + + +def _request_seen( + record: dict[str, Any], fingerprint: str, operation: str, step_id: str | None +) -> bool: + matches = [ + request + for request in record["state"]["processed_requests"] + if request["fingerprint"] == fingerprint + ] + if not matches: + return False + if any( + request["operation"] == operation and request["step_id"] == step_id + for request in matches + ): + return True + raise ReleaseRunConflict( + "The request identifier was already used for a different run-state action." + ) + + +def _refresh_status(record: dict[str, Any]) -> None: + record["state"]["status"] = _status(record) + + +def _status(record: dict[str, Any]) -> str: + if record["immutable"]["plan"].get("status") == "blocked": + return "blocked" + states = [step["state"] for step in record["state"]["steps"]] + if states and all(state == "succeeded" for state in states): + return "completed" + if "running" in states: + return "running" + if any(state in {"failed", "interrupted"} for state in states): + return "attention" + if states: + first_pending = next( + step + for step, mutable in zip( + record["immutable"]["plan"]["dry_run_steps"], + record["state"]["steps"], + strict=True, + ) + if mutable["state"] == "pending" + ) + if first_pending.get("status", "planned") != "planned": + return "blocked" + return "planned" + + +def _immutable_digest(immutable: dict[str, Any]) -> str: + return hashlib.sha256( + _canonical_json( + {"input": immutable.get("input"), "plan": immutable.get("plan")} + ) + ).hexdigest() + + +def _record_digest(record: dict[str, Any]) -> str: + return hashlib.sha256( + _canonical_json( + {key: value for key, value in record.items() if key != "record_digest"} + ) + ).hexdigest() + + +def _request_fingerprint(request_id: str) -> str: + if not isinstance(request_id, str) or not REQUEST_ID_PATTERN.fullmatch(request_id): + raise ReleaseRunConflict("Run-state request identifier is invalid.") + return hashlib.sha256(request_id.encode("utf-8")).hexdigest() + + +def _validate_step_id(step_id: Any) -> None: + if not isinstance(step_id, str) or not STEP_ID_PATTERN.fullmatch(step_id): + raise ReleaseRunConflict("Release plan step identifier is invalid.") + + +def _safe_code(value: str) -> str: + if ( + not isinstance(value, str) + or len(value) > 64 + or not re.fullmatch(r"[a-z][a-z0-9_]*", value) + ): + raise ReleaseRunConflict("Release result code is invalid.") + return value + + +def _is_sha256(value: Any) -> bool: + return isinstance(value, str) and bool(re.fullmatch(r"[0-9a-f]{64}", value)) + + +def _is_timestamp(value: Any) -> bool: + if not isinstance(value, str) or not re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", value + ): + return False + try: + datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return False + return True + + +def _validate_step_state_fields(step: dict[str, Any]) -> None: + state = step["state"] + attempt_count = step["attempt_count"] + fingerprint = step["attempt_fingerprint"] + started_at = step["started_at"] + finished_at = step["finished_at"] + result_code = step["result_code"] + if started_at is not None and not _is_timestamp(started_at): + raise ValueError + if finished_at is not None and not _is_timestamp(finished_at): + raise ValueError + if state == "pending": + if any( + value is not None + for value in (fingerprint, started_at, finished_at, result_code) + ): + raise ValueError + return + if attempt_count < 1 or fingerprint is None or started_at is None: + raise ValueError + if state == "running": + if finished_at is not None or result_code is not None: + raise ValueError + return + if finished_at is None or result_code is None: + raise ValueError + if state == "interrupted" and result_code != "process_interrupted": + raise ValueError + + +def _validate_event_fields(event: dict[str, Any]) -> None: + event_type = event["type"] + has_step = "step_id" in event + has_result = "result_code" in event + if event_type in {"run_created", "run_resumed"}: + if has_step or has_result: + raise ValueError + return + if not has_step: + raise ValueError + if event_type == "step_started": + if has_result: + raise ValueError + return + if not has_result: + raise ValueError + + +def _reject_symlink_components(path: Path) -> None: + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + if current.is_symlink(): + raise ReleaseRunCorrupt( + "Release run storage must not contain symbolic-link components." + ) + + +def _canonical_json(value: Any) -> bytes: + try: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ReleaseRunConflict("Release run contains non-JSON data.") from exc + + +def _timestamp() -> str: + return ( + datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + ) diff --git a/tools/release/server/app.py b/tools/release/server/app.py index f3ab4e5..6685f47 100644 --- a/tools/release/server/app.py +++ b/tools/release/server/app.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib from pathlib import Path from fastapi import FastAPI, HTTPException, Request @@ -21,6 +22,12 @@ from govoplan_release import ( tag_repositories, ) from govoplan_release.model import to_jsonable +from govoplan_release.release_run import ( + ReleaseRunConflict, + ReleaseRunCorrupt, + ReleaseRunNotFound, + ReleaseRunStore, +) from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT, META_ROOT @@ -90,10 +97,35 @@ class RepositoryTagRequest(BaseModel): confirm: str = "" -def create_app(*, workspace_root: Path = DEFAULT_WORKSPACE_ROOT, token: str | None = None) -> FastAPI: +class ReleaseRunCreateRequest(BaseModel): + repo_versions: dict[str, str] = Field(default_factory=dict) + channel: str = Field(default="stable", min_length=1, max_length=64) + online: bool = False + remote_tags: bool = False + public_catalog: bool = True + include_migrations: bool = False + + +class ReleaseRunCommandRequest(BaseModel): + request_id: str = Field(min_length=8, max_length=128) + reconciled: bool = False + + +def create_app( + *, + workspace_root: Path = DEFAULT_WORKSPACE_ROOT, + token: str | None = None, + run_state_root: Path | None = None, +) -> FastAPI: app = FastAPI(title="GovOPlaN Release Console", version="0.1.0") app.state.workspace_root = workspace_root app.state.token = token + app.state.release_runs = ReleaseRunStore( + run_state_root + if run_state_root is not None + else default_release_run_root(workspace_root) + ) + app.state.workspace_fingerprint = release_workspace_fingerprint(workspace_root) @app.middleware("http") async def require_token(request: Request, call_next): # type: ignore[no-untyped-def] @@ -202,6 +234,105 @@ def create_app(*, workspace_root: Path = DEFAULT_WORKSPACE_ROOT, token: str | No ) return to_jsonable(release_plan) + @app.get("/api/release-runs") + def release_runs(limit: int = 20) -> dict[str, object]: + return {"runs": app.state.release_runs.list(limit=limit)} + + @app.post("/api/release-runs") + def create_release_run(request: ReleaseRunCreateRequest) -> dict[str, object]: + if not request.repo_versions: + raise HTTPException( + status_code=400, detail="At least one repository version is required" + ) + snapshot = build_dashboard( + workspace_root=app.state.workspace_root, + online=request.online, + check_remote_tags=request.online or request.remote_tags, + check_public_catalog=request.public_catalog, + include_migrations=request.include_migrations, + channel=request.channel, + ) + plan = build_selective_release_plan( + snapshot, + selected_repos=tuple(request.repo_versions), + repo_versions=dict(request.repo_versions), + channel=request.channel, + ) + plan_payload = to_jsonable(plan) + planned_units = plan_payload.get("units") + planned_versions = ( + { + unit.get("repo"): unit.get("target_version") + for unit in planned_units + if isinstance(unit, dict) + and isinstance(unit.get("repo"), str) + and isinstance(unit.get("target_version"), str) + } + if isinstance(planned_units, list) + else {} + ) + if planned_versions != dict(request.repo_versions) or len( + planned_versions + ) != len(planned_units or []): + raise HTTPException( + status_code=409, + detail=( + "The release plan did not resolve exactly the requested repository " + "versions; refresh the dashboard and select known release units." + ), + ) + try: + return app.state.release_runs.create( + input_snapshot={ + "channel": request.channel, + "repo_versions": dict(request.repo_versions), + "online": request.online, + "remote_tags": request.remote_tags, + "public_catalog": request.public_catalog, + "include_migrations": request.include_migrations, + "workspace_fingerprint": app.state.workspace_fingerprint, + }, + plan_snapshot=plan_payload, + ) + except (ReleaseRunConflict, ReleaseRunCorrupt) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + @app.get("/api/release-runs/{run_id}") + def release_run(run_id: str) -> dict[str, object]: + try: + return app.state.release_runs.get(run_id) + except ReleaseRunNotFound as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ReleaseRunCorrupt as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + @app.post("/api/release-runs/{run_id}/resume") + def resume_release_run( + run_id: str, request: ReleaseRunCommandRequest + ) -> dict[str, object]: + try: + return app.state.release_runs.resume(run_id, request_id=request.request_id) + except ReleaseRunNotFound as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (ReleaseRunConflict, ReleaseRunCorrupt) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + @app.post("/api/release-runs/{run_id}/steps/{step_id}/retry") + def retry_release_run_step( + run_id: str, step_id: str, request: ReleaseRunCommandRequest + ) -> dict[str, object]: + try: + return app.state.release_runs.retry_step( + run_id, + step_id, + request_id=request.request_id, + reconciled=request.reconciled, + ) + except ReleaseRunNotFound as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (ReleaseRunConflict, ReleaseRunCorrupt) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + @app.get("/api/catalog-candidates") def catalog_candidates() -> dict[str, object]: return {"candidates": list_catalog_candidates()} @@ -319,6 +450,22 @@ def create_app(*, workspace_root: Path = DEFAULT_WORKSPACE_ROOT, token: str | No return app +def default_release_run_root(workspace_root: Path) -> Path: + workspace = workspace_root.expanduser().resolve() + if (workspace / "repositories.json").is_file(): + return workspace / "runtime" / "release-runs" + workspace_meta = workspace / "govoplan" + if (workspace_meta / "repositories.json").is_file(): + return workspace_meta / "runtime" / "release-runs" + fingerprint = release_workspace_fingerprint(workspace) + return META_ROOT / "runtime" / "release-runs" / f"workspace-{fingerprint[:16]}" + + +def release_workspace_fingerprint(workspace_root: Path) -> str: + canonical = str(workspace_root.expanduser().resolve()).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + def parse_csv(value: str | None) -> tuple[str, ...]: if not value: return () diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html index dcb9a8e..f0c8131 100644 --- a/tools/release/webui/index.html +++ b/tools/release/webui/index.html @@ -96,7 +96,8 @@ } input[type="text"], - input[type="password"] { + input[type="password"], + select { width: 100%; min-height: 36px; border: 1px solid var(--line); @@ -139,7 +140,7 @@ } button:disabled { - cursor: wait; + cursor: not-allowed; opacity: 0.65; } @@ -635,6 +636,31 @@
+
+
+

Durable Run State

+ no run selected +
+
+

State tracking only. A run freezes the selected inputs and plan, guides recovery, and records bounded state events. It does not execute or confirm tags, signing, publication, or catalog changes.

+
+
+ + +
+
+
+ + +
+
+
Create a run after selecting exact repository versions, or choose a saved run.
+
+
+
+

Release Workflow

@@ -857,6 +883,11 @@ previewPrepare: document.getElementById("previewPrepare"), commitPrepare: document.getElementById("commitPrepare"), buildSelectedPlan: document.getElementById("buildSelectedPlan"), + releaseRun: document.getElementById("releaseRun"), + runStatus: document.getElementById("runStatus"), + runOutput: document.getElementById("runOutput"), + createReleaseRun: document.getElementById("createReleaseRun"), + resumeReleaseRun: document.getElementById("resumeReleaseRun"), selectUnpushedUnits: document.getElementById("selectUnpushedUnits"), selectChangedUnits: document.getElementById("selectChangedUnits"), selectUnreleasedUnits: document.getElementById("selectUnreleasedUnits"), @@ -874,6 +905,8 @@ rows: {}, dashboard: null, manualRepo: "", + currentRun: null, + runs: [], }; elements.refresh.addEventListener("click", load); @@ -889,6 +922,9 @@ elements.previewPrepare.addEventListener("click", () => prepareSelectedRepos({ apply: false })); elements.commitPrepare.addEventListener("click", () => prepareSelectedRepos({ apply: true })); elements.buildSelectedPlan.addEventListener("click", buildSelectedPlan); + elements.createReleaseRun.addEventListener("click", createReleaseRun); + elements.resumeReleaseRun.addEventListener("click", resumeReleaseRun); + elements.releaseRun.addEventListener("change", () => loadReleaseRun(elements.releaseRun.value)); elements.previewReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: false, push: true })); elements.createReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: true, push: false })); elements.publishReleaseTags.addEventListener("click", () => releaseSelectedTags({ apply: true, push: true })); @@ -949,10 +985,15 @@ async function load() { elements.statusLine.textContent = "Refreshing..."; try { - const [dashboard, intelligence] = await Promise.all([api("/api/dashboard"), api("/api/release-intelligence")]); + const [dashboard, intelligence, runs] = await Promise.all([api("/api/dashboard"), api("/api/release-intelligence"), api("/api/release-runs")]); renderDashboard(dashboard); renderReleaseIntelligence(intelligence); - renderIdlePlan(); + renderReleaseRunList(runs.runs || []); + if (releaseState.currentRun?.run_id) { + await loadReleaseRun(releaseState.currentRun.run_id, { restoreSelection: false }); + } else { + renderIdlePlan(); + } elements.statusLine.textContent = `Generated ${dashboard.generated_at}`; } catch (error) { elements.statusLine.innerHTML = `${escapeHtml(error.message)}`; @@ -977,6 +1018,156 @@ renderCatalog(data.catalog, data.target_version); } + function renderReleaseRunList(runs) { + releaseState.runs = Array.isArray(runs) ? runs : []; + const selectedId = releaseState.currentRun?.run_id || ""; + const options = releaseState.runs.map((run) => { + const repositories = run.repo_versions ? Object.keys(run.repo_versions).length : 0; + const label = `${run.run_id} · ${run.status || "unknown"} · ${repositories} repos`; + return ``; + }).join(""); + elements.releaseRun.innerHTML = `${options}`; + if (selectedId && releaseState.runs.some((run) => run.run_id === selectedId)) { + elements.releaseRun.value = selectedId; + } + } + + async function createReleaseRun() { + const repoVersions = selectedRepoVersions(); + if (!Object.keys(repoVersions).length) { + elements.runStatus.textContent = "selection required"; + elements.runOutput.innerHTML = `

${pill("unavailable", "warn")} Select release units

Select one or more repositories and exact target versions before creating a durable run.

`; + return; + } + setBusy([elements.createReleaseRun], true); + elements.runStatus.textContent = "freezing plan..."; + try { + const run = await postJson("/api/release-runs", { + repo_versions: repoVersions, + channel: elements.channel.value.trim() || "stable", + online: false, + remote_tags: elements.online.checked, + public_catalog: elements.publicCatalog.checked, + include_migrations: elements.migrations.checked, + }); + releaseState.currentRun = run; + restoreReleaseRunSelection(run); + renderReleaseRun(run); + const runs = await api("/api/release-runs"); + renderReleaseRunList(runs.runs || []); + elements.releaseRun.value = run.run_id; + } catch (error) { + elements.runStatus.textContent = "error"; + elements.runOutput.innerHTML = `

${pill("error", "block")} Run creation failed

${escapeHtml(error.message)}

`; + } finally { + setBusy([elements.createReleaseRun], false); + } + } + + async function loadReleaseRun(runId, options = {}) { + if (!runId) { + releaseState.currentRun = null; + elements.runStatus.textContent = "no run selected"; + elements.resumeReleaseRun.disabled = true; + elements.runOutput.innerHTML = `
Create a run after selecting exact repository versions, or choose a saved run.
`; + return; + } + elements.runStatus.textContent = "loading..."; + try { + const run = await api(`/api/release-runs/${encodeURIComponent(runId)}`); + releaseState.currentRun = run; + if (options.restoreSelection !== false) restoreReleaseRunSelection(run); + renderReleaseRun(run); + elements.releaseRun.value = run.run_id; + } catch (error) { + releaseState.currentRun = null; + elements.runStatus.textContent = "unavailable"; + elements.resumeReleaseRun.disabled = true; + elements.runOutput.innerHTML = `

${pill("unavailable", "block")} Run cannot be read

${escapeHtml(error.message)}

`; + } + } + + function restoreReleaseRunSelection(run) { + const input = run.immutable?.input || {}; + const repoVersions = input.repo_versions || {}; + elements.channel.value = input.channel || "stable"; + elements.publicCatalog.checked = input.public_catalog === true; + elements.online.checked = input.remote_tags === true || input.online === true; + elements.migrations.checked = input.include_migrations === true; + for (const repo of Object.keys(releaseState.rows)) { + releaseState.rows[repo] = { + ...releaseState.rows[repo], + selected: Object.prototype.hasOwnProperty.call(repoVersions, repo), + ...(Object.prototype.hasOwnProperty.call(repoVersions, repo) ? { target: repoVersions[repo] } : {}), + }; + } + if (releaseState.dashboard) renderReleaseUnits(releaseState.dashboard); + if (run.immutable?.plan) renderPlan(run.immutable.plan); + } + + function renderReleaseRun(run) { + const state = run.state || {}; + const recommendation = run.recommended_next || {}; + const steps = Array.isArray(state.steps) ? state.steps : []; + const events = Array.isArray(state.events) ? state.events.slice(-5).reverse() : []; + const recommendationKind = state.status === "blocked" ? "block" : recommendation.available ? "ok" : "warn"; + const recommendationHtml = recommendation.id ? `
+

${pill("recommended next", recommendationKind)} ${escapeHtml(recommendation.title || recommendation.id)}

+

${escapeHtml(recommendation.detail || "")}

+

How to continue: ${escapeHtml(recommendation.remediation || "")}

+
` : ""; + const stepHtml = steps.map((step) => { + const kind = step.state === "succeeded" ? "ok" : ["failed", "interrupted"].includes(step.state) ? "block" : step.available ? "ok" : "warn"; + const retryReason = step.retry_available ? "Prepare this known failed or read-only interrupted step for an explicit retry." : (step.disabled_reason || "Retry is unavailable for this state."); + return `
+

${escapeHtml(`${step.order}. ${step.title || step.id}`)}

${pill(step.state, kind)}
+

${escapeHtml(step.detail || "")}

+ ${step.result_code ? `

Result code: ${escapeHtml(step.result_code)}

` : ""} +
+
`; + }).join(""); + const eventHtml = events.length ? `

Recent bounded state events

${events.map((event) => `

${escapeHtml(`${event.at} · ${event.type}${event.step_id ? ` · ${event.step_id}` : ""}${event.result_code ? ` · ${event.result_code}` : ""}`)}

`).join("")}
` : ""; + elements.runStatus.textContent = state.status || "unknown"; + elements.resumeReleaseRun.disabled = state.status === "completed"; + elements.runOutput.innerHTML = `

${pill(state.status || "unknown", state.status === "blocked" ? "block" : state.status === "completed" ? "ok" : "warn")} ${escapeHtml(run.run_id)}

Frozen plan ${escapeHtml((run.immutable?.digest || "").slice(0, 16))}… · updated ${escapeHtml(run.updated_at || "-")}

Executor controls remain separate and keep their existing confirmations.

${recommendationHtml}${stepHtml}${eventHtml}`; + for (const button of elements.runOutput.querySelectorAll("[data-run-retry-step]")) { + button.addEventListener("click", () => retryReleaseRunStep(button.dataset.runRetryStep)); + } + } + + async function resumeReleaseRun() { + const run = releaseState.currentRun; + if (!run) return; + setBusy([elements.resumeReleaseRun], true); + try { + const resumed = await postJson(`/api/release-runs/${encodeURIComponent(run.run_id)}/resume`, { request_id: requestId("resume") }); + releaseState.currentRun = resumed; + renderReleaseRun(resumed); + } catch (error) { + elements.runStatus.textContent = "error"; + elements.runOutput.insertAdjacentHTML("afterbegin", `

${pill("error", "block")} Resume failed

${escapeHtml(error.message)}

`); + } finally { + elements.resumeReleaseRun.disabled = releaseState.currentRun?.state?.status === "completed"; + } + } + + async function retryReleaseRunStep(stepId) { + const run = releaseState.currentRun; + if (!run || !stepId) return; + try { + const retried = await postJson(`/api/release-runs/${encodeURIComponent(run.run_id)}/steps/${encodeURIComponent(stepId)}/retry`, { request_id: requestId("retry"), reconciled: false }); + releaseState.currentRun = retried; + renderReleaseRun(retried); + } catch (error) { + elements.runOutput.insertAdjacentHTML("afterbegin", `

${pill("error", "block")} Retry preparation failed

${escapeHtml(error.message)}

`); + } + } + + function requestId(prefix) { + const identifier = window.crypto?.randomUUID ? window.crypto.randomUUID() : `${Date.now()}-${Math.random().toString(16).slice(2)}`; + return `${prefix}-${identifier}`; + } + function renderWorkflowStages(data) { const s = data.summary || {}; const selected = elements.releaseUnits ? elements.releaseUnits.querySelectorAll("[data-release-check]:checked").length : 0;