942 lines
37 KiB
Python
942 lines
37 KiB
Python
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
META_ROOT = Path(__file__).resolve().parents[1]
|
|
RELEASE_ROOT = META_ROOT / "tools" / "release"
|
|
if str(RELEASE_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(RELEASE_ROOT))
|
|
|
|
from govoplan_release.release_run import ( # noqa: E402
|
|
MAX_EVENTS,
|
|
ReleaseRunConflict,
|
|
ReleaseRunCorrupt,
|
|
ReleaseRunNotFound,
|
|
ReleaseRunStore,
|
|
_record_digest,
|
|
)
|
|
from server.app import ( # noqa: E402
|
|
create_app,
|
|
default_release_run_root,
|
|
release_workspace_fingerprint,
|
|
)
|
|
|
|
|
|
WORKSPACE_FINGERPRINT = "a" * 64
|
|
|
|
|
|
class ReleaseRunStoreTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temporary.name) / "release-runs"
|
|
self.store = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temporary.cleanup()
|
|
|
|
def test_create_persists_private_immutable_plan_and_ordered_steps(self) -> None:
|
|
run = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)
|
|
|
|
reopened = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
).get(run["run_id"])
|
|
record_path = self.root / f"{run['run_id']}.json"
|
|
|
|
self.assertEqual("govoplan.release-run", reopened["schema"])
|
|
self.assertEqual(2, reopened["schema_version"])
|
|
self.assertEqual(
|
|
{"govoplan-core": "1.2.3", "govoplan-files": "1.2.4"},
|
|
reopened["immutable"]["input"]["repo_versions"],
|
|
)
|
|
self.assertEqual("planned", reopened["state"]["status"])
|
|
self.assertTrue(reopened["state"]["steps"][0]["available"])
|
|
self.assertFalse(reopened["state"]["steps"][1]["available"])
|
|
self.assertEqual("core:preflight", reopened["recommended_next"]["step_id"])
|
|
self.assertEqual(0o700, os.stat(self.root).st_mode & 0o777)
|
|
self.assertEqual(0o600, os.stat(record_path).st_mode & 0o777)
|
|
self.assertNotIn("processed_requests", reopened["state"])
|
|
self.assertNotIn("attempt_fingerprint", reopened["state"]["steps"][0])
|
|
|
|
def test_zero_step_plan_is_rejected(self) -> None:
|
|
plan = release_plan()
|
|
plan["dry_run_steps"] = []
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "step collection"):
|
|
self.store.create(input_snapshot=run_input(), plan_snapshot=plan)
|
|
|
|
def test_attempt_transitions_are_idempotent_and_strictly_ordered(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "Complete core:preflight"):
|
|
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
|
|
|
|
started = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="attempt-preflight-0001"
|
|
)
|
|
duplicate_start = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="attempt-preflight-0001"
|
|
)
|
|
self.assertEqual(1, started["state"]["steps"][0]["attempt_count"])
|
|
self.assertEqual(
|
|
len(started["state"]["events"]),
|
|
len(duplicate_start["state"]["events"]),
|
|
)
|
|
|
|
finished = self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="attempt-preflight-0001",
|
|
succeeded=True,
|
|
result_code="preflight_valid",
|
|
)
|
|
duplicate_finish = self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="attempt-preflight-0001",
|
|
succeeded=True,
|
|
result_code="preflight_valid",
|
|
)
|
|
self.assertEqual("succeeded", finished["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(finished["state"]["events"]),
|
|
len(duplicate_finish["state"]["events"]),
|
|
)
|
|
self.assertTrue(finished["state"]["steps"][1]["available"])
|
|
|
|
def test_failed_attempt_and_delayed_start_replays_are_idempotent(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
attempt_id = "failed-attempt-replay-0001"
|
|
self.store.start_step(run_id, "core:preflight", attempt_id=attempt_id)
|
|
failed = self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id=attempt_id,
|
|
succeeded=False,
|
|
result_code="preflight_failed",
|
|
)
|
|
duplicate = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id=attempt_id
|
|
)
|
|
self.assertEqual("failed", duplicate["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(failed["state"]["events"]), len(duplicate["state"]["events"])
|
|
)
|
|
|
|
retried = self.store.retry_step(
|
|
run_id,
|
|
"core:preflight",
|
|
request_id="failed-retry-request-0001",
|
|
)
|
|
delayed = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id=attempt_id
|
|
)
|
|
self.assertEqual("pending", delayed["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(retried["state"]["events"]), len(delayed["state"]["events"])
|
|
)
|
|
|
|
def test_delayed_resume_request_is_idempotent_for_the_run_lifetime(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
original_request = "resume-original-request-0001"
|
|
self.store.resume(run_id, request_id=original_request)
|
|
for index in range(130):
|
|
self.store.resume(
|
|
run_id, request_id=f"resume-intervening-request-{index:04d}"
|
|
)
|
|
running = self.store.start_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="resume-delayed-attempt-0001",
|
|
)
|
|
duplicate = self.store.resume(run_id, request_id=original_request)
|
|
self.assertEqual("running", duplicate["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(running["state"]["events"]), len(duplicate["state"]["events"])
|
|
)
|
|
|
|
def test_command_and_attempt_capacity_fail_closed_without_eviction(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
with patch("govoplan_release.release_run.MAX_COMMANDS", 2):
|
|
self.store.resume(run_id, request_id="capacity-resume-request-0001")
|
|
self.store.resume(run_id, request_id="capacity-resume-request-0002")
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "safety limit"):
|
|
self.store.resume(run_id, request_id="capacity-resume-request-0003")
|
|
duplicate = self.store.resume(
|
|
run_id, request_id="capacity-resume-request-0001"
|
|
)
|
|
self.assertEqual("planned", duplicate["state"]["status"])
|
|
|
|
attempt_run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
with patch("govoplan_release.release_run.MAX_COMMANDS", 1):
|
|
self.store.start_step(
|
|
attempt_run_id,
|
|
"core:preflight",
|
|
attempt_id="capacity-attempt-0001",
|
|
)
|
|
self.store.finish_step(
|
|
attempt_run_id,
|
|
"core:preflight",
|
|
attempt_id="capacity-attempt-0001",
|
|
succeeded=False,
|
|
result_code="preflight_failed",
|
|
)
|
|
self.store.retry_step(
|
|
attempt_run_id,
|
|
"core:preflight",
|
|
request_id="capacity-retry-request-0001",
|
|
)
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "attempt history"):
|
|
self.store.start_step(
|
|
attempt_run_id,
|
|
"core:preflight",
|
|
attempt_id="capacity-attempt-0002",
|
|
)
|
|
delayed_attempt = self.store.start_step(
|
|
attempt_run_id,
|
|
"core:preflight",
|
|
attempt_id="capacity-attempt-0001",
|
|
)
|
|
self.assertEqual("pending", delayed_attempt["state"]["steps"][0]["state"])
|
|
|
|
def test_reopen_and_resume_preserve_ambiguous_mutating_effect(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
|
|
)["run_id"]
|
|
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
|
|
|
|
reopened_store = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
)
|
|
before_resume = reopened_store.get(run_id)
|
|
self.assertEqual("running", before_resume["state"]["steps"][0]["state"])
|
|
|
|
resumed = reopened_store.resume(run_id, request_id="resume-request-0001")
|
|
duplicate = reopened_store.resume(run_id, request_id="resume-request-0001")
|
|
self.assertEqual("interrupted", resumed["state"]["steps"][0]["state"])
|
|
self.assertEqual("reconcile_step", resumed["recommended_next"]["id"])
|
|
self.assertTrue(resumed["recommended_next"]["available"])
|
|
self.assertEqual(
|
|
len(resumed["state"]["events"]), len(duplicate["state"]["events"])
|
|
)
|
|
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "observed effect"):
|
|
reopened_store.retry_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="retry-request-0001",
|
|
)
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "Type RECONCILE"):
|
|
reopened_store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-request-0001",
|
|
outcome="effect_absent",
|
|
confirmation="",
|
|
)
|
|
retried = reopened_store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-request-0002",
|
|
outcome="effect_absent",
|
|
confirmation="RECONCILE",
|
|
)
|
|
duplicate_retry = reopened_store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-request-0002",
|
|
outcome="effect_absent",
|
|
confirmation="RECONCILE",
|
|
)
|
|
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(retried["state"]["events"]),
|
|
len(duplicate_retry["state"]["events"]),
|
|
)
|
|
|
|
def test_reconciliation_can_record_unresolved_or_verified_success(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
|
|
)["run_id"]
|
|
self.store.start_step(run_id, "core:tag", attempt_id="reconcile-attempt-0001")
|
|
self.store.resume(run_id, request_id="reconcile-resume-request-0001")
|
|
|
|
unresolved = self.store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-outcome-request-0001",
|
|
outcome="unresolved",
|
|
confirmation="RECONCILE",
|
|
)
|
|
self.assertEqual("interrupted", unresolved["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
"unresolved", unresolved["state"]["events"][-1]["result_code"]
|
|
)
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "different run-state action"):
|
|
self.store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-outcome-request-0001",
|
|
outcome="effect_succeeded",
|
|
confirmation="RECONCILE",
|
|
)
|
|
|
|
succeeded = self.store.reconcile_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="reconcile-outcome-request-0002",
|
|
outcome="effect_succeeded",
|
|
confirmation="RECONCILE",
|
|
)
|
|
self.assertEqual("succeeded", succeeded["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
"reconciled_effect_succeeded",
|
|
succeeded["state"]["steps"][0]["result_code"],
|
|
)
|
|
self.assertEqual("completed", succeeded["state"]["status"])
|
|
|
|
def test_tampered_record_fails_closed_and_is_not_rewritten(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["immutable"]["input"]["channel"] = "testing"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
tampered = path.read_bytes()
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
self.assertEqual(tampered, path.read_bytes())
|
|
self.assertEqual("unavailable", self.store.list()[0]["status"])
|
|
|
|
def test_broad_record_mode_and_symlinked_root_fail_closed(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
path.chmod(0o640)
|
|
with self.assertRaisesRegex(ReleaseRunCorrupt, "broader than 0600"):
|
|
self.store.get(run_id)
|
|
self.assertEqual(0o640, os.stat(path).st_mode & 0o777)
|
|
|
|
target = Path(self.temporary.name) / "target"
|
|
target.mkdir()
|
|
linked = Path(self.temporary.name) / "linked"
|
|
linked.symlink_to(target, target_is_directory=True)
|
|
with self.assertRaisesRegex(ReleaseRunCorrupt, "symbolic-link"):
|
|
ReleaseRunStore(
|
|
linked,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
).list()
|
|
|
|
def test_state_specific_fields_and_boolean_counts_fail_closed(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["attempt_count"] = True
|
|
payload["state"]["steps"][0]["started_at"] = "2026-07-22T00:00:00Z"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
def test_valid_looking_mutable_state_tampering_fails_record_checksum(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
|
|
)["run_id"]
|
|
self.store.start_step(run_id, "core:tag", attempt_id="checksum-attempt-0001")
|
|
self.store.finish_step(
|
|
run_id,
|
|
"core:tag",
|
|
attempt_id="checksum-attempt-0001",
|
|
succeeded=True,
|
|
result_code="tag_created",
|
|
)
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["result_code"] = "tag_published"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
tampered = path.read_bytes()
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
self.assertEqual(tampered, path.read_bytes())
|
|
|
|
def test_impossible_step_chronology_fails_even_with_recomputed_checksum(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="chronology-attempt-0001"
|
|
)
|
|
self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="chronology-attempt-0001",
|
|
succeeded=True,
|
|
result_code="preflight_valid",
|
|
)
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["started_at"] = "2026-07-22T20:00:00Z"
|
|
payload["state"]["steps"][0]["finished_at"] = "2026-07-22T19:00:00Z"
|
|
payload["record_digest"] = _record_digest(payload)
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
def test_record_and_retained_event_timestamps_are_strictly_ordered(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
self.store.resume(run_id, request_id="timestamp-resume-request-0001")
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["created_at"] = "2026-07-22T00:00:00Z"
|
|
payload["updated_at"] = "2026-07-22T00:00:03Z"
|
|
payload["state"]["events"][0]["at"] = "2026-07-22T00:00:02Z"
|
|
payload["state"]["events"][1]["at"] = "2026-07-22T00:00:01Z"
|
|
payload["record_digest"] = _record_digest(payload)
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
def test_attempt_count_must_match_lifetime_attempt_history(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="counted-attempt-0001"
|
|
)
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["attempt_count"] = 2
|
|
payload["record_digest"] = _record_digest(payload)
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
def test_workspace_fingerprint_is_enforced_for_every_store_operation(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
foreign = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint="b" * 64,
|
|
)
|
|
|
|
self.assertEqual([], foreign.list())
|
|
with self.assertRaises(ReleaseRunNotFound):
|
|
foreign.get(run_id)
|
|
with self.assertRaises(ReleaseRunNotFound):
|
|
foreign.resume(run_id, request_id="foreign-resume-request-0001")
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "different workspace"):
|
|
foreign.create(input_snapshot=run_input(), plan_snapshot=release_plan())
|
|
|
|
def test_list_limit_is_applied_after_foreign_workspace_records_are_skipped(
|
|
self,
|
|
) -> None:
|
|
with patch(
|
|
"govoplan_release.release_run._timestamp",
|
|
return_value="2026-07-22T00:00:00Z",
|
|
):
|
|
local_run = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)
|
|
foreign_store = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint="b" * 64,
|
|
)
|
|
foreign_input = run_input()
|
|
foreign_input["workspace_fingerprint"] = "b" * 64
|
|
with patch(
|
|
"govoplan_release.release_run._timestamp",
|
|
return_value="2026-07-22T00:00:01Z",
|
|
):
|
|
foreign_store.create(
|
|
input_snapshot=foreign_input, plan_snapshot=release_plan()
|
|
)
|
|
|
|
listed = self.store.list(limit=1)
|
|
self.assertEqual([local_run["run_id"]], [item["run_id"] for item in listed])
|
|
|
|
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
barrier = threading.Barrier(2)
|
|
|
|
def claim(attempt_id: str) -> str:
|
|
contender = ReleaseRunStore(
|
|
self.root,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
)
|
|
barrier.wait(timeout=5)
|
|
try:
|
|
contender.start_step(run_id, "core:preflight", attempt_id=attempt_id)
|
|
except ReleaseRunConflict:
|
|
return "conflict"
|
|
return "started"
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(
|
|
executor.map(
|
|
claim, ("concurrent-attempt-0001", "concurrent-attempt-0002")
|
|
)
|
|
)
|
|
|
|
reopened = self.store.get(run_id)
|
|
self.assertEqual(["conflict", "started"], sorted(results))
|
|
self.assertEqual(1, reopened["state"]["steps"][0]["attempt_count"])
|
|
self.assertEqual(
|
|
1,
|
|
sum(
|
|
event["type"] == "step_started" for event in reopened["state"]["events"]
|
|
),
|
|
)
|
|
|
|
def test_event_history_is_bounded_and_command_history_is_not_evicted(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
for index in range(90):
|
|
attempt_id = f"bounded-attempt-{index:04d}"
|
|
self.store.start_step(run_id, "core:preflight", attempt_id=attempt_id)
|
|
self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id=attempt_id,
|
|
succeeded=False,
|
|
result_code="preflight_failed",
|
|
)
|
|
self.store.retry_step(
|
|
run_id,
|
|
"core:preflight",
|
|
request_id=f"bounded-retry-{index:04d}",
|
|
)
|
|
|
|
view = self.store.get(run_id)
|
|
raw = json.loads((self.root / f"{run_id}.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(MAX_EVENTS, len(view["state"]["events"]))
|
|
self.assertEqual(90, len(raw["state"]["processed_requests"]))
|
|
self.assertEqual(90, len(raw["state"]["processed_attempts"]))
|
|
self.assertNotIn("bounded-retry-0089", json.dumps(raw))
|
|
for event in view["state"]["events"]:
|
|
self.assertLessEqual(
|
|
set(event), {"sequence", "at", "type", "step_id", "result_code"}
|
|
)
|
|
|
|
|
|
class ReleaseRunApiTests(unittest.TestCase):
|
|
def test_token_guarded_create_list_read_resume_and_retry(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir),
|
|
run_state_root=root,
|
|
token="release-token",
|
|
)
|
|
headers = {"X-Release-Console-Token": "release-token"}
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
unauthorized = client.get("/api/release-runs")
|
|
created_response = client.post(
|
|
"/api/release-runs",
|
|
headers=headers,
|
|
json=run_input(),
|
|
)
|
|
self.assertEqual(200, created_response.status_code)
|
|
created = created_response.json()
|
|
run_id = created["run_id"]
|
|
listed = client.get("/api/release-runs", headers=headers).json()
|
|
loaded = client.get(
|
|
f"/api/release-runs/{run_id}", headers=headers
|
|
).json()
|
|
|
|
app.state.release_runs.start_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="api-attempt-0001",
|
|
)
|
|
resumed = client.post(
|
|
f"/api/release-runs/{run_id}/resume",
|
|
headers=headers,
|
|
json={"request_id": "api-resume-0001"},
|
|
)
|
|
retried = client.post(
|
|
f"/api/release-runs/{run_id}/steps/core:preflight/retry",
|
|
headers=headers,
|
|
json={"request_id": "api-retry-0001"},
|
|
)
|
|
|
|
self.assertEqual(401, unauthorized.status_code)
|
|
self.assertEqual(run_id, listed["runs"][0]["run_id"])
|
|
self.assertEqual(run_id, loaded["run_id"])
|
|
self.assertEqual(
|
|
"interrupted", resumed.json()["state"]["steps"][0]["state"]
|
|
)
|
|
self.assertEqual(200, retried.status_code)
|
|
self.assertEqual("pending", retried.json()["state"]["steps"][0]["state"])
|
|
|
|
def test_create_rejects_unknown_or_unresolved_repository(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir), run_state_root=root, token="token"
|
|
)
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
response = client.post(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
json={
|
|
"channel": "stable",
|
|
"repo_versions": {"govoplan-unknown": "1.0.0"},
|
|
"public_catalog": False,
|
|
},
|
|
)
|
|
|
|
self.assertEqual(409, response.status_code)
|
|
self.assertIn("did not resolve exactly", response.json()["detail"])
|
|
self.assertEqual([], list(root.glob("*.json")))
|
|
|
|
def test_api_rejects_tampered_record_without_rewriting_it(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
store = ReleaseRunStore(
|
|
root,
|
|
expected_workspace_fingerprint=WORKSPACE_FINGERPRINT,
|
|
)
|
|
run_id = store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = root / f"{run_id}.json"
|
|
path.write_text("{}", encoding="utf-8")
|
|
malformed = path.read_bytes()
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir), run_state_root=root, token="token"
|
|
)
|
|
with TestClient(app) as client:
|
|
response = client.get(
|
|
f"/api/release-runs/{run_id}",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
)
|
|
|
|
self.assertEqual(409, response.status_code)
|
|
self.assertEqual(malformed, path.read_bytes())
|
|
|
|
def test_api_maps_unavailable_run_storage_without_server_exception(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
parent = Path(temp_dir)
|
|
target = parent / "target"
|
|
target.mkdir()
|
|
linked_root = parent / "linked-runs"
|
|
linked_root.symlink_to(target, target_is_directory=True)
|
|
app = create_app(
|
|
workspace_root=parent,
|
|
run_state_root=linked_root,
|
|
token="token",
|
|
)
|
|
with TestClient(app) as client:
|
|
response = client.get(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
)
|
|
self.assertEqual(409, response.status_code)
|
|
self.assertIn("symbolic-link", response.json()["detail"])
|
|
|
|
def test_api_requires_confirmation_and_records_reconciliation_outcome(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir), run_state_root=root, token="token"
|
|
)
|
|
headers = {"X-Release-Console-Token": "token"}
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=mutating_first_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
created = client.post(
|
|
"/api/release-runs", headers=headers, json=run_input()
|
|
).json()
|
|
run_id = created["run_id"]
|
|
app.state.release_runs.start_step(
|
|
run_id, "core:tag", attempt_id="api-reconcile-attempt-0001"
|
|
)
|
|
client.post(
|
|
f"/api/release-runs/{run_id}/resume",
|
|
headers=headers,
|
|
json={"request_id": "api-reconcile-resume-0001"},
|
|
)
|
|
missing_confirmation = client.post(
|
|
f"/api/release-runs/{run_id}/steps/core:tag/reconcile",
|
|
headers=headers,
|
|
json={
|
|
"request_id": "api-reconcile-request-0001",
|
|
"outcome": "effect_succeeded",
|
|
},
|
|
)
|
|
reconciled = client.post(
|
|
f"/api/release-runs/{run_id}/steps/core:tag/reconcile",
|
|
headers=headers,
|
|
json={
|
|
"request_id": "api-reconcile-request-0002",
|
|
"outcome": "effect_succeeded",
|
|
"confirm": "RECONCILE",
|
|
},
|
|
)
|
|
|
|
self.assertEqual(409, missing_confirmation.status_code)
|
|
self.assertEqual(200, reconciled.status_code)
|
|
self.assertEqual("completed", reconciled.json()["state"]["status"])
|
|
self.assertEqual(
|
|
"effect_succeeded",
|
|
reconciled.json()["state"]["events"][-1]["result_code"],
|
|
)
|
|
|
|
def test_default_storage_and_immutable_input_are_workspace_bound(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
parent = Path(temp_dir)
|
|
state_home = parent / "operator-state"
|
|
first = parent / "first"
|
|
second = parent / "second"
|
|
first.mkdir()
|
|
second.mkdir()
|
|
with patch.dict(
|
|
os.environ, {"XDG_STATE_HOME": str(state_home)}, clear=False
|
|
):
|
|
first_root = default_release_run_root(first)
|
|
second_root = default_release_run_root(second)
|
|
default_app = create_app(workspace_root=first, token="token")
|
|
with TestClient(default_app) as client:
|
|
default_list = client.get(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
)
|
|
self.assertEqual(200, default_list.status_code)
|
|
self.assertEqual(0o700, first_root.stat().st_mode & 0o777)
|
|
self.assertNotEqual(first_root, second_root)
|
|
self.assertEqual("release-runs", first_root.name)
|
|
self.assertEqual(state_home, first_root.parents[3])
|
|
self.assertEqual(
|
|
f"workspace-{release_workspace_fingerprint(first)}",
|
|
first_root.parent.name,
|
|
)
|
|
self.assertFalse(first_root.is_relative_to(first))
|
|
|
|
local_workspace = parent / "local"
|
|
local_meta = local_workspace / "govoplan"
|
|
local_meta.mkdir(parents=True)
|
|
(local_meta / "repositories.json").write_text("{}", encoding="utf-8")
|
|
with patch.dict(
|
|
os.environ, {"XDG_STATE_HOME": str(state_home)}, clear=False
|
|
):
|
|
self.assertTrue(
|
|
default_release_run_root(local_workspace).is_relative_to(state_home)
|
|
)
|
|
|
|
explicit_root = parent / "explicit"
|
|
app = create_app(
|
|
workspace_root=first,
|
|
run_state_root=explicit_root,
|
|
token="token",
|
|
)
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
response = client.post(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
json=run_input(),
|
|
)
|
|
self.assertEqual(200, response.status_code)
|
|
self.assertEqual(
|
|
release_workspace_fingerprint(first),
|
|
response.json()["immutable"]["input"]["workspace_fingerprint"],
|
|
)
|
|
self.assertEqual(explicit_root.resolve(), app.state.release_runs.root)
|
|
|
|
def test_shared_explicit_root_does_not_cross_workspace_boundary(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
parent = Path(temp_dir)
|
|
root = parent / "shared-runs"
|
|
first = parent / "first"
|
|
second = parent / "second"
|
|
first.mkdir()
|
|
second.mkdir()
|
|
first_app = create_app(
|
|
workspace_root=first, run_state_root=root, token="token"
|
|
)
|
|
headers = {"X-Release-Console-Token": "token"}
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(first_app) as client,
|
|
):
|
|
created = client.post(
|
|
"/api/release-runs", headers=headers, json=run_input()
|
|
).json()
|
|
|
|
second_app = create_app(
|
|
workspace_root=second, run_state_root=root, token="token"
|
|
)
|
|
with TestClient(second_app) as client:
|
|
listed = client.get("/api/release-runs", headers=headers)
|
|
loaded = client.get(
|
|
f"/api/release-runs/{created['run_id']}", headers=headers
|
|
)
|
|
self.assertEqual([], listed.json()["runs"])
|
|
self.assertEqual(404, loaded.status_code)
|
|
|
|
def test_ui_and_runbook_state_tracking_boundary_are_explicit(self) -> None:
|
|
webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
|
|
runbook = (META_ROOT / "docs" / "RELEASE_CONSOLE.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
self.assertIn("Durable Run State", webui)
|
|
self.assertIn("State tracking only.", webui)
|
|
self.assertIn("Create Run from Selection", webui)
|
|
self.assertIn("Prepare Retry", webui)
|
|
self.assertIn("step.retry_available", webui)
|
|
self.assertIn("Record Reconciliation", webui)
|
|
self.assertIn("effect_succeeded", webui)
|
|
self.assertIn("renderReleaseRunStoreUnavailable", webui)
|
|
self.assertIn('const runsRequest = api("/api/release-runs")', webui)
|
|
self.assertIn("This foundation tracks state only.", runbook)
|
|
self.assertIn("effect_absent", runbook)
|
|
self.assertIn("$XDG_STATE_HOME", runbook)
|
|
self.assertIn("same-directory", runbook)
|
|
|
|
|
|
def run_input() -> dict[str, object]:
|
|
return {
|
|
"channel": "stable",
|
|
"repo_versions": {
|
|
"govoplan-files": "1.2.4",
|
|
"govoplan-core": "1.2.3",
|
|
},
|
|
"online": False,
|
|
"remote_tags": False,
|
|
"public_catalog": False,
|
|
"include_migrations": True,
|
|
"workspace_fingerprint": WORKSPACE_FINGERPRINT,
|
|
}
|
|
|
|
|
|
def release_plan() -> dict[str, object]:
|
|
return {
|
|
"generated_at": "2026-07-22T00:00:00Z",
|
|
"target_channel": "stable",
|
|
"status": "attention",
|
|
"units": [
|
|
{"repo": "govoplan-core", "target_version": "1.2.3"},
|
|
{"repo": "govoplan-files", "target_version": "1.2.4"},
|
|
],
|
|
"compatibility": [],
|
|
"gate_findings": [],
|
|
"recommended_action": {
|
|
"id": "preview_source_release",
|
|
"title": "Preview source release tags",
|
|
"detail": "Plan-visible gates pass.",
|
|
"remediation": "Preview before mutation.",
|
|
},
|
|
"source_preflight_ready": True,
|
|
"dry_run_steps": [
|
|
{
|
|
"id": "core:preflight",
|
|
"title": "Inspect Core",
|
|
"detail": "Run a read-only preflight.",
|
|
"command": "git status --short --branch",
|
|
"cwd": "/workspace/govoplan-core",
|
|
"mutating": False,
|
|
"repo": "govoplan-core",
|
|
"status": "planned",
|
|
},
|
|
{
|
|
"id": "core:tag",
|
|
"title": "Create Core tag",
|
|
"detail": "Create an annotated tag after confirmation.",
|
|
"command": "git tag -a v1.2.3",
|
|
"cwd": "/workspace/govoplan-core",
|
|
"mutating": True,
|
|
"repo": "govoplan-core",
|
|
"status": "planned",
|
|
},
|
|
],
|
|
"notes": [],
|
|
}
|
|
|
|
|
|
def mutating_first_plan() -> dict[str, object]:
|
|
plan = release_plan()
|
|
plan["dry_run_steps"] = [plan["dry_run_steps"][1]] # type: ignore[index]
|
|
return plan
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|