521 lines
20 KiB
Python
521 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
META_ROOT = Path(__file__).resolve().parents[1]
|
|
RELEASE_ROOT = META_ROOT / "tools" / "release"
|
|
if str(RELEASE_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(RELEASE_ROOT))
|
|
|
|
from govoplan_release.release_run import ( # noqa: E402
|
|
MAX_EVENTS,
|
|
ReleaseRunConflict,
|
|
ReleaseRunCorrupt,
|
|
ReleaseRunStore,
|
|
)
|
|
from server.app import ( # noqa: E402
|
|
create_app,
|
|
default_release_run_root,
|
|
release_workspace_fingerprint,
|
|
)
|
|
|
|
|
|
class ReleaseRunStoreTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temporary.name) / "release-runs"
|
|
self.store = ReleaseRunStore(self.root)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temporary.cleanup()
|
|
|
|
def test_create_persists_private_immutable_plan_and_ordered_steps(self) -> None:
|
|
run = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)
|
|
|
|
reopened = ReleaseRunStore(self.root).get(run["run_id"])
|
|
record_path = self.root / f"{run['run_id']}.json"
|
|
|
|
self.assertEqual("govoplan.release-run", reopened["schema"])
|
|
self.assertEqual(1, reopened["schema_version"])
|
|
self.assertEqual(
|
|
{"govoplan-core": "1.2.3", "govoplan-files": "1.2.4"},
|
|
reopened["immutable"]["input"]["repo_versions"],
|
|
)
|
|
self.assertEqual("planned", reopened["state"]["status"])
|
|
self.assertTrue(reopened["state"]["steps"][0]["available"])
|
|
self.assertFalse(reopened["state"]["steps"][1]["available"])
|
|
self.assertEqual("core:preflight", reopened["recommended_next"]["step_id"])
|
|
self.assertEqual(0o700, os.stat(self.root).st_mode & 0o777)
|
|
self.assertEqual(0o600, os.stat(record_path).st_mode & 0o777)
|
|
self.assertNotIn("processed_requests", reopened["state"])
|
|
self.assertNotIn("attempt_fingerprint", reopened["state"]["steps"][0])
|
|
|
|
def test_attempt_transitions_are_idempotent_and_strictly_ordered(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "Complete core:preflight"):
|
|
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
|
|
|
|
started = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="attempt-preflight-0001"
|
|
)
|
|
duplicate_start = self.store.start_step(
|
|
run_id, "core:preflight", attempt_id="attempt-preflight-0001"
|
|
)
|
|
self.assertEqual(1, started["state"]["steps"][0]["attempt_count"])
|
|
self.assertEqual(
|
|
len(started["state"]["events"]),
|
|
len(duplicate_start["state"]["events"]),
|
|
)
|
|
|
|
finished = self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="attempt-preflight-0001",
|
|
succeeded=True,
|
|
result_code="preflight_valid",
|
|
)
|
|
duplicate_finish = self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="attempt-preflight-0001",
|
|
succeeded=True,
|
|
result_code="preflight_valid",
|
|
)
|
|
self.assertEqual("succeeded", finished["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(finished["state"]["events"]),
|
|
len(duplicate_finish["state"]["events"]),
|
|
)
|
|
self.assertTrue(finished["state"]["steps"][1]["available"])
|
|
|
|
def test_reopen_and_resume_preserve_ambiguous_mutating_effect(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
|
|
)["run_id"]
|
|
self.store.start_step(run_id, "core:tag", attempt_id="attempt-tag-0001")
|
|
|
|
reopened_store = ReleaseRunStore(self.root)
|
|
before_resume = reopened_store.get(run_id)
|
|
self.assertEqual("running", before_resume["state"]["steps"][0]["state"])
|
|
|
|
resumed = reopened_store.resume(run_id, request_id="resume-request-0001")
|
|
duplicate = reopened_store.resume(run_id, request_id="resume-request-0001")
|
|
self.assertEqual("interrupted", resumed["state"]["steps"][0]["state"])
|
|
self.assertEqual("reconcile_step", resumed["recommended_next"]["id"])
|
|
self.assertFalse(resumed["recommended_next"]["available"])
|
|
self.assertEqual(
|
|
len(resumed["state"]["events"]), len(duplicate["state"]["events"])
|
|
)
|
|
|
|
with self.assertRaisesRegex(ReleaseRunConflict, "must be reconciled"):
|
|
reopened_store.retry_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="retry-request-0001",
|
|
)
|
|
retried = reopened_store.retry_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="retry-request-0002",
|
|
reconciled=True,
|
|
)
|
|
duplicate_retry = reopened_store.retry_step(
|
|
run_id,
|
|
"core:tag",
|
|
request_id="retry-request-0002",
|
|
reconciled=True,
|
|
)
|
|
self.assertEqual("pending", retried["state"]["steps"][0]["state"])
|
|
self.assertEqual(
|
|
len(retried["state"]["events"]),
|
|
len(duplicate_retry["state"]["events"]),
|
|
)
|
|
|
|
def test_tampered_record_fails_closed_and_is_not_rewritten(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["immutable"]["input"]["channel"] = "testing"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
tampered = path.read_bytes()
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
self.assertEqual(tampered, path.read_bytes())
|
|
self.assertEqual("unavailable", self.store.list()[0]["status"])
|
|
|
|
def test_broad_record_mode_and_symlinked_root_fail_closed(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
path.chmod(0o640)
|
|
with self.assertRaisesRegex(ReleaseRunCorrupt, "broader than 0600"):
|
|
self.store.get(run_id)
|
|
self.assertEqual(0o640, os.stat(path).st_mode & 0o777)
|
|
|
|
target = Path(self.temporary.name) / "target"
|
|
target.mkdir()
|
|
linked = Path(self.temporary.name) / "linked"
|
|
linked.symlink_to(target, target_is_directory=True)
|
|
with self.assertRaisesRegex(ReleaseRunCorrupt, "symbolic-link"):
|
|
ReleaseRunStore(linked).list()
|
|
|
|
def test_state_specific_fields_and_boolean_counts_fail_closed(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["attempt_count"] = True
|
|
payload["state"]["steps"][0]["started_at"] = "2026-07-22T00:00:00Z"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
def test_valid_looking_mutable_state_tampering_fails_record_checksum(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=mutating_first_plan()
|
|
)["run_id"]
|
|
self.store.start_step(run_id, "core:tag", attempt_id="checksum-attempt-0001")
|
|
self.store.finish_step(
|
|
run_id,
|
|
"core:tag",
|
|
attempt_id="checksum-attempt-0001",
|
|
succeeded=True,
|
|
result_code="tag_created",
|
|
)
|
|
path = self.root / f"{run_id}.json"
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["state"]["steps"][0]["result_code"] = "tag_published"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
tampered = path.read_bytes()
|
|
|
|
with self.assertRaises(ReleaseRunCorrupt):
|
|
self.store.get(run_id)
|
|
|
|
self.assertEqual(tampered, path.read_bytes())
|
|
|
|
def test_concurrent_claims_have_one_winner_and_no_lost_update(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
barrier = threading.Barrier(2)
|
|
|
|
def claim(attempt_id: str) -> str:
|
|
contender = ReleaseRunStore(self.root)
|
|
barrier.wait(timeout=5)
|
|
try:
|
|
contender.start_step(run_id, "core:preflight", attempt_id=attempt_id)
|
|
except ReleaseRunConflict:
|
|
return "conflict"
|
|
return "started"
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(
|
|
executor.map(
|
|
claim, ("concurrent-attempt-0001", "concurrent-attempt-0002")
|
|
)
|
|
)
|
|
|
|
reopened = self.store.get(run_id)
|
|
self.assertEqual(["conflict", "started"], sorted(results))
|
|
self.assertEqual(1, reopened["state"]["steps"][0]["attempt_count"])
|
|
self.assertEqual(
|
|
1,
|
|
sum(
|
|
event["type"] == "step_started" for event in reopened["state"]["events"]
|
|
),
|
|
)
|
|
|
|
def test_event_and_request_history_are_bounded_and_code_only(self) -> None:
|
|
run_id = self.store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
for index in range(90):
|
|
attempt_id = f"bounded-attempt-{index:04d}"
|
|
self.store.start_step(run_id, "core:preflight", attempt_id=attempt_id)
|
|
self.store.finish_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id=attempt_id,
|
|
succeeded=False,
|
|
result_code="preflight_failed",
|
|
)
|
|
self.store.retry_step(
|
|
run_id,
|
|
"core:preflight",
|
|
request_id=f"bounded-retry-{index:04d}",
|
|
)
|
|
|
|
view = self.store.get(run_id)
|
|
raw = json.loads((self.root / f"{run_id}.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(MAX_EVENTS, len(view["state"]["events"]))
|
|
self.assertLessEqual(len(raw["state"]["processed_requests"]), 128)
|
|
self.assertNotIn("bounded-retry-0089", json.dumps(raw))
|
|
for event in view["state"]["events"]:
|
|
self.assertLessEqual(
|
|
set(event), {"sequence", "at", "type", "step_id", "result_code"}
|
|
)
|
|
|
|
|
|
class ReleaseRunApiTests(unittest.TestCase):
|
|
def test_token_guarded_create_list_read_resume_and_retry(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir),
|
|
run_state_root=root,
|
|
token="release-token",
|
|
)
|
|
headers = {"X-Release-Console-Token": "release-token"}
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
unauthorized = client.get("/api/release-runs")
|
|
created_response = client.post(
|
|
"/api/release-runs",
|
|
headers=headers,
|
|
json=run_input(),
|
|
)
|
|
self.assertEqual(200, created_response.status_code)
|
|
created = created_response.json()
|
|
run_id = created["run_id"]
|
|
listed = client.get("/api/release-runs", headers=headers).json()
|
|
loaded = client.get(
|
|
f"/api/release-runs/{run_id}", headers=headers
|
|
).json()
|
|
|
|
app.state.release_runs.start_step(
|
|
run_id,
|
|
"core:preflight",
|
|
attempt_id="api-attempt-0001",
|
|
)
|
|
resumed = client.post(
|
|
f"/api/release-runs/{run_id}/resume",
|
|
headers=headers,
|
|
json={"request_id": "api-resume-0001"},
|
|
)
|
|
retried = client.post(
|
|
f"/api/release-runs/{run_id}/steps/core:preflight/retry",
|
|
headers=headers,
|
|
json={"request_id": "api-retry-0001"},
|
|
)
|
|
|
|
self.assertEqual(401, unauthorized.status_code)
|
|
self.assertEqual(run_id, listed["runs"][0]["run_id"])
|
|
self.assertEqual(run_id, loaded["run_id"])
|
|
self.assertEqual(
|
|
"interrupted", resumed.json()["state"]["steps"][0]["state"]
|
|
)
|
|
self.assertEqual(200, retried.status_code)
|
|
self.assertEqual("pending", retried.json()["state"]["steps"][0]["state"])
|
|
|
|
def test_create_rejects_unknown_or_unresolved_repository(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir), run_state_root=root, token="token"
|
|
)
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
response = client.post(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
json={
|
|
"channel": "stable",
|
|
"repo_versions": {"govoplan-unknown": "1.0.0"},
|
|
"public_catalog": False,
|
|
},
|
|
)
|
|
|
|
self.assertEqual(409, response.status_code)
|
|
self.assertIn("did not resolve exactly", response.json()["detail"])
|
|
self.assertEqual([], list(root.glob("*.json")))
|
|
|
|
def test_api_rejects_tampered_record_without_rewriting_it(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "runs"
|
|
store = ReleaseRunStore(root)
|
|
run_id = store.create(
|
|
input_snapshot=run_input(), plan_snapshot=release_plan()
|
|
)["run_id"]
|
|
path = root / f"{run_id}.json"
|
|
path.write_text("{}", encoding="utf-8")
|
|
malformed = path.read_bytes()
|
|
app = create_app(
|
|
workspace_root=Path(temp_dir), run_state_root=root, token="token"
|
|
)
|
|
with TestClient(app) as client:
|
|
response = client.get(
|
|
f"/api/release-runs/{run_id}",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
)
|
|
|
|
self.assertEqual(409, response.status_code)
|
|
self.assertEqual(malformed, path.read_bytes())
|
|
|
|
def test_default_storage_and_immutable_input_are_workspace_bound(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
parent = Path(temp_dir)
|
|
first = parent / "first"
|
|
second = parent / "second"
|
|
first.mkdir()
|
|
second.mkdir()
|
|
first_root = default_release_run_root(first)
|
|
second_root = default_release_run_root(second)
|
|
self.assertNotEqual(first_root, second_root)
|
|
self.assertIn(release_workspace_fingerprint(first)[:16], first_root.name)
|
|
|
|
local_workspace = parent / "local"
|
|
local_meta = local_workspace / "govoplan"
|
|
local_meta.mkdir(parents=True)
|
|
(local_meta / "repositories.json").write_text("{}", encoding="utf-8")
|
|
self.assertEqual(
|
|
local_meta / "runtime" / "release-runs",
|
|
default_release_run_root(local_workspace),
|
|
)
|
|
|
|
explicit_root = parent / "explicit"
|
|
app = create_app(
|
|
workspace_root=first,
|
|
run_state_root=explicit_root,
|
|
token="token",
|
|
)
|
|
with (
|
|
patch("server.app.build_dashboard", return_value=object()),
|
|
patch(
|
|
"server.app.build_selective_release_plan",
|
|
return_value=release_plan(),
|
|
),
|
|
TestClient(app) as client,
|
|
):
|
|
response = client.post(
|
|
"/api/release-runs",
|
|
headers={"X-Release-Console-Token": "token"},
|
|
json=run_input(),
|
|
)
|
|
self.assertEqual(200, response.status_code)
|
|
self.assertEqual(
|
|
release_workspace_fingerprint(first),
|
|
response.json()["immutable"]["input"]["workspace_fingerprint"],
|
|
)
|
|
self.assertEqual(explicit_root.resolve(), app.state.release_runs.root)
|
|
|
|
def test_ui_and_runbook_state_tracking_boundary_are_explicit(self) -> None:
|
|
webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
|
|
runbook = (META_ROOT / "docs" / "RELEASE_CONSOLE.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
self.assertIn("Durable Run State", webui)
|
|
self.assertIn("State tracking only.", webui)
|
|
self.assertIn("Create Run from Selection", webui)
|
|
self.assertIn("Prepare Retry", webui)
|
|
self.assertIn("step.retry_available", webui)
|
|
self.assertIn("This foundation tracks state only.", runbook)
|
|
self.assertIn("mutating step remains unavailable", runbook)
|
|
self.assertIn("same-directory temporary file", runbook)
|
|
|
|
|
|
def run_input() -> dict[str, object]:
|
|
return {
|
|
"channel": "stable",
|
|
"repo_versions": {
|
|
"govoplan-files": "1.2.4",
|
|
"govoplan-core": "1.2.3",
|
|
},
|
|
"online": False,
|
|
"remote_tags": False,
|
|
"public_catalog": False,
|
|
"include_migrations": True,
|
|
"workspace_fingerprint": "a" * 64,
|
|
}
|
|
|
|
|
|
def release_plan() -> dict[str, object]:
|
|
return {
|
|
"generated_at": "2026-07-22T00:00:00Z",
|
|
"target_channel": "stable",
|
|
"status": "attention",
|
|
"units": [
|
|
{"repo": "govoplan-core", "target_version": "1.2.3"},
|
|
{"repo": "govoplan-files", "target_version": "1.2.4"},
|
|
],
|
|
"compatibility": [],
|
|
"gate_findings": [],
|
|
"recommended_action": {
|
|
"id": "preview_source_release",
|
|
"title": "Preview source release tags",
|
|
"detail": "Plan-visible gates pass.",
|
|
"remediation": "Preview before mutation.",
|
|
},
|
|
"source_preflight_ready": True,
|
|
"dry_run_steps": [
|
|
{
|
|
"id": "core:preflight",
|
|
"title": "Inspect Core",
|
|
"detail": "Run a read-only preflight.",
|
|
"command": "git status --short --branch",
|
|
"cwd": "/workspace/govoplan-core",
|
|
"mutating": False,
|
|
"repo": "govoplan-core",
|
|
"status": "planned",
|
|
},
|
|
{
|
|
"id": "core:tag",
|
|
"title": "Create Core tag",
|
|
"detail": "Create an annotated tag after confirmation.",
|
|
"command": "git tag -a v1.2.3",
|
|
"cwd": "/workspace/govoplan-core",
|
|
"mutating": True,
|
|
"repo": "govoplan-core",
|
|
"status": "planned",
|
|
},
|
|
],
|
|
"notes": [],
|
|
}
|
|
|
|
|
|
def mutating_first_plan() -> dict[str, object]:
|
|
plan = release_plan()
|
|
plan["dry_run_steps"] = [plan["dry_run_steps"][1]] # type: ignore[index]
|
|
return plan
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|