GovOPlaN/govoplan
1
0
Fork 0
Code Issues 11 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(release): make durable runs recovery safe

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 19:30:12 +02:00
parent 19c4b63ade
commit 22f5c2ff4e
5 changed files with 881 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
tools/release/govoplan_release/release_run.py
View File

@@ -19,20 +19,21 @@ import stat
import tempfile
import threading
from typing import Any
import uuid
from filelock import FileLock, Timeout as FileLockTimeout
SCHEMA = "govoplan.release-run"
SCHEMA_VERSION = 2
SCHEMA_VERSION = 3
MAX_RECORD_BYTES = 2 * 1024 * 1024
MAX_PLAN_STEPS = 512
MAX_REPOSITORY_VERSIONS = 128
MAX_EVENTS = 256
MAX_COMMANDS = 2_048
MAX_LIST_LIMIT = 100
RUN_ID_PATTERN = re.compile(r"^rr-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$")
RUN_ID_PATTERN = re.compile(
r"^(?:rr-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}|rr-request-[0-9a-f]{64})$"
)
STEP_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,159}$")
REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{7,127}$")
CHANNEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
@@ -57,6 +58,7 @@ EVENT_TYPES = {
ATTEMPT_OUTCOMES = {"running", "succeeded", "failed", "interrupted"}
RECONCILIATION_OUTCOMES = {"effect_absent", "effect_succeeded", "unresolved"}
RECONCILIATION_CONFIRMATION = "RECONCILE"
START_RECOVERY_RESERVE = 2
class ReleaseRunError(RuntimeError):
@@ -97,9 +99,11 @@ class ReleaseRunStore:
def create(
self,
*,
request_id: str,
input_snapshot: dict[str, Any],
plan_snapshot: dict[str, Any],
) -> dict[str, Any]:
request_fingerprint = _request_fingerprint(request_id)
immutable_input = _validated_input(input_snapshot)
if (
immutable_input["workspace_fingerprint"]
@@ -110,9 +114,7 @@ class ReleaseRunStore:
)
immutable_plan = _validated_plan(plan_snapshot)
now = _timestamp()
run_id = (
f"rr-{now.replace('-', '').replace(':', '')[:15]}Z-{uuid.uuid4().hex[:12]}"
)
run_id = _creation_run_id(request_fingerprint)
immutable = {
"input": immutable_input,
"plan": immutable_plan,
@@ -134,6 +136,7 @@ class ReleaseRunStore:
"schema": SCHEMA,
"schema_version": SCHEMA_VERSION,
"run_id": run_id,
"create_request_fingerprint": request_fingerprint,
"created_at": now,
"updated_at": now,
"immutable": immutable,
@@ -152,9 +155,42 @@ class ReleaseRunStore:
record["record_digest"] = _record_digest(record)
with self._locked():
self._ensure_root()
existing = self._read_optional(run_id)
if existing is not None:
_validate_create_replay(
existing,
request_fingerprint=request_fingerprint,
input_snapshot=immutable_input,
)
return _view(existing)
self._write_new(record)
return _view(record)
def find_created_run(
self, *, request_id: str, input_snapshot: dict[str, Any]
) -> dict[str, Any] | None:
"""Return an exact prior create command before rebuilding its plan."""
request_fingerprint = _request_fingerprint(request_id)
immutable_input = _validated_input(input_snapshot)
if (
immutable_input["workspace_fingerprint"]
!= self.expected_workspace_fingerprint
):
raise ReleaseRunConflict(
"Release run input belongs to a different workspace."
)
with self._locked():
record = self._read_optional(_creation_run_id(request_fingerprint))
if record is None:
return None
_validate_create_replay(
record,
request_fingerprint=request_fingerprint,
input_snapshot=immutable_input,
)
return _view(record)
def get(self, run_id: str) -> dict[str, Any]:
with self._locked():
record = self._read(run_id)
@@ -214,10 +250,17 @@ class ReleaseRunStore:
record = self._read(run_id)
if _request_seen(record, request_fingerprint, "resume", None, None):
return _view(record)
running_steps = [
step
for step in record["state"]["steps"]
if step["state"] == "running"
]
if not running_steps:
raise ReleaseRunConflict(
"A release run can only be resumed while a step is running."
)
_ensure_command_capacity(record)
for step in record["state"]["steps"]:
if step["state"] != "running":
continue
for step in running_steps:
attempt = _attempt_for_fingerprint(
record, step["attempt_fingerprint"], step_id=step["id"]
)
@@ -232,7 +275,14 @@ class ReleaseRunStore:
step_id=step["id"],
result_code="process_interrupted",
)
_remember_request(record, request_fingerprint, "resume", None, None)
_remember_request(
record,
request_fingerprint,
"resume",
None,
None,
attempt_fingerprint=running_steps[0]["attempt_fingerprint"],
)
_append_event(record, event_type="run_resumed")
self._persist_update(record)
return _view(record)
@@ -260,6 +310,7 @@ class ReleaseRunStore:
"Record the observed effect of an interrupted mutating step before retry."
)
_ensure_command_capacity(record)
attempt_fingerprint = step["attempt_fingerprint"]
retry_reason = (
"known_failure"
if step["state"] == "failed"
@@ -274,7 +325,14 @@ class ReleaseRunStore:
"result_code": None,
}
)
_remember_request(record, request_fingerprint, "retry", step_id, None)
_remember_request(
record,
request_fingerprint,
"retry",
step_id,
None,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
record,
event_type="step_retry_requested",
@@ -316,7 +374,20 @@ class ReleaseRunStore:
raise ReleaseRunConflict(
"Only an interrupted mutating step can be reconciled."
)
_ensure_command_capacity(record)
attempt_fingerprint = step["attempt_fingerprint"]
if outcome == "unresolved":
if any(
request["operation"] == "reconcile"
and request["parameter"] == "unresolved"
and request["attempt_fingerprint"] == attempt_fingerprint
for request in record["state"]["processed_requests"]
):
raise ReleaseRunConflict(
"An unresolved outcome is already recorded for this attempt."
)
_ensure_command_capacity(record, reserve_after=1)
else:
_ensure_command_capacity(record)
if outcome == "effect_absent":
step.update(
{
@@ -336,7 +407,12 @@ class ReleaseRunStore:
}
)
_remember_request(
record, request_fingerprint, "reconcile", step_id, outcome
record,
request_fingerprint,
"reconcile",
step_id,
outcome,
attempt_fingerprint=attempt_fingerprint,
)
_append_event(
record,
@@ -374,6 +450,7 @@ class ReleaseRunStore:
available, reason = _step_available(record, step_id)
if not available:
raise ReleaseRunConflict(reason)
_ensure_start_recovery_capacity(record)
_ensure_attempt_capacity(record)
step.update(
{
@@ -506,11 +583,22 @@ class ReleaseRunStore:
raise ReleaseRunCorrupt("Release run record is unreadable.") from exc
finally:
if descriptor >= 0:
os.close(descriptor)
try:
os.close(descriptor)
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run record could not be closed safely."
) from exc
_validate_record(payload, expected_run_id=run_id)
self._assert_workspace(payload)
return payload
def _read_optional(self, run_id: str) -> dict[str, Any] | None:
try:
return self._read(run_id)
except ReleaseRunNotFound:
return None
def _assert_workspace(self, record: dict[str, Any]) -> None:
if (
record["immutable"]["input"]["workspace_fingerprint"]
@@ -577,19 +665,48 @@ class ReleaseRunStore:
"Release run record permissions could not be secured."
)
_fsync_directory(self.root)
except ReleaseRunError:
raise
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run record could not be persisted safely."
) from exc
finally:
if descriptor >= 0:
os.close(descriptor)
try:
os.close(descriptor)
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run record could not be persisted safely."
) from exc
if temporary_path is not None:
try:
os.unlink(temporary_path)
except FileNotFoundError:
except OSError:
pass
def _locked(self): # type: ignore[no-untyped-def]
self._ensure_root()
self._validate_lock_path()
return _CombinedLock(self._thread_lock, FileLock(str(self._lock_path)))
def _validate_lock_path(self) -> None:
try:
metadata = self._lock_path.lstat()
except FileNotFoundError:
return
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run lock could not be inspected safely."
) from exc
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink != 1
or (hasattr(os, "geteuid") and metadata.st_uid != os.geteuid())
):
raise ReleaseRunCorrupt("Release run lock path is unsafe.")
class _CombinedLock:
def __init__(self, thread_lock: threading.RLock, file_lock: FileLock) -> None:
@@ -605,13 +722,23 @@ class _CombinedLock:
raise ReleaseRunConflict(
"Release run storage is busy; retry the operation."
) from exc
except OSError as exc:
self.thread_lock.release()
raise ReleaseRunCorrupt(
"Release run lock could not be acquired safely."
) from exc
except BaseException:
self.thread_lock.release()
raise
def __exit__(self, *exc_info: object) -> None:
try:
self.file_lock.release()
try:
self.file_lock.release()
except OSError as exc:
raise ReleaseRunCorrupt(
"Release run lock could not be released safely."
) from exc
finally:
self.thread_lock.release()
@@ -755,6 +882,7 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
"schema",
"schema_version",
"run_id",
"create_request_fingerprint",
"created_at",
"updated_at",
"immutable",
@@ -773,6 +901,10 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
raise ValueError
if expected_run_id is not None and run_id != expected_run_id:
raise ValueError
if not _is_sha256(value.get("create_request_fingerprint")):
raise ValueError
if run_id != _creation_run_id(value["create_request_fingerprint"]):
raise ValueError
for key in ("created_at", "updated_at"):
if not _is_timestamp(value.get(key)):
raise ValueError
@@ -908,8 +1040,15 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
if (
not isinstance(request, dict)
or set(request)
!= {"fingerprint", "operation", "step_id", "parameter"}
!= {
"fingerprint",
"operation",
"step_id",
"parameter",
"attempt_fingerprint",
}
or not _is_sha256(request.get("fingerprint"))
or not _is_sha256(request.get("attempt_fingerprint"))
or request.get("operation")
not in {"resume", "retry", "reconcile"}
):
@@ -961,6 +1100,29 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
attempts_by_fingerprint = {
attempt["fingerprint"]: attempt for attempt in attempts
}
for request in requests:
request_attempt = attempts_by_fingerprint.get(
request["attempt_fingerprint"]
)
if request_attempt is None:
raise ValueError
if (
request["step_id"] is not None
and request_attempt["step_id"] != request["step_id"]
):
raise ValueError
if request["operation"] == "resume" and request_attempt[
"outcome"
] != "interrupted":
raise ValueError
if request["operation"] == "retry" and request_attempt[
"outcome"
] not in {"failed", "interrupted"}:
raise ValueError
if request["operation"] == "reconcile" and request_attempt[
"outcome"
] != "interrupted":
raise ValueError
attempts_per_step = {
step_id: sum(
attempt["step_id"] == step_id for attempt in attempts
@@ -1193,13 +1355,28 @@ def _append_event(
record["state"]["events"] = record["state"]["events"][-MAX_EVENTS:]
def _ensure_command_capacity(record: dict[str, Any]) -> None:
if len(record["state"]["processed_requests"]) >= MAX_COMMANDS:
def _ensure_command_capacity(
record: dict[str, Any], *, reserve_after: int = 0
) -> None:
if (
len(record["state"]["processed_requests"]) + 1 + reserve_after
> MAX_COMMANDS
):
raise ReleaseRunConflict(
"Release run command history reached its safety limit; create a new run."
)
def _ensure_start_recovery_capacity(record: dict[str, Any]) -> None:
if (
len(record["state"]["processed_requests"]) + START_RECOVERY_RESERVE
> MAX_COMMANDS
):
raise ReleaseRunConflict(
"Release run has insufficient reserved recovery capacity for a new attempt; create a new run."
)
def _ensure_attempt_capacity(record: dict[str, Any]) -> None:
if len(record["state"]["processed_attempts"]) >= MAX_COMMANDS:
raise ReleaseRunConflict(
@@ -1213,6 +1390,8 @@ def _remember_request(
operation: str,
step_id: str | None,
parameter: str | None,
*,
attempt_fingerprint: str | None,
) -> None:
_ensure_command_capacity(record)
record["state"]["processed_requests"].append(
@@ -1221,6 +1400,7 @@ def _remember_request(
"operation": operation,
"step_id": step_id,
"parameter": parameter,
"attempt_fingerprint": attempt_fingerprint,
}
)
@@ -1324,6 +1504,28 @@ def _record_digest(record: dict[str, Any]) -> str:
).hexdigest()
def _creation_run_id(request_fingerprint: str) -> str:
if not _is_sha256(request_fingerprint):
raise ReleaseRunConflict("Run creation request identifier is invalid.")
return f"rr-request-{request_fingerprint}"
def _validate_create_replay(
record: dict[str, Any],
*,
request_fingerprint: str,
input_snapshot: dict[str, Any],
) -> None:
if record["create_request_fingerprint"] != request_fingerprint:
raise ReleaseRunConflict(
"The run creation identifier is already mapped to another record."
)
if record["immutable"]["input"] != input_snapshot:
raise ReleaseRunConflict(
"The run creation identifier was already used with different inputs."
)
def _request_fingerprint(request_id: str) -> str:
if not isinstance(request_id, str) or not REQUEST_ID_PATTERN.fullmatch(request_id):
raise ReleaseRunConflict("Run-state request identifier is invalid.")