GovOPlaN/govoplan
1
0
Fork 0
Code Issues 10 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Execute receipt-bound durable release runs

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 21:37:18 +02:00
parent 7ecf1f17b0
commit 5381e37a9e
7 changed files with 1961 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

415
tools/release/govoplan_release/release_run.py
View File

@@ -1,9 +1,8 @@
"""Durable state for guided, explicitly executed release runs.
"""Durable state and bounded receipts for explicitly executed release runs.
The run store records an immutable selective-plan snapshot and a small mutable
state machine. It does not execute release commands. Existing release
executors remain responsible for their narrow confirmations and will be wired
to this state in a later slice.
The store freezes a selective plan, claims each supported attempt before its
executor is called, and records only bounded outcomes. Narrow executors remain
separate from this persistence module.
"""
from __future__ import annotations
@@ -11,6 +10,7 @@ from __future__ import annotations
import base64
import binascii
from copy import deepcopy
from dataclasses import dataclass
from datetime import UTC, datetime
import hashlib
import json
@@ -24,9 +24,12 @@ from typing import Any
from filelock import FileLock, Timeout as FileLockTimeout
from .candidate_artifact import CandidateArtifactError, validate_release_channel
SCHEMA = "govoplan.release-run"
SCHEMA_VERSION = 3
SCHEMA_VERSION = 4
LEGACY_SCHEMA_VERSIONS = {3}
MAX_RECORD_BYTES = 2 * 1024 * 1024
MAX_PLAN_STEPS = 512
MAX_REPOSITORY_VERSIONS = 128
@@ -40,7 +43,6 @@ RUN_ID_PATTERN = re.compile(
)
STEP_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,159}$")
REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{7,127}$")
CHANNEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
WORKSPACE_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$")
VERSION_PATTERN = re.compile(
@@ -65,6 +67,13 @@ RECONCILIATION_CONFIRMATION = "RECONCILE"
START_RECOVERY_RESERVE = 2
PROJECTION_TIMESTAMP = "9999-12-31T23:59:59Z"
PROJECTION_RESULT_CODE = "r" * 64
RECEIPT_KINDS = {
"catalog_candidate",
"catalog_publication",
"repository_state",
}
RECEIPT_ID_PATTERN = re.compile(r"^candidate-[0-9a-f]{32}$")
GIT_OBJECT_PATTERN = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
class ReleaseRunError(RuntimeError):
@@ -87,6 +96,17 @@ class ReleaseRunCorrupt(ReleaseRunError):
"""Raised when a persisted record fails its integrity or schema checks."""
@dataclass(frozen=True, slots=True)
class ReleaseStepClaim:
"""Private executor claim result; attempt identifiers remain fingerprinted."""
run: dict[str, Any]
claimed: bool
outcome: str
result_code: str | None
result_receipt: dict[str, Any] | None
class ReleaseRunStore:
"""Atomic, bounded local JSON store for release-run state."""
@@ -135,6 +155,7 @@ class ReleaseRunStore:
"started_at": None,
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
for item in immutable_plan["dry_run_steps"]
]
@@ -251,7 +272,7 @@ class ReleaseRunStore:
(
(
1,
view["updated_at"],
view["created_at"],
view["created_at"],
view["run_id"],
),
@@ -364,6 +385,7 @@ class ReleaseRunStore:
"started_at": None,
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
)
_remember_request(
@@ -391,6 +413,7 @@ class ReleaseRunStore:
request_id: str,
outcome: str,
confirmation: str,
result_receipt: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Record an operator-observed outcome for an interrupted mutation."""
@@ -401,12 +424,22 @@ class ReleaseRunStore:
raise ReleaseRunConflict(
f"Type {RECONCILIATION_CONFIRMATION} to record reconciliation."
)
receipt = _validated_result_receipt(result_receipt)
if receipt is not None and outcome != "effect_succeeded":
raise ReleaseRunConflict(
"Only a reconciled successful effect can carry a result receipt."
)
request_fingerprint = _request_fingerprint(request_id)
with self._locked():
record = self._read(run_id)
if _request_seen(
record, request_fingerprint, "reconcile", step_id, outcome
):
step, _plan_step = _step_pair(record, step_id)
if step.get("result_receipt") != receipt:
raise ReleaseRunConflict(
"The reconciliation request already has a different result receipt."
)
return _view(record)
step, plan_step = _step_pair(record, step_id)
if step["state"] != "interrupted" or not bool(
@@ -438,6 +471,7 @@ class ReleaseRunStore:
"started_at": None,
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
)
elif outcome == "effect_succeeded":
@@ -446,6 +480,7 @@ class ReleaseRunStore:
"state": "succeeded",
"finished_at": _timestamp(),
"result_code": "reconciled_effect_succeeded",
"result_receipt": receipt,
}
)
_remember_request(
@@ -472,10 +507,20 @@ class ReleaseRunStore:
*,
attempt_id: str,
) -> dict[str, Any]:
"""Claim one step for a future confirmed executor.
"""Claim one step while preserving the original store API."""
This is deliberately not an HTTP endpoint in the foundation slice.
"""
return self.claim_step(
run_id, step_id, attempt_id=attempt_id
).run
def claim_step(
self,
run_id: str,
step_id: str,
*,
attempt_id: str,
) -> ReleaseStepClaim:
"""Durably claim a step and distinguish a delayed attempt replay."""
_validate_step_id(step_id)
attempt_fingerprint = _request_fingerprint(attempt_id)
@@ -487,7 +532,14 @@ class ReleaseRunStore:
raise ReleaseRunConflict(
"The attempt identifier was already used for another release step."
)
return _view(record)
step, _plan_step = _step_pair(record, step_id)
return ReleaseStepClaim(
run=_view(record),
claimed=False,
outcome=existing_attempt["outcome"],
result_code=existing_attempt["result_code"],
result_receipt=deepcopy(step.get("result_receipt")),
)
step, plan_step = _step_pair(record, step_id)
available, reason = _step_available(record, step_id)
if not available:
@@ -502,6 +554,7 @@ class ReleaseRunStore:
"started_at": _timestamp(),
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
)
record["state"]["processed_attempts"].append(
@@ -519,7 +572,13 @@ class ReleaseRunStore:
mutating=bool(plan_step.get("mutating")),
)
self._persist_update(record)
return _view(record)
return ReleaseStepClaim(
run=_view(record),
claimed=True,
outcome="running",
result_code=None,
result_receipt=None,
)
def finish_step(
self,
@@ -529,12 +588,16 @@ class ReleaseRunStore:
attempt_id: str,
succeeded: bool,
result_code: str,
result_receipt: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Record a future executor outcome without persisting its output."""
"""Record a bounded executor outcome without persisting process output."""
_validate_step_id(step_id)
attempt_fingerprint = _request_fingerprint(attempt_id)
result_code = _safe_code(result_code)
receipt = _validated_result_receipt(result_receipt)
if receipt is not None and not succeeded:
raise ReleaseRunConflict("A failed release step cannot carry a result receipt.")
target_state = "succeeded" if succeeded else "failed"
with self._locked():
record = self._read(run_id)
@@ -548,6 +611,7 @@ class ReleaseRunStore:
if (
attempt["outcome"] == target_state
and attempt["result_code"] == result_code
and step.get("result_receipt") == receipt
):
return _view(record)
raise ReleaseRunConflict(
@@ -565,6 +629,7 @@ class ReleaseRunStore:
"state": target_state,
"finished_at": _timestamp(),
"result_code": result_code,
"result_receipt": receipt,
}
)
attempt["outcome"] = target_state
@@ -578,6 +643,76 @@ class ReleaseRunStore:
self._persist_update(record)
return _view(record)
def interrupt_step(
self,
run_id: str,
step_id: str,
*,
attempt_id: str,
result_code: str = "executor_outcome_unknown",
) -> dict[str, Any]:
"""Freeze an executor exception as an ambiguous, reconcilable attempt."""
_validate_step_id(step_id)
attempt_fingerprint = _request_fingerprint(attempt_id)
result_code = _safe_code(result_code)
if result_code != "executor_outcome_unknown":
raise ReleaseRunConflict("Executor interruption result code is invalid.")
with self._locked():
record = self._read(run_id)
step, _plan_step = _step_pair(record, step_id)
attempt = _find_attempt(record, attempt_fingerprint)
if attempt is None or attempt["step_id"] != step_id:
raise ReleaseRunConflict(
"The release attempt identifier was not claimed for this step."
)
if attempt["outcome"] == "interrupted":
if attempt["result_code"] == result_code:
return _view(record)
raise ReleaseRunConflict(
"The release attempt already has a different interruption outcome."
)
if (
attempt["outcome"] != "running"
or step["state"] != "running"
or step["attempt_fingerprint"] != attempt_fingerprint
):
raise ReleaseRunConflict(
"The release step is not running with this attempt identifier."
)
step.update(
{
"state": "interrupted",
"finished_at": _timestamp(),
"result_code": result_code,
"result_receipt": None,
}
)
attempt["outcome"] = "interrupted"
attempt["result_code"] = result_code
_append_event(
record,
event_type="step_interrupted",
step_id=step_id,
result_code=result_code,
)
self._persist_update(record)
return _view(record)
def current_attempt_fingerprint(self, run_id: str, step_id: str) -> str:
"""Return the opaque current fingerprint for server-side artifact recovery."""
_validate_step_id(step_id)
with self._locked():
record = self._read(run_id)
step, _plan_step = _step_pair(record, step_id)
fingerprint = step.get("attempt_fingerprint")
if not isinstance(fingerprint, str):
raise ReleaseRunConflict(
"Release step has no current attempt to reconcile."
)
return fingerprint
def _persist_update(self, record: dict[str, Any]) -> None:
self._assert_workspace(record)
record["updated_at"] = _timestamp()
@@ -638,6 +773,9 @@ class ReleaseRunStore:
) from exc
_validate_record(payload, expected_run_id=run_id)
self._assert_workspace(payload)
if payload["schema_version"] in LEGACY_SCHEMA_VERSIONS:
payload = _upgrade_record(payload)
self._atomic_write(path, payload)
return payload
def _read_optional(self, run_id: str) -> dict[str, Any] | None:
@@ -888,7 +1026,9 @@ def _validated_input(value: dict[str, Any]) -> dict[str, Any]:
raise ReleaseRunConflict("Release run input fields are invalid.")
channel = value.get("channel")
repo_versions = value.get("repo_versions")
if not isinstance(channel, str) or not CHANNEL_PATTERN.fullmatch(channel):
try:
channel = validate_release_channel(channel)
except CandidateArtifactError:
raise ReleaseRunConflict("Release channel is invalid.")
if (
not isinstance(repo_versions, dict)
@@ -924,7 +1064,7 @@ def _validated_plan(value: dict[str, Any]) -> dict[str, Any]:
if not isinstance(value, dict):
raise ReleaseRunConflict("Release plan snapshot must be an object.")
copied = deepcopy(value)
if set(copied) != {
base_fields = {
"generated_at",
"target_channel",
"status",
@@ -935,15 +1075,32 @@ def _validated_plan(value: dict[str, Any]) -> dict[str, Any]:
"gate_findings",
"recommended_action",
"source_preflight_ready",
}:
}
if set(copied) not in (base_fields, base_fields | {"runtime_binding"}):
raise ReleaseRunConflict("Release plan snapshot fields are invalid.")
if "runtime_binding" in copied:
runtime_binding = copied["runtime_binding"]
validated_runtime = _validated_result_receipt(runtime_binding)
if (
validated_runtime != runtime_binding
or runtime_binding.get("kind") != "repository_state"
or runtime_binding.get("repo") != "govoplan"
or runtime_binding.get("target_tag") != ""
or runtime_binding.get("tag_object") is not None
or runtime_binding.get("worktree_clean") is not True
):
raise ReleaseRunConflict(
"Release plan runtime binding is invalid."
)
if not _is_timestamp(copied.get("generated_at")):
raise ReleaseRunConflict("Release plan timestamp is invalid.")
if copied.get("status") not in PLAN_STATUSES:
raise ReleaseRunConflict("Release plan status is invalid.")
if not isinstance(
copied.get("target_channel"), str
) or not CHANNEL_PATTERN.fullmatch(copied["target_channel"]):
try:
target_channel = validate_release_channel(copied.get("target_channel"))
except CandidateArtifactError:
raise ReleaseRunConflict("Release plan channel is invalid.")
if target_channel != copied["target_channel"]:
raise ReleaseRunConflict("Release plan channel is invalid.")
if not isinstance(copied.get("units"), list):
raise ReleaseRunConflict("Release plan units are invalid.")
@@ -969,7 +1126,7 @@ def _validated_plan(value: dict[str, Any]) -> dict[str, Any]:
raise ReleaseRunConflict("Release plan step collection is invalid.")
seen: set[str] = set()
for step in steps:
if not isinstance(step, dict) or set(step) != {
if not isinstance(step, dict) or set(step) not in ({
"id",
"title",
"detail",
@@ -978,7 +1135,17 @@ def _validated_plan(value: dict[str, Any]) -> dict[str, Any]:
"mutating",
"repo",
"status",
}:
}, {
"id",
"title",
"detail",
"command",
"cwd",
"mutating",
"repo",
"status",
"source_binding",
}):
raise ReleaseRunConflict("Release plan step is invalid.")
step_id = step.get("id")
_validate_step_id(step_id)
@@ -997,6 +1164,21 @@ def _validated_plan(value: dict[str, Any]) -> dict[str, Any]:
not isinstance(step[field], str) or len(step[field]) > 8_192
):
raise ReleaseRunConflict("Release plan step metadata is invalid.")
if "source_binding" in step:
binding = step["source_binding"]
if binding is not None:
validated_binding = _validated_result_receipt(binding)
if (
validated_binding != binding
or binding.get("kind") != "repository_state"
or (
step.get("repo") is not None
and binding.get("repo") != step.get("repo")
)
):
raise ReleaseRunConflict(
"Release plan source binding is invalid."
)
encoded = _canonical_json(copied)
if len(encoded) > MAX_RECORD_BYTES:
raise ReleaseRunConflict("Release plan snapshot exceeds its size limit.")
@@ -1022,9 +1204,11 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
if (
value.get("schema") != SCHEMA
or type(value.get("schema_version")) is not int
or value["schema_version"] != SCHEMA_VERSION
or value["schema_version"]
not in {SCHEMA_VERSION, *LEGACY_SCHEMA_VERSIONS}
):
raise ValueError
schema_version = value["schema_version"]
run_id = value.get("run_id")
if not isinstance(run_id, str) or not RUN_ID_PATTERN.fullmatch(run_id):
raise ValueError
@@ -1084,18 +1268,20 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
if not isinstance(steps, list) or len(steps) != len(plan_steps):
raise ValueError
for step, plan_step in zip(steps, plan_steps, strict=True):
expected_step_fields = {
"id",
"state",
"attempt_count",
"attempt_fingerprint",
"started_at",
"finished_at",
"result_code",
}
if schema_version >= 4:
expected_step_fields.add("result_receipt")
if (
not isinstance(step, dict)
or set(step)
!= {
"id",
"state",
"attempt_count",
"attempt_fingerprint",
"started_at",
"finished_at",
"result_code",
}
or set(step) != expected_step_fields
or step.get("id") != plan_step["id"]
):
raise ValueError
@@ -1114,6 +1300,9 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
result_code = step.get("result_code")
if result_code is not None:
_safe_code(result_code)
receipt = _validated_result_receipt(step.get("result_receipt"))
if receipt != step.get("result_receipt"):
raise ValueError
_validate_step_state_fields(step)
for timestamp in (step.get("started_at"), step.get("finished_at")):
if timestamp is not None and not (
@@ -1221,7 +1410,8 @@ def _validate_record(value: Any, *, expected_run_id: str | None = None) -> None:
_safe_code(attempt_result)
if (
attempt["outcome"] == "interrupted"
and attempt_result != "process_interrupted"
and attempt_result
not in {"process_interrupted", "executor_outcome_unknown"}
):
raise ValueError
if len({attempt["fingerprint"] for attempt in attempts}) != len(attempts):
@@ -1550,6 +1740,7 @@ def _project_finish(
"state": target_state,
"finished_at": PROJECTION_TIMESTAMP,
"result_code": PROJECTION_RESULT_CODE,
"result_receipt": _projection_receipt() if succeeded else None,
}
)
attempt["outcome"] = target_state
@@ -1565,6 +1756,22 @@ def _project_finish(
return projected
def _projection_receipt() -> dict[str, Any]:
return {
"kind": "catalog_publication",
"candidate_id": "candidate-" + "c" * 32,
"catalog_sha256": "c" * 64,
"keyring_sha256": "c" * 64,
"publication_commit_sha": "c" * 64,
"publication_tag_object_sha": "c" * 64,
"publication_tag_commit_sha": "c" * 64,
"branch": "b" * 255,
"tag_name": "t" * 160,
"remote": "origin",
"remote_sha256": "c" * 64,
}
def _project_resume(record: dict[str, Any]) -> dict[str, Any]:
projected = deepcopy(record)
running_steps = [
@@ -1615,6 +1822,7 @@ def _project_retry(record: dict[str, Any], step_id: str) -> dict[str, Any]:
"started_at": None,
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
)
_remember_request(
@@ -1650,6 +1858,7 @@ def _project_reconcile(
"started_at": None,
"finished_at": None,
"result_code": None,
"result_receipt": None,
}
)
elif outcome == "effect_succeeded":
@@ -1658,6 +1867,7 @@ def _project_reconcile(
"state": "succeeded",
"finished_at": PROJECTION_TIMESTAMP,
"result_code": "reconciled_effect_succeeded",
"result_receipt": _projection_receipt(),
}
)
_remember_request(
@@ -1944,6 +2154,118 @@ def _safe_code(value: str) -> str:
return value
def _validated_result_receipt(
value: dict[str, Any] | None,
) -> dict[str, Any] | None:
if value is None:
return None
if not isinstance(value, dict) or value.get("kind") not in RECEIPT_KINDS:
raise ReleaseRunConflict("Release result receipt is invalid.")
if value["kind"] == "catalog_candidate":
if set(value) != {"kind", "candidate_id", "catalog_sha256"}:
raise ReleaseRunConflict("Release result receipt is invalid.")
candidate_id = value.get("candidate_id")
digest = value.get("catalog_sha256")
if (
not isinstance(candidate_id, str)
or not RECEIPT_ID_PATTERN.fullmatch(candidate_id)
or not _is_sha256(digest)
):
raise ReleaseRunConflict("Release result receipt identity is invalid.")
return dict(value)
if value["kind"] == "catalog_publication":
if set(value) != {
"kind",
"candidate_id",
"catalog_sha256",
"keyring_sha256",
"publication_commit_sha",
"publication_tag_object_sha",
"publication_tag_commit_sha",
"branch",
"tag_name",
"remote",
"remote_sha256",
}:
raise ReleaseRunConflict("Catalog publication receipt is invalid.")
candidate_id = value.get("candidate_id")
commit = value.get("publication_commit_sha")
tagged_commit = value.get("publication_tag_commit_sha")
if (
not isinstance(candidate_id, str)
or not RECEIPT_ID_PATTERN.fullmatch(candidate_id)
or not _is_sha256(value.get("catalog_sha256"))
or not _is_sha256(value.get("keyring_sha256"))
or not isinstance(commit, str)
or not GIT_OBJECT_PATTERN.fullmatch(commit)
or not isinstance(value.get("publication_tag_object_sha"), str)
or not GIT_OBJECT_PATTERN.fullmatch(
value["publication_tag_object_sha"]
)
or not isinstance(tagged_commit, str)
or not GIT_OBJECT_PATTERN.fullmatch(tagged_commit)
or tagged_commit != commit
or not isinstance(value.get("branch"), str)
or not value["branch"]
or len(value["branch"]) > 255
or not isinstance(value.get("tag_name"), str)
or not value["tag_name"]
or len(value["tag_name"]) > 160
or value.get("remote") != "origin"
or not _is_sha256(value.get("remote_sha256"))
):
raise ReleaseRunConflict(
"Catalog publication receipt identity is invalid."
)
return dict(value)
if set(value) != {
"kind",
"repo",
"head",
"branch",
"remote",
"remote_sha256",
"worktree_clean",
"target_tag",
"tag_object",
}:
raise ReleaseRunConflict("Repository result receipt is invalid.")
if (
not isinstance(value.get("repo"), str)
or not REPOSITORY_PATTERN.fullmatch(value["repo"])
or not isinstance(value.get("head"), str)
or not GIT_OBJECT_PATTERN.fullmatch(value["head"])
or not isinstance(value.get("branch"), str)
or not value["branch"]
or len(value["branch"]) > 255
or value.get("remote") != "origin"
or not _is_sha256(value.get("remote_sha256"))
or type(value.get("worktree_clean")) is not bool
or not isinstance(value.get("target_tag"), str)
or len(value["target_tag"]) > 160
or (
value.get("tag_object") is not None
and (
not isinstance(value["tag_object"], str)
or not GIT_OBJECT_PATTERN.fullmatch(value["tag_object"])
)
)
):
raise ReleaseRunConflict("Repository result receipt identity is invalid.")
return dict(value)
def _upgrade_record(record: dict[str, Any]) -> dict[str, Any]:
upgraded = deepcopy(record)
if upgraded.get("schema_version") == 3:
for step in upgraded["state"]["steps"]:
step["result_receipt"] = None
upgraded["schema_version"] = SCHEMA_VERSION
upgraded["record_digest"] = _record_digest(upgraded)
_validate_record(upgraded)
return upgraded
def _is_sha256(value: Any) -> bool:
return isinstance(value, str) and bool(re.fullmatch(r"[0-9a-f]{64}", value))
@@ -1967,6 +2289,7 @@ def _validate_step_state_fields(step: dict[str, Any]) -> None:
started_at = step["started_at"]
finished_at = step["finished_at"]
result_code = step["result_code"]
result_receipt = step.get("result_receipt")
if started_at is not None and not _is_timestamp(started_at):
raise ValueError
if finished_at is not None and not _is_timestamp(finished_at):
@@ -1976,19 +2299,34 @@ def _validate_step_state_fields(step: dict[str, Any]) -> None:
if state == "pending":
if any(
value is not None
for value in (fingerprint, started_at, finished_at, result_code)
for value in (
fingerprint,
started_at,
finished_at,
result_code,
result_receipt,
)
):
raise ValueError
return
if attempt_count < 1 or fingerprint is None or started_at is None:
raise ValueError
if state == "running":
if finished_at is not None or result_code is not None:
if (
finished_at is not None
or result_code is not None
or result_receipt is not None
):
raise ValueError
return
if finished_at is None or result_code is None:
raise ValueError
if state == "interrupted" and result_code != "process_interrupted":
if state != "succeeded" and result_receipt is not None:
raise ValueError
if state == "interrupted" and result_code not in {
"process_interrupted",
"executor_outcome_unknown",
}:
raise ValueError
@@ -2014,7 +2352,8 @@ def _validate_event_fields(event: dict[str, Any]) -> None:
raise ValueError
if (
event_type == "step_interrupted"
and event["result_code"] != "process_interrupted"
and event["result_code"]
not in {"process_interrupted", "executor_outcome_unknown"}
):
raise ValueError