feat(release): bound durable run retention
This commit is contained in:
@@ -8,6 +8,8 @@ to this state in a later slice.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
@@ -31,6 +33,8 @@ MAX_REPOSITORY_VERSIONS = 128
|
||||
MAX_EVENTS = 256
|
||||
MAX_COMMANDS = 2_048
|
||||
MAX_LIST_LIMIT = 100
|
||||
MAX_RUN_RECORDS = 512
|
||||
MAX_CURSOR_BYTES = 1_024
|
||||
RUN_ID_PATTERN = re.compile(
|
||||
r"^(?:rr-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}|rr-request-[0-9a-f]{64})$"
|
||||
)
|
||||
@@ -165,7 +169,16 @@ class ReleaseRunStore:
|
||||
input_snapshot=immutable_input,
|
||||
)
|
||||
return _view(existing)
|
||||
retirement = self._retirement_candidate()
|
||||
self._write_new(record)
|
||||
if retirement is not None:
|
||||
try:
|
||||
self._unlink_retained_record(retirement)
|
||||
except ReleaseRunError:
|
||||
# Do not leave the store beyond its hard record bound when
|
||||
# retirement fails before it can make room.
|
||||
self._unlink_new_record(self._path(run_id))
|
||||
raise
|
||||
return _view(record)
|
||||
|
||||
def find_created_run(
|
||||
@@ -199,59 +212,78 @@ class ReleaseRunStore:
|
||||
return _view(record)
|
||||
|
||||
def list(self, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Return the first page for backwards-compatible local callers."""
|
||||
|
||||
return self.list_page(limit=limit)["runs"]
|
||||
|
||||
def list_page(
|
||||
self, *, limit: int = 20, cursor: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Return a deterministic descending page, including corrupt entries."""
|
||||
|
||||
limit = min(max(int(limit), 1), MAX_LIST_LIMIT)
|
||||
cursor_key = _decode_list_cursor(
|
||||
cursor,
|
||||
expected_workspace_fingerprint=self.expected_workspace_fingerprint,
|
||||
)
|
||||
with self._locked():
|
||||
self._ensure_root()
|
||||
try:
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.name.endswith(".json")
|
||||
and RUN_ID_PATTERN.fullmatch(path.stem)
|
||||
]
|
||||
except OSError as exc:
|
||||
raise ReleaseRunCorrupt(
|
||||
"Release run storage could not be enumerated safely."
|
||||
) from exc
|
||||
summaries: list[dict[str, Any]] = []
|
||||
unavailable: list[dict[str, Any]] = []
|
||||
for path in paths:
|
||||
entries: list[tuple[tuple[int, str, str, str], dict[str, Any]]] = []
|
||||
for path in self._record_paths():
|
||||
try:
|
||||
record = self._read(path.stem)
|
||||
except ReleaseRunWorkspaceMismatch:
|
||||
continue
|
||||
except (ReleaseRunCorrupt, ReleaseRunNotFound):
|
||||
unavailable.append(
|
||||
{
|
||||
"run_id": path.stem,
|
||||
"status": "unavailable",
|
||||
"error": "Run record is unavailable or failed integrity validation.",
|
||||
}
|
||||
entries.append(
|
||||
(
|
||||
(0, "", "", path.stem),
|
||||
{
|
||||
"run_id": path.stem,
|
||||
"status": "unavailable",
|
||||
"error": "Run record is unavailable or failed integrity validation.",
|
||||
},
|
||||
)
|
||||
)
|
||||
continue
|
||||
view = _view(record)
|
||||
immutable_input = view["immutable"]["input"]
|
||||
summaries.append(
|
||||
{
|
||||
"run_id": view["run_id"],
|
||||
"created_at": view["created_at"],
|
||||
"updated_at": view["updated_at"],
|
||||
"status": view["state"]["status"],
|
||||
"channel": immutable_input["channel"],
|
||||
"repo_versions": immutable_input["repo_versions"],
|
||||
"recommended_next": view["recommended_next"],
|
||||
}
|
||||
entries.append(
|
||||
(
|
||||
(
|
||||
1,
|
||||
view["updated_at"],
|
||||
view["created_at"],
|
||||
view["run_id"],
|
||||
),
|
||||
{
|
||||
"run_id": view["run_id"],
|
||||
"created_at": view["created_at"],
|
||||
"updated_at": view["updated_at"],
|
||||
"status": view["state"]["status"],
|
||||
"channel": immutable_input["channel"],
|
||||
"repo_versions": immutable_input["repo_versions"],
|
||||
"recommended_next": view["recommended_next"],
|
||||
},
|
||||
)
|
||||
)
|
||||
summaries.sort(
|
||||
key=lambda item: (
|
||||
item["updated_at"],
|
||||
item["created_at"],
|
||||
item["run_id"],
|
||||
),
|
||||
reverse=True,
|
||||
entries.sort(key=lambda item: item[0], reverse=True)
|
||||
if cursor_key is not None:
|
||||
entries = [entry for entry in entries if entry[0] < cursor_key]
|
||||
page = entries[: limit + 1]
|
||||
has_more = len(page) > limit
|
||||
visible = page[:limit]
|
||||
next_cursor = (
|
||||
_encode_list_cursor(
|
||||
visible[-1][0],
|
||||
workspace_fingerprint=self.expected_workspace_fingerprint,
|
||||
)
|
||||
unavailable.sort(key=lambda item: item["run_id"], reverse=True)
|
||||
return (summaries + unavailable)[:limit]
|
||||
if has_more and visible
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"runs": [summary for _key, summary in visible],
|
||||
"next_cursor": next_cursor,
|
||||
}
|
||||
|
||||
def resume(self, run_id: str, *, request_id: str) -> dict[str, Any]:
|
||||
request_fingerprint = _request_fingerprint(request_id)
|
||||
@@ -614,6 +646,88 @@ class ReleaseRunStore:
|
||||
except ReleaseRunNotFound:
|
||||
return None
|
||||
|
||||
def _record_paths(self) -> list[Path]:
|
||||
self._ensure_root()
|
||||
try:
|
||||
return [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.name.endswith(".json")
|
||||
and RUN_ID_PATTERN.fullmatch(path.stem)
|
||||
]
|
||||
except OSError as exc:
|
||||
raise ReleaseRunCorrupt(
|
||||
"Release run storage could not be enumerated safely."
|
||||
) from exc
|
||||
|
||||
def _retirement_candidate(self) -> Path | None:
|
||||
"""Select one verified terminal record without deleting it yet."""
|
||||
|
||||
paths = self._record_paths()
|
||||
if len(paths) < MAX_RUN_RECORDS:
|
||||
return None
|
||||
if len(paths) > MAX_RUN_RECORDS:
|
||||
raise ReleaseRunConflict(
|
||||
"Release run storage exceeds its retention limit; inspect and "
|
||||
"repair the private store before creating another run."
|
||||
)
|
||||
completed: list[tuple[tuple[str, str, str], Path]] = []
|
||||
for path in paths:
|
||||
try:
|
||||
record = self._read(path.stem)
|
||||
except (ReleaseRunCorrupt, ReleaseRunNotFound, ReleaseRunWorkspaceMismatch):
|
||||
# Unavailable evidence is deliberately never deleted implicitly.
|
||||
continue
|
||||
if record["state"]["status"] == "completed":
|
||||
completed.append(
|
||||
(
|
||||
(
|
||||
record["updated_at"],
|
||||
record["created_at"],
|
||||
record["run_id"],
|
||||
),
|
||||
path,
|
||||
)
|
||||
)
|
||||
completed.sort(key=lambda item: item[0])
|
||||
if not completed:
|
||||
raise ReleaseRunConflict(
|
||||
"Release run retention limit is reached; complete or explicitly "
|
||||
"remove retained runs before creating another run."
|
||||
)
|
||||
return completed[0][1]
|
||||
|
||||
def _unlink_retained_record(self, path: Path) -> None:
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(metadata.st_mode)
|
||||
or metadata.st_nlink != 1
|
||||
or (hasattr(os, "geteuid") and metadata.st_uid != os.geteuid())
|
||||
or metadata.st_mode & 0o077
|
||||
):
|
||||
raise ReleaseRunCorrupt(
|
||||
"Completed release run could not be retired safely."
|
||||
)
|
||||
path.unlink()
|
||||
_fsync_directory(self.root)
|
||||
except ReleaseRunError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ReleaseRunCorrupt(
|
||||
"Completed release run could not be retired safely."
|
||||
) from exc
|
||||
|
||||
def _unlink_new_record(self, path: Path) -> None:
|
||||
try:
|
||||
path.unlink()
|
||||
_fsync_directory(self.root)
|
||||
except OSError as exc:
|
||||
raise ReleaseRunCorrupt(
|
||||
"Release run creation could not be rolled back safely after a "
|
||||
"retention failure. Inspect the private store before continuing."
|
||||
) from exc
|
||||
|
||||
def _assert_workspace(self, record: dict[str, Any]) -> None:
|
||||
if (
|
||||
record["immutable"]["input"]["workspace_fingerprint"]
|
||||
@@ -1736,6 +1850,63 @@ def _creation_run_id(request_fingerprint: str) -> str:
|
||||
return f"rr-request-{request_fingerprint}"
|
||||
|
||||
|
||||
def _encode_list_cursor(
|
||||
key: tuple[int, str, str, str], *, workspace_fingerprint: str
|
||||
) -> str:
|
||||
payload = {
|
||||
"v": 1,
|
||||
"workspace": workspace_fingerprint,
|
||||
"key": list(key),
|
||||
}
|
||||
encoded = base64.urlsafe_b64encode(_canonical_json(payload)).decode("ascii")
|
||||
return encoded.rstrip("=")
|
||||
|
||||
|
||||
def _decode_list_cursor(
|
||||
cursor: str | None, *, expected_workspace_fingerprint: str
|
||||
) -> tuple[int, str, str, str] | None:
|
||||
if cursor is None or cursor == "":
|
||||
return None
|
||||
if not isinstance(cursor, str) or len(cursor) > MAX_CURSOR_BYTES:
|
||||
raise ReleaseRunConflict("Release run list cursor is invalid.")
|
||||
try:
|
||||
padding = "=" * (-len(cursor) % 4)
|
||||
encoded = base64.b64decode(
|
||||
cursor + padding,
|
||||
altchars=b"-_",
|
||||
validate=True,
|
||||
)
|
||||
payload = json.loads(encoded.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError
|
||||
key = payload.get("key")
|
||||
if (
|
||||
set(payload) != {"v", "workspace", "key"}
|
||||
or payload.get("v") != 1
|
||||
or payload.get("workspace") != expected_workspace_fingerprint
|
||||
or not isinstance(key, list)
|
||||
or len(key) != 4
|
||||
or type(key[0]) is not int
|
||||
or key[0] not in {0, 1}
|
||||
or not all(isinstance(item, str) for item in key[1:])
|
||||
or not RUN_ID_PATTERN.fullmatch(key[3])
|
||||
):
|
||||
raise ValueError
|
||||
if key[0] == 1:
|
||||
if not _is_timestamp(key[1]) or not _is_timestamp(key[2]):
|
||||
raise ValueError
|
||||
elif key[1] or key[2]:
|
||||
raise ValueError
|
||||
except (
|
||||
binascii.Error,
|
||||
UnicodeDecodeError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
raise ReleaseRunConflict("Release run list cursor is invalid.") from exc
|
||||
return key[0], key[1], key[2], key[3]
|
||||
|
||||
|
||||
def _validate_create_replay(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user