feat(devkit): add resumable workspace automation and UI review tooling
Verified with the coordinated workspace changes by devkit full run 2026-09-08T225814-186389-0000-3e3ed7cd (all seven phases passed). This shared UI pass does not mark the individual module reviews complete.
This commit is contained in:
Executable
+417
@@ -0,0 +1,417 @@
|
||||
"""Versioned repository-scoped input identities with run-local content memoization.
|
||||
|
||||
No stored receipt supplies commands to this engine. Every snapshot re-reads Git
|
||||
HEAD/index/flags/membership and opens its inputs; only stable file-content hashes
|
||||
are memoized, never repository snapshots or prior-run results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
|
||||
from .common import META_ROOT, canonical, digest, identifier
|
||||
from .workspace import Project, git_bytes
|
||||
|
||||
FINGERPRINT_VERSION = "repository-inputs-v1"
|
||||
MAX_SOURCE_BYTES = 64 * 1024 * 1024
|
||||
MAX_CACHE_FILES = 200_000
|
||||
MAX_REPOSITORY_ENTRIES = 200_000
|
||||
_INDEX_RECORD = re.compile(rb"[A-Za-z] [0-7]{6} [a-fA-F0-9]{40,64} [0-3]\t(.*)\Z", re.S)
|
||||
_RUNTIME_FIELDS = {
|
||||
"status",
|
||||
"exit_code",
|
||||
"duration_seconds",
|
||||
"log_path",
|
||||
"log_sha256",
|
||||
"error",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"reused_from",
|
||||
"output_truncated",
|
||||
"checkpoint_verified",
|
||||
"checkpoint_at",
|
||||
"cache_key",
|
||||
"input_fingerprint",
|
||||
"reuse_reason",
|
||||
"omitted_output_bytes",
|
||||
}
|
||||
|
||||
|
||||
def validate_input_declaration(value: object, repository_names) -> dict:
|
||||
"""Pure validation shared with planning; does not inspect files or run Git."""
|
||||
if not isinstance(value, dict) or set(value) != {"repos"}:
|
||||
raise ValueError(
|
||||
"Stage inputs must contain only the required repos declaration"
|
||||
)
|
||||
names = value["repos"]
|
||||
if not isinstance(names, list) or not 1 <= len(names) <= 256:
|
||||
raise ValueError(
|
||||
"Input repos must be a nonempty bounded list of canonical repository names"
|
||||
)
|
||||
for name in names:
|
||||
identifier(name)
|
||||
if len(set(names)) != len(names):
|
||||
raise ValueError("Duplicate input repository reference")
|
||||
if set(names) - set(repository_names):
|
||||
raise ValueError(
|
||||
"Input repos reference an unknown/noncanonical repository name"
|
||||
)
|
||||
return {"repos": sorted(names)}
|
||||
|
||||
|
||||
def _file_identity(metadata: os.stat_result) -> tuple[int, ...]:
|
||||
return (
|
||||
metadata.st_dev,
|
||||
metadata.st_ino,
|
||||
metadata.st_mode,
|
||||
metadata.st_size,
|
||||
metadata.st_ctime_ns,
|
||||
metadata.st_mtime_ns,
|
||||
)
|
||||
|
||||
|
||||
def _stats() -> dict[str, int]:
|
||||
return {
|
||||
name: 0
|
||||
for name in (
|
||||
"repositories",
|
||||
"git_calls",
|
||||
"entries",
|
||||
"files",
|
||||
"bytes",
|
||||
"cache_hits",
|
||||
"tooling_files",
|
||||
"tooling_bytes",
|
||||
"tooling_cache_hits",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class InputSnapshotter:
|
||||
"""One run's in-memory memo; create a fresh instance for each invocation."""
|
||||
|
||||
def __init__(self, project: Project, *, workspace_root: Path):
|
||||
self.project = project
|
||||
self.workspace_root = Path(workspace_root).resolve()
|
||||
self.repositories = {}
|
||||
for repo in project.repositories:
|
||||
identifier(repo.name)
|
||||
if repo.name in self.repositories:
|
||||
raise ValueError("Duplicate input repository name")
|
||||
if not repo.path.is_absolute() or not repo.path.resolve().is_relative_to(
|
||||
self.workspace_root
|
||||
):
|
||||
raise ValueError("Input repository path escapes its workspace")
|
||||
self.repositories[repo.name] = repo
|
||||
if not self.repositories or len(self.repositories) > 256:
|
||||
raise ValueError("Input project requires 1–256 repositories")
|
||||
# One most recent stable identity per path. This is deliberately not
|
||||
# serializable/persisted, and hashes are never reused on mtime alone.
|
||||
self._files: dict[Path, tuple[tuple[int, ...], bytes]] = {}
|
||||
|
||||
def _repo_names(self, names: object) -> list[str]:
|
||||
return validate_input_declaration({"repos": names}, self.repositories)["repos"]
|
||||
|
||||
def scope(self, stage: dict) -> dict:
|
||||
"""Validate one explicit scope; absent inputs means the whole workspace."""
|
||||
if "inputs" not in stage:
|
||||
return {
|
||||
"version": 1,
|
||||
"kind": "workspace",
|
||||
"declared": False,
|
||||
"repos": sorted(self.repositories),
|
||||
}
|
||||
value = validate_input_declaration(stage["inputs"], self.repositories)
|
||||
return {
|
||||
"version": 1,
|
||||
"kind": "repositories",
|
||||
"declared": True,
|
||||
"repos": value["repos"],
|
||||
}
|
||||
|
||||
def _file_hash(
|
||||
self, path: Path, statistics: dict, *, expected=None, tooling: bool = False
|
||||
) -> bytes:
|
||||
prefix = "tooling_" if tooling else ""
|
||||
maximum = 4 * 1024 * 1024 if tooling else MAX_SOURCE_BYTES
|
||||
descriptor = os.open(
|
||||
path,
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as handle:
|
||||
before = os.fstat(handle.fileno())
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
|
||||
raise ValueError("Input must be a bounded regular file")
|
||||
identity = _file_identity(before)
|
||||
if expected is not None and _file_identity(expected) != identity:
|
||||
raise ValueError(
|
||||
"Input changed before its source identity could be established"
|
||||
)
|
||||
statistics[prefix + "files"] += 1
|
||||
cached = self._files.get(path)
|
||||
if cached is not None and cached[0] == identity:
|
||||
value = cached[1]
|
||||
statistics[prefix + "cache_hits"] += 1
|
||||
else:
|
||||
hasher, size = hashlib.sha256(), 0
|
||||
for chunk in iter(
|
||||
lambda: handle.read(min(1024 * 1024, maximum - size + 1)), b""
|
||||
):
|
||||
size += len(chunk)
|
||||
if size > maximum:
|
||||
raise ValueError("Input grew beyond its source identity limit")
|
||||
hasher.update(chunk)
|
||||
statistics[prefix + "bytes"] += size
|
||||
value = hasher.digest()
|
||||
after = os.fstat(handle.fileno())
|
||||
# Rechecking the pathname also rejects replacement after the open;
|
||||
# a stable old FD is not evidence about a newly replaced source file.
|
||||
current = path.lstat()
|
||||
if _file_identity(after) != identity or _file_identity(current) != identity:
|
||||
raise ValueError("Input changed while calculating its source identity")
|
||||
if path not in self._files and len(self._files) >= MAX_CACHE_FILES:
|
||||
self._files.clear() # Eviction only causes extra reads, never reuse.
|
||||
self._files[path] = (identity, value)
|
||||
return value
|
||||
|
||||
def _entry(self, repo, encoded_name: bytes, statistics: dict) -> dict:
|
||||
relative = Path(os.fsdecode(encoded_name))
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise ValueError("Unsafe source membership path")
|
||||
path = repo.path / relative
|
||||
statistics["entries"] += 1
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return {"kind": "missing"}
|
||||
if not path.parent.resolve().is_relative_to(repo.path.resolve()):
|
||||
raise ValueError("Input path escapes its declared repository")
|
||||
result = {"mode": stat.S_IMODE(metadata.st_mode)}
|
||||
if stat.S_ISLNK(metadata.st_mode):
|
||||
target_name = os.readlink(path)
|
||||
try:
|
||||
target = path.resolve()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise ValueError("Input symlink cannot be resolved safely") from exc
|
||||
if not target.is_relative_to(repo.path.resolve()):
|
||||
raise ValueError(
|
||||
"Input symlink target escapes its repository; undeclared target bytes cannot be reused"
|
||||
)
|
||||
result.update(
|
||||
kind="symlink",
|
||||
target_sha256=hashlib.sha256(os.fsencode(target_name)).hexdigest(),
|
||||
)
|
||||
try:
|
||||
target_metadata = target.lstat()
|
||||
except FileNotFoundError:
|
||||
result["target_state"] = "missing"
|
||||
else:
|
||||
if not stat.S_ISREG(target_metadata.st_mode):
|
||||
raise ValueError(
|
||||
"Input symlink must target a regular file inside its repository"
|
||||
)
|
||||
result.update(
|
||||
target_state="file",
|
||||
target_mode=stat.S_IMODE(target_metadata.st_mode),
|
||||
target_content=self._file_hash(
|
||||
target, statistics, expected=target_metadata
|
||||
).hex(),
|
||||
)
|
||||
if (
|
||||
_file_identity(path.lstat()) != _file_identity(metadata)
|
||||
or os.readlink(path) != target_name
|
||||
):
|
||||
raise ValueError(
|
||||
"Input symlink changed while calculating source identity"
|
||||
)
|
||||
elif stat.S_ISREG(metadata.st_mode):
|
||||
result.update(
|
||||
kind="file",
|
||||
content=self._file_hash(path, statistics, expected=metadata).hex(),
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unsupported source entry; scoped repository inputs require files or in-repository file symlinks"
|
||||
)
|
||||
return result
|
||||
|
||||
def _repository(self, name: str, statistics: dict) -> str:
|
||||
repo = self.repositories[name]
|
||||
statistics["repositories"] += 1
|
||||
identity = {
|
||||
"version": FINGERPRINT_VERSION,
|
||||
"repo": name,
|
||||
"path": str(repo.path),
|
||||
}
|
||||
if not repo.path.exists():
|
||||
return digest({**identity, "state": "missing"})
|
||||
if (
|
||||
not repo.path.resolve().is_relative_to(self.workspace_root)
|
||||
or not (repo.path / ".git").exists()
|
||||
):
|
||||
raise ValueError(
|
||||
"Cannot establish scoped source identity for an escaped/non-Git repository"
|
||||
)
|
||||
statistics["git_calls"] += 1
|
||||
head = git_bytes(
|
||||
repo.path, "rev-parse", "--verify", "HEAD", allow_failure=True
|
||||
).strip()
|
||||
statistics["git_calls"] += 1
|
||||
# This preserves stage numbers and assume-unchanged/skip-worktree flags,
|
||||
# while also listing untracked membership in the same bounded process.
|
||||
index = git_bytes(
|
||||
repo.path,
|
||||
"ls-files",
|
||||
"--stage",
|
||||
"-v",
|
||||
"--cached",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
)
|
||||
members = set()
|
||||
for record in index.split(b"\0"):
|
||||
if not record:
|
||||
continue
|
||||
if record.startswith(b"? "):
|
||||
members.add(record[2:])
|
||||
else:
|
||||
match = _INDEX_RECORD.fullmatch(record)
|
||||
if not match:
|
||||
raise ValueError(
|
||||
"Unsupported Git membership record; cannot establish scoped source identity"
|
||||
)
|
||||
members.add(match[1])
|
||||
if len(members) > MAX_REPOSITORY_ENTRIES:
|
||||
raise ValueError(
|
||||
"Repository source membership exceeds its bounded entry count"
|
||||
)
|
||||
hasher = hashlib.sha256(
|
||||
canonical(
|
||||
{
|
||||
**identity,
|
||||
"head_sha256": hashlib.sha256(head).hexdigest(),
|
||||
"index_sha256": hashlib.sha256(index).hexdigest(),
|
||||
}
|
||||
)
|
||||
)
|
||||
for member in sorted(members):
|
||||
hasher.update(
|
||||
canonical(
|
||||
{
|
||||
"name_sha256": hashlib.sha256(member).hexdigest(),
|
||||
"value": self._entry(repo, member, statistics),
|
||||
}
|
||||
)
|
||||
)
|
||||
hasher.update(b"\0")
|
||||
return hasher.hexdigest()
|
||||
|
||||
def _source_identity(self, names: list[str], identities: dict[str, str]) -> str:
|
||||
return digest(
|
||||
{
|
||||
"version": FINGERPRINT_VERSION,
|
||||
"workspace_root": str(self.workspace_root),
|
||||
"repositories": {name: identities[name] for name in names},
|
||||
}
|
||||
)
|
||||
|
||||
def _observe(self, names: list[str]) -> dict:
|
||||
statistics = _stats()
|
||||
identities = {name: self._repository(name, statistics) for name in names}
|
||||
complete = set(names) == self.repositories.keys()
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"fingerprint_version": FINGERPRINT_VERSION,
|
||||
"workspace_root": str(self.workspace_root),
|
||||
"repository_fingerprints": identities,
|
||||
"observed_source_fingerprint": self._source_identity(names, identities),
|
||||
"observed_scope": {
|
||||
"version": 1,
|
||||
"kind": "workspace" if complete else "repositories",
|
||||
"repos": names,
|
||||
},
|
||||
"complete_workspace": complete,
|
||||
"scan_stats": statistics,
|
||||
}
|
||||
|
||||
def source_snapshot(self, repos: list[str] | None = None) -> dict:
|
||||
"""Compare observed source only, without interpreting any stored commands."""
|
||||
return self._observe(
|
||||
sorted(self.repositories) if repos is None else self._repo_names(repos)
|
||||
)
|
||||
|
||||
def _tooling_identity(self, statistics: dict) -> str:
|
||||
package = Path(__file__).resolve().parent
|
||||
paths = set()
|
||||
for path in package.rglob("*.py"):
|
||||
paths.add(path)
|
||||
if len(paths) > 256:
|
||||
raise ValueError("Devkit tooling source exceeds its bounded inventory")
|
||||
paths.update(
|
||||
{
|
||||
package.parent / "devkit.py",
|
||||
package.parent / "project.schema.json",
|
||||
META_ROOT / "devkit",
|
||||
}
|
||||
)
|
||||
return digest(
|
||||
{
|
||||
str(path.relative_to(META_ROOT)): self._file_hash(
|
||||
path, statistics, tooling=True
|
||||
).hex()
|
||||
for path in sorted(paths)
|
||||
}
|
||||
)
|
||||
|
||||
def snapshot(self, stages: list[dict], *, tooling_fingerprint: str = "") -> dict:
|
||||
if not isinstance(stages, list) or len(stages) > 512:
|
||||
raise ValueError("Input snapshot requires a bounded stage list")
|
||||
if not isinstance(tooling_fingerprint, str) or len(tooling_fingerprint) > 4096:
|
||||
raise ValueError("Invalid supplied tooling identity")
|
||||
scopes = {}
|
||||
for stage in stages:
|
||||
if not isinstance(stage, dict):
|
||||
raise ValueError("Input stages must be objects")
|
||||
if set(stage) & _RUNTIME_FIELDS:
|
||||
raise ValueError(
|
||||
"Input identities require freshly planned stages, not mutable receipt/runtime fields"
|
||||
)
|
||||
identity = identifier(stage.get("id"))
|
||||
if identity in scopes:
|
||||
raise ValueError("Duplicate input stage identity")
|
||||
scopes[identity] = self.scope(stage)
|
||||
names = sorted({name for scope in scopes.values() for name in scope["repos"]})
|
||||
observed = self._observe(names)
|
||||
tools = digest(
|
||||
{
|
||||
"declared_tools": self.project.config.get("tools", {}),
|
||||
"devkit_source": self._tooling_identity(observed["scan_stats"]),
|
||||
"supplied_tooling": tooling_fingerprint,
|
||||
}
|
||||
)
|
||||
entries = {}
|
||||
for stage in stages:
|
||||
scope = scopes[stage["id"]]
|
||||
source = self._source_identity(
|
||||
scope["repos"], observed["repository_fingerprints"]
|
||||
)
|
||||
plan = digest(stage)
|
||||
entries[stage["id"]] = {
|
||||
"source_fingerprint": source,
|
||||
"plan_fingerprint": plan,
|
||||
"scope": scope,
|
||||
"fingerprint": digest(
|
||||
{
|
||||
"version": FINGERPRINT_VERSION,
|
||||
"source": source,
|
||||
"plan": plan,
|
||||
"tooling": tools,
|
||||
}
|
||||
),
|
||||
}
|
||||
return {**observed, "tooling_fingerprint": tools, "stages": entries}
|
||||
Reference in New Issue
Block a user