zemion
2ffdb23f69
Verified with the coordinated workspace changes by devkit full run 2026-09-08T225814-186389-0000-3e3ed7cd (all seven phases passed). This shared UI pass does not mark the individual module reviews complete.
418 lines
16 KiB
Python
Executable File
418 lines
16 KiB
Python
Executable File
"""Versioned repository-scoped input identities with run-local content memoization.
|
||
|
||
No stored receipt supplies commands to this engine. Every snapshot re-reads Git
|
||
HEAD/index/flags/membership and opens its inputs; only stable file-content hashes
|
||
are memoized, never repository snapshots or prior-run results.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import stat
|
||
|
||
from .common import META_ROOT, canonical, digest, identifier
|
||
from .workspace import Project, git_bytes
|
||
|
||
FINGERPRINT_VERSION = "repository-inputs-v1"
|
||
MAX_SOURCE_BYTES = 64 * 1024 * 1024
|
||
MAX_CACHE_FILES = 200_000
|
||
MAX_REPOSITORY_ENTRIES = 200_000
|
||
_INDEX_RECORD = re.compile(rb"[A-Za-z] [0-7]{6} [a-fA-F0-9]{40,64} [0-3]\t(.*)\Z", re.S)
|
||
_RUNTIME_FIELDS = {
|
||
"status",
|
||
"exit_code",
|
||
"duration_seconds",
|
||
"log_path",
|
||
"log_sha256",
|
||
"error",
|
||
"started_at",
|
||
"finished_at",
|
||
"reused_from",
|
||
"output_truncated",
|
||
"checkpoint_verified",
|
||
"checkpoint_at",
|
||
"cache_key",
|
||
"input_fingerprint",
|
||
"reuse_reason",
|
||
"omitted_output_bytes",
|
||
}
|
||
|
||
|
||
def validate_input_declaration(value: object, repository_names) -> dict:
|
||
"""Pure validation shared with planning; does not inspect files or run Git."""
|
||
if not isinstance(value, dict) or set(value) != {"repos"}:
|
||
raise ValueError(
|
||
"Stage inputs must contain only the required repos declaration"
|
||
)
|
||
names = value["repos"]
|
||
if not isinstance(names, list) or not 1 <= len(names) <= 256:
|
||
raise ValueError(
|
||
"Input repos must be a nonempty bounded list of canonical repository names"
|
||
)
|
||
for name in names:
|
||
identifier(name)
|
||
if len(set(names)) != len(names):
|
||
raise ValueError("Duplicate input repository reference")
|
||
if set(names) - set(repository_names):
|
||
raise ValueError(
|
||
"Input repos reference an unknown/noncanonical repository name"
|
||
)
|
||
return {"repos": sorted(names)}
|
||
|
||
|
||
def _file_identity(metadata: os.stat_result) -> tuple[int, ...]:
|
||
return (
|
||
metadata.st_dev,
|
||
metadata.st_ino,
|
||
metadata.st_mode,
|
||
metadata.st_size,
|
||
metadata.st_ctime_ns,
|
||
metadata.st_mtime_ns,
|
||
)
|
||
|
||
|
||
def _stats() -> dict[str, int]:
|
||
return {
|
||
name: 0
|
||
for name in (
|
||
"repositories",
|
||
"git_calls",
|
||
"entries",
|
||
"files",
|
||
"bytes",
|
||
"cache_hits",
|
||
"tooling_files",
|
||
"tooling_bytes",
|
||
"tooling_cache_hits",
|
||
)
|
||
}
|
||
|
||
|
||
class InputSnapshotter:
|
||
"""One run's in-memory memo; create a fresh instance for each invocation."""
|
||
|
||
def __init__(self, project: Project, *, workspace_root: Path):
|
||
self.project = project
|
||
self.workspace_root = Path(workspace_root).resolve()
|
||
self.repositories = {}
|
||
for repo in project.repositories:
|
||
identifier(repo.name)
|
||
if repo.name in self.repositories:
|
||
raise ValueError("Duplicate input repository name")
|
||
if not repo.path.is_absolute() or not repo.path.resolve().is_relative_to(
|
||
self.workspace_root
|
||
):
|
||
raise ValueError("Input repository path escapes its workspace")
|
||
self.repositories[repo.name] = repo
|
||
if not self.repositories or len(self.repositories) > 256:
|
||
raise ValueError("Input project requires 1–256 repositories")
|
||
# One most recent stable identity per path. This is deliberately not
|
||
# serializable/persisted, and hashes are never reused on mtime alone.
|
||
self._files: dict[Path, tuple[tuple[int, ...], bytes]] = {}
|
||
|
||
def _repo_names(self, names: object) -> list[str]:
|
||
return validate_input_declaration({"repos": names}, self.repositories)["repos"]
|
||
|
||
def scope(self, stage: dict) -> dict:
|
||
"""Validate one explicit scope; absent inputs means the whole workspace."""
|
||
if "inputs" not in stage:
|
||
return {
|
||
"version": 1,
|
||
"kind": "workspace",
|
||
"declared": False,
|
||
"repos": sorted(self.repositories),
|
||
}
|
||
value = validate_input_declaration(stage["inputs"], self.repositories)
|
||
return {
|
||
"version": 1,
|
||
"kind": "repositories",
|
||
"declared": True,
|
||
"repos": value["repos"],
|
||
}
|
||
|
||
def _file_hash(
|
||
self, path: Path, statistics: dict, *, expected=None, tooling: bool = False
|
||
) -> bytes:
|
||
prefix = "tooling_" if tooling else ""
|
||
maximum = 4 * 1024 * 1024 if tooling else MAX_SOURCE_BYTES
|
||
descriptor = os.open(
|
||
path,
|
||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
|
||
)
|
||
with os.fdopen(descriptor, "rb") as handle:
|
||
before = os.fstat(handle.fileno())
|
||
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
|
||
raise ValueError("Input must be a bounded regular file")
|
||
identity = _file_identity(before)
|
||
if expected is not None and _file_identity(expected) != identity:
|
||
raise ValueError(
|
||
"Input changed before its source identity could be established"
|
||
)
|
||
statistics[prefix + "files"] += 1
|
||
cached = self._files.get(path)
|
||
if cached is not None and cached[0] == identity:
|
||
value = cached[1]
|
||
statistics[prefix + "cache_hits"] += 1
|
||
else:
|
||
hasher, size = hashlib.sha256(), 0
|
||
for chunk in iter(
|
||
lambda: handle.read(min(1024 * 1024, maximum - size + 1)), b""
|
||
):
|
||
size += len(chunk)
|
||
if size > maximum:
|
||
raise ValueError("Input grew beyond its source identity limit")
|
||
hasher.update(chunk)
|
||
statistics[prefix + "bytes"] += size
|
||
value = hasher.digest()
|
||
after = os.fstat(handle.fileno())
|
||
# Rechecking the pathname also rejects replacement after the open;
|
||
# a stable old FD is not evidence about a newly replaced source file.
|
||
current = path.lstat()
|
||
if _file_identity(after) != identity or _file_identity(current) != identity:
|
||
raise ValueError("Input changed while calculating its source identity")
|
||
if path not in self._files and len(self._files) >= MAX_CACHE_FILES:
|
||
self._files.clear() # Eviction only causes extra reads, never reuse.
|
||
self._files[path] = (identity, value)
|
||
return value
|
||
|
||
def _entry(self, repo, encoded_name: bytes, statistics: dict) -> dict:
|
||
relative = Path(os.fsdecode(encoded_name))
|
||
if relative.is_absolute() or ".." in relative.parts:
|
||
raise ValueError("Unsafe source membership path")
|
||
path = repo.path / relative
|
||
statistics["entries"] += 1
|
||
try:
|
||
metadata = path.lstat()
|
||
except FileNotFoundError:
|
||
return {"kind": "missing"}
|
||
if not path.parent.resolve().is_relative_to(repo.path.resolve()):
|
||
raise ValueError("Input path escapes its declared repository")
|
||
result = {"mode": stat.S_IMODE(metadata.st_mode)}
|
||
if stat.S_ISLNK(metadata.st_mode):
|
||
target_name = os.readlink(path)
|
||
try:
|
||
target = path.resolve()
|
||
except (OSError, RuntimeError) as exc:
|
||
raise ValueError("Input symlink cannot be resolved safely") from exc
|
||
if not target.is_relative_to(repo.path.resolve()):
|
||
raise ValueError(
|
||
"Input symlink target escapes its repository; undeclared target bytes cannot be reused"
|
||
)
|
||
result.update(
|
||
kind="symlink",
|
||
target_sha256=hashlib.sha256(os.fsencode(target_name)).hexdigest(),
|
||
)
|
||
try:
|
||
target_metadata = target.lstat()
|
||
except FileNotFoundError:
|
||
result["target_state"] = "missing"
|
||
else:
|
||
if not stat.S_ISREG(target_metadata.st_mode):
|
||
raise ValueError(
|
||
"Input symlink must target a regular file inside its repository"
|
||
)
|
||
result.update(
|
||
target_state="file",
|
||
target_mode=stat.S_IMODE(target_metadata.st_mode),
|
||
target_content=self._file_hash(
|
||
target, statistics, expected=target_metadata
|
||
).hex(),
|
||
)
|
||
if (
|
||
_file_identity(path.lstat()) != _file_identity(metadata)
|
||
or os.readlink(path) != target_name
|
||
):
|
||
raise ValueError(
|
||
"Input symlink changed while calculating source identity"
|
||
)
|
||
elif stat.S_ISREG(metadata.st_mode):
|
||
result.update(
|
||
kind="file",
|
||
content=self._file_hash(path, statistics, expected=metadata).hex(),
|
||
)
|
||
else:
|
||
raise ValueError(
|
||
"Unsupported source entry; scoped repository inputs require files or in-repository file symlinks"
|
||
)
|
||
return result
|
||
|
||
def _repository(self, name: str, statistics: dict) -> str:
|
||
repo = self.repositories[name]
|
||
statistics["repositories"] += 1
|
||
identity = {
|
||
"version": FINGERPRINT_VERSION,
|
||
"repo": name,
|
||
"path": str(repo.path),
|
||
}
|
||
if not repo.path.exists():
|
||
return digest({**identity, "state": "missing"})
|
||
if (
|
||
not repo.path.resolve().is_relative_to(self.workspace_root)
|
||
or not (repo.path / ".git").exists()
|
||
):
|
||
raise ValueError(
|
||
"Cannot establish scoped source identity for an escaped/non-Git repository"
|
||
)
|
||
statistics["git_calls"] += 1
|
||
head = git_bytes(
|
||
repo.path, "rev-parse", "--verify", "HEAD", allow_failure=True
|
||
).strip()
|
||
statistics["git_calls"] += 1
|
||
# This preserves stage numbers and assume-unchanged/skip-worktree flags,
|
||
# while also listing untracked membership in the same bounded process.
|
||
index = git_bytes(
|
||
repo.path,
|
||
"ls-files",
|
||
"--stage",
|
||
"-v",
|
||
"--cached",
|
||
"--others",
|
||
"--exclude-standard",
|
||
"-z",
|
||
)
|
||
members = set()
|
||
for record in index.split(b"\0"):
|
||
if not record:
|
||
continue
|
||
if record.startswith(b"? "):
|
||
members.add(record[2:])
|
||
else:
|
||
match = _INDEX_RECORD.fullmatch(record)
|
||
if not match:
|
||
raise ValueError(
|
||
"Unsupported Git membership record; cannot establish scoped source identity"
|
||
)
|
||
members.add(match[1])
|
||
if len(members) > MAX_REPOSITORY_ENTRIES:
|
||
raise ValueError(
|
||
"Repository source membership exceeds its bounded entry count"
|
||
)
|
||
hasher = hashlib.sha256(
|
||
canonical(
|
||
{
|
||
**identity,
|
||
"head_sha256": hashlib.sha256(head).hexdigest(),
|
||
"index_sha256": hashlib.sha256(index).hexdigest(),
|
||
}
|
||
)
|
||
)
|
||
for member in sorted(members):
|
||
hasher.update(
|
||
canonical(
|
||
{
|
||
"name_sha256": hashlib.sha256(member).hexdigest(),
|
||
"value": self._entry(repo, member, statistics),
|
||
}
|
||
)
|
||
)
|
||
hasher.update(b"\0")
|
||
return hasher.hexdigest()
|
||
|
||
def _source_identity(self, names: list[str], identities: dict[str, str]) -> str:
|
||
return digest(
|
||
{
|
||
"version": FINGERPRINT_VERSION,
|
||
"workspace_root": str(self.workspace_root),
|
||
"repositories": {name: identities[name] for name in names},
|
||
}
|
||
)
|
||
|
||
def _observe(self, names: list[str]) -> dict:
|
||
statistics = _stats()
|
||
identities = {name: self._repository(name, statistics) for name in names}
|
||
complete = set(names) == self.repositories.keys()
|
||
return {
|
||
"schema_version": 1,
|
||
"fingerprint_version": FINGERPRINT_VERSION,
|
||
"workspace_root": str(self.workspace_root),
|
||
"repository_fingerprints": identities,
|
||
"observed_source_fingerprint": self._source_identity(names, identities),
|
||
"observed_scope": {
|
||
"version": 1,
|
||
"kind": "workspace" if complete else "repositories",
|
||
"repos": names,
|
||
},
|
||
"complete_workspace": complete,
|
||
"scan_stats": statistics,
|
||
}
|
||
|
||
def source_snapshot(self, repos: list[str] | None = None) -> dict:
|
||
"""Compare observed source only, without interpreting any stored commands."""
|
||
return self._observe(
|
||
sorted(self.repositories) if repos is None else self._repo_names(repos)
|
||
)
|
||
|
||
def _tooling_identity(self, statistics: dict) -> str:
|
||
package = Path(__file__).resolve().parent
|
||
paths = set()
|
||
for path in package.rglob("*.py"):
|
||
paths.add(path)
|
||
if len(paths) > 256:
|
||
raise ValueError("Devkit tooling source exceeds its bounded inventory")
|
||
paths.update(
|
||
{
|
||
package.parent / "devkit.py",
|
||
package.parent / "project.schema.json",
|
||
META_ROOT / "devkit",
|
||
}
|
||
)
|
||
return digest(
|
||
{
|
||
str(path.relative_to(META_ROOT)): self._file_hash(
|
||
path, statistics, tooling=True
|
||
).hex()
|
||
for path in sorted(paths)
|
||
}
|
||
)
|
||
|
||
def snapshot(self, stages: list[dict], *, tooling_fingerprint: str = "") -> dict:
|
||
if not isinstance(stages, list) or len(stages) > 512:
|
||
raise ValueError("Input snapshot requires a bounded stage list")
|
||
if not isinstance(tooling_fingerprint, str) or len(tooling_fingerprint) > 4096:
|
||
raise ValueError("Invalid supplied tooling identity")
|
||
scopes = {}
|
||
for stage in stages:
|
||
if not isinstance(stage, dict):
|
||
raise ValueError("Input stages must be objects")
|
||
if set(stage) & _RUNTIME_FIELDS:
|
||
raise ValueError(
|
||
"Input identities require freshly planned stages, not mutable receipt/runtime fields"
|
||
)
|
||
identity = identifier(stage.get("id"))
|
||
if identity in scopes:
|
||
raise ValueError("Duplicate input stage identity")
|
||
scopes[identity] = self.scope(stage)
|
||
names = sorted({name for scope in scopes.values() for name in scope["repos"]})
|
||
observed = self._observe(names)
|
||
tools = digest(
|
||
{
|
||
"declared_tools": self.project.config.get("tools", {}),
|
||
"devkit_source": self._tooling_identity(observed["scan_stats"]),
|
||
"supplied_tooling": tooling_fingerprint,
|
||
}
|
||
)
|
||
entries = {}
|
||
for stage in stages:
|
||
scope = scopes[stage["id"]]
|
||
source = self._source_identity(
|
||
scope["repos"], observed["repository_fingerprints"]
|
||
)
|
||
plan = digest(stage)
|
||
entries[stage["id"]] = {
|
||
"source_fingerprint": source,
|
||
"plan_fingerprint": plan,
|
||
"scope": scope,
|
||
"fingerprint": digest(
|
||
{
|
||
"version": FINGERPRINT_VERSION,
|
||
"source": source,
|
||
"plan": plan,
|
||
"tooling": tools,
|
||
}
|
||
),
|
||
}
|
||
return {**observed, "tooling_fingerprint": tools, "stages": entries}
|