"""Versioned repository-scoped input identities with run-local content memoization. No stored receipt supplies commands to this engine. Every snapshot re-reads Git HEAD/index/flags/membership and opens its inputs; only stable file-content hashes are memoized, never repository snapshots or prior-run results. """ from __future__ import annotations import hashlib import os from pathlib import Path import re import stat from .common import META_ROOT, canonical, digest, identifier from .workspace import Project, git_bytes FINGERPRINT_VERSION = "repository-inputs-v1" MAX_SOURCE_BYTES = 64 * 1024 * 1024 MAX_CACHE_FILES = 200_000 MAX_REPOSITORY_ENTRIES = 200_000 _INDEX_RECORD = re.compile(rb"[A-Za-z] [0-7]{6} [a-fA-F0-9]{40,64} [0-3]\t(.*)\Z", re.S) _RUNTIME_FIELDS = { "status", "exit_code", "duration_seconds", "log_path", "log_sha256", "error", "started_at", "finished_at", "reused_from", "output_truncated", "checkpoint_verified", "checkpoint_at", "cache_key", "input_fingerprint", "reuse_reason", "omitted_output_bytes", } def validate_input_declaration(value: object, repository_names) -> dict: """Pure validation shared with planning; does not inspect files or run Git.""" if not isinstance(value, dict) or set(value) != {"repos"}: raise ValueError( "Stage inputs must contain only the required repos declaration" ) names = value["repos"] if not isinstance(names, list) or not 1 <= len(names) <= 256: raise ValueError( "Input repos must be a nonempty bounded list of canonical repository names" ) for name in names: identifier(name) if len(set(names)) != len(names): raise ValueError("Duplicate input repository reference") if set(names) - set(repository_names): raise ValueError( "Input repos reference an unknown/noncanonical repository name" ) return {"repos": sorted(names)} def _file_identity(metadata: os.stat_result) -> tuple[int, ...]: return ( metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_size, metadata.st_ctime_ns, metadata.st_mtime_ns, ) def _stats() -> dict[str, int]: return { name: 0 for name in ( "repositories", "git_calls", "entries", "files", "bytes", "cache_hits", "tooling_files", "tooling_bytes", "tooling_cache_hits", ) } class InputSnapshotter: """One run's in-memory memo; create a fresh instance for each invocation.""" def __init__(self, project: Project, *, workspace_root: Path): self.project = project self.workspace_root = Path(workspace_root).resolve() self.repositories = {} for repo in project.repositories: identifier(repo.name) if repo.name in self.repositories: raise ValueError("Duplicate input repository name") if not repo.path.is_absolute() or not repo.path.resolve().is_relative_to( self.workspace_root ): raise ValueError("Input repository path escapes its workspace") self.repositories[repo.name] = repo if not self.repositories or len(self.repositories) > 256: raise ValueError("Input project requires 1–256 repositories") # One most recent stable identity per path. This is deliberately not # serializable/persisted, and hashes are never reused on mtime alone. self._files: dict[Path, tuple[tuple[int, ...], bytes]] = {} def _repo_names(self, names: object) -> list[str]: return validate_input_declaration({"repos": names}, self.repositories)["repos"] def scope(self, stage: dict) -> dict: """Validate one explicit scope; absent inputs means the whole workspace.""" if "inputs" not in stage: return { "version": 1, "kind": "workspace", "declared": False, "repos": sorted(self.repositories), } value = validate_input_declaration(stage["inputs"], self.repositories) return { "version": 1, "kind": "repositories", "declared": True, "repos": value["repos"], } def _file_hash( self, path: Path, statistics: dict, *, expected=None, tooling: bool = False ) -> bytes: prefix = "tooling_" if tooling else "" maximum = 4 * 1024 * 1024 if tooling else MAX_SOURCE_BYTES descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), ) with os.fdopen(descriptor, "rb") as handle: before = os.fstat(handle.fileno()) if not stat.S_ISREG(before.st_mode) or before.st_size > maximum: raise ValueError("Input must be a bounded regular file") identity = _file_identity(before) if expected is not None and _file_identity(expected) != identity: raise ValueError( "Input changed before its source identity could be established" ) statistics[prefix + "files"] += 1 cached = self._files.get(path) if cached is not None and cached[0] == identity: value = cached[1] statistics[prefix + "cache_hits"] += 1 else: hasher, size = hashlib.sha256(), 0 for chunk in iter( lambda: handle.read(min(1024 * 1024, maximum - size + 1)), b"" ): size += len(chunk) if size > maximum: raise ValueError("Input grew beyond its source identity limit") hasher.update(chunk) statistics[prefix + "bytes"] += size value = hasher.digest() after = os.fstat(handle.fileno()) # Rechecking the pathname also rejects replacement after the open; # a stable old FD is not evidence about a newly replaced source file. current = path.lstat() if _file_identity(after) != identity or _file_identity(current) != identity: raise ValueError("Input changed while calculating its source identity") if path not in self._files and len(self._files) >= MAX_CACHE_FILES: self._files.clear() # Eviction only causes extra reads, never reuse. self._files[path] = (identity, value) return value def _entry(self, repo, encoded_name: bytes, statistics: dict) -> dict: relative = Path(os.fsdecode(encoded_name)) if relative.is_absolute() or ".." in relative.parts: raise ValueError("Unsafe source membership path") path = repo.path / relative statistics["entries"] += 1 try: metadata = path.lstat() except FileNotFoundError: return {"kind": "missing"} if not path.parent.resolve().is_relative_to(repo.path.resolve()): raise ValueError("Input path escapes its declared repository") result = {"mode": stat.S_IMODE(metadata.st_mode)} if stat.S_ISLNK(metadata.st_mode): target_name = os.readlink(path) try: target = path.resolve() except (OSError, RuntimeError) as exc: raise ValueError("Input symlink cannot be resolved safely") from exc if not target.is_relative_to(repo.path.resolve()): raise ValueError( "Input symlink target escapes its repository; undeclared target bytes cannot be reused" ) result.update( kind="symlink", target_sha256=hashlib.sha256(os.fsencode(target_name)).hexdigest(), ) try: target_metadata = target.lstat() except FileNotFoundError: result["target_state"] = "missing" else: if not stat.S_ISREG(target_metadata.st_mode): raise ValueError( "Input symlink must target a regular file inside its repository" ) result.update( target_state="file", target_mode=stat.S_IMODE(target_metadata.st_mode), target_content=self._file_hash( target, statistics, expected=target_metadata ).hex(), ) if ( _file_identity(path.lstat()) != _file_identity(metadata) or os.readlink(path) != target_name ): raise ValueError( "Input symlink changed while calculating source identity" ) elif stat.S_ISREG(metadata.st_mode): result.update( kind="file", content=self._file_hash(path, statistics, expected=metadata).hex(), ) else: raise ValueError( "Unsupported source entry; scoped repository inputs require files or in-repository file symlinks" ) return result def _repository(self, name: str, statistics: dict) -> str: repo = self.repositories[name] statistics["repositories"] += 1 identity = { "version": FINGERPRINT_VERSION, "repo": name, "path": str(repo.path), } if not repo.path.exists(): return digest({**identity, "state": "missing"}) if ( not repo.path.resolve().is_relative_to(self.workspace_root) or not (repo.path / ".git").exists() ): raise ValueError( "Cannot establish scoped source identity for an escaped/non-Git repository" ) statistics["git_calls"] += 1 head = git_bytes( repo.path, "rev-parse", "--verify", "HEAD", allow_failure=True ).strip() statistics["git_calls"] += 1 # This preserves stage numbers and assume-unchanged/skip-worktree flags, # while also listing untracked membership in the same bounded process. index = git_bytes( repo.path, "ls-files", "--stage", "-v", "--cached", "--others", "--exclude-standard", "-z", ) members = set() for record in index.split(b"\0"): if not record: continue if record.startswith(b"? "): members.add(record[2:]) else: match = _INDEX_RECORD.fullmatch(record) if not match: raise ValueError( "Unsupported Git membership record; cannot establish scoped source identity" ) members.add(match[1]) if len(members) > MAX_REPOSITORY_ENTRIES: raise ValueError( "Repository source membership exceeds its bounded entry count" ) hasher = hashlib.sha256( canonical( { **identity, "head_sha256": hashlib.sha256(head).hexdigest(), "index_sha256": hashlib.sha256(index).hexdigest(), } ) ) for member in sorted(members): hasher.update( canonical( { "name_sha256": hashlib.sha256(member).hexdigest(), "value": self._entry(repo, member, statistics), } ) ) hasher.update(b"\0") return hasher.hexdigest() def _source_identity(self, names: list[str], identities: dict[str, str]) -> str: return digest( { "version": FINGERPRINT_VERSION, "workspace_root": str(self.workspace_root), "repositories": {name: identities[name] for name in names}, } ) def _observe(self, names: list[str]) -> dict: statistics = _stats() identities = {name: self._repository(name, statistics) for name in names} complete = set(names) == self.repositories.keys() return { "schema_version": 1, "fingerprint_version": FINGERPRINT_VERSION, "workspace_root": str(self.workspace_root), "repository_fingerprints": identities, "observed_source_fingerprint": self._source_identity(names, identities), "observed_scope": { "version": 1, "kind": "workspace" if complete else "repositories", "repos": names, }, "complete_workspace": complete, "scan_stats": statistics, } def source_snapshot(self, repos: list[str] | None = None) -> dict: """Compare observed source only, without interpreting any stored commands.""" return self._observe( sorted(self.repositories) if repos is None else self._repo_names(repos) ) def _tooling_identity(self, statistics: dict) -> str: package = Path(__file__).resolve().parent paths = set() for path in package.rglob("*.py"): paths.add(path) if len(paths) > 256: raise ValueError("Devkit tooling source exceeds its bounded inventory") paths.update( { package.parent / "devkit.py", package.parent / "project.schema.json", META_ROOT / "devkit", } ) return digest( { str(path.relative_to(META_ROOT)): self._file_hash( path, statistics, tooling=True ).hex() for path in sorted(paths) } ) def snapshot(self, stages: list[dict], *, tooling_fingerprint: str = "") -> dict: if not isinstance(stages, list) or len(stages) > 512: raise ValueError("Input snapshot requires a bounded stage list") if not isinstance(tooling_fingerprint, str) or len(tooling_fingerprint) > 4096: raise ValueError("Invalid supplied tooling identity") scopes = {} for stage in stages: if not isinstance(stage, dict): raise ValueError("Input stages must be objects") if set(stage) & _RUNTIME_FIELDS: raise ValueError( "Input identities require freshly planned stages, not mutable receipt/runtime fields" ) identity = identifier(stage.get("id")) if identity in scopes: raise ValueError("Duplicate input stage identity") scopes[identity] = self.scope(stage) names = sorted({name for scope in scopes.values() for name in scope["repos"]}) observed = self._observe(names) tools = digest( { "declared_tools": self.project.config.get("tools", {}), "devkit_source": self._tooling_identity(observed["scan_stats"]), "supplied_tooling": tooling_fingerprint, } ) entries = {} for stage in stages: scope = scopes[stage["id"]] source = self._source_identity( scope["repos"], observed["repository_fingerprints"] ) plan = digest(stage) entries[stage["id"]] = { "source_fingerprint": source, "plan_fingerprint": plan, "scope": scope, "fingerprint": digest( { "version": FINGERPRINT_VERSION, "source": source, "plan": plan, "tooling": tools, } ), } return {**observed, "tooling_fingerprint": tools, "stages": entries}