zemion
2ffdb23f69
Verified with the coordinated workspace changes by devkit full run 2026-09-08T225814-186389-0000-3e3ed7cd (all seven phases passed). This shared UI pass does not mark the individual module reviews complete.
1035 lines
40 KiB
Python
Executable File
1035 lines
40 KiB
Python
Executable File
"""Resource-aware checks with bounded logs and resumable, source-bound receipts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
|
|
from contextlib import ExitStack
|
|
from copy import deepcopy
|
|
import os
|
|
import hashlib
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from .common import (
|
|
atomic_json,
|
|
atomic_text,
|
|
digest,
|
|
identifier,
|
|
now,
|
|
private_directory,
|
|
read_bounded_bytes,
|
|
read_json,
|
|
redact,
|
|
redact_argv,
|
|
resource_lock,
|
|
state_root,
|
|
)
|
|
from .environment import environment_fingerprint, execution_environment, resolve_tools
|
|
from .workspace import load_project
|
|
from .checkpoints import Checkpoints, FINGERPRINT_VERSION, validate_checkpoint_receipt
|
|
from .inputs import InputSnapshotter
|
|
from .monitoring import (
|
|
bounded_display,
|
|
capture_text,
|
|
elapsed_seconds,
|
|
latest_run,
|
|
list_runs,
|
|
read_live,
|
|
write_live,
|
|
)
|
|
|
|
MAX_LOG_BYTES = 8 * 1024 * 1024
|
|
TERMINAL = {
|
|
"passed",
|
|
"failed",
|
|
"timed_out",
|
|
"blocked",
|
|
"skipped",
|
|
"interrupted",
|
|
"stale",
|
|
}
|
|
|
|
|
|
def validate_stages(
|
|
stages: list[dict], workspace_root: Path, tools: dict[str, str]
|
|
) -> list[dict]:
|
|
if not isinstance(stages, list) or len(stages) > 512:
|
|
raise ValueError("A check plan must contain at most 512 stages")
|
|
result, ids = [], set()
|
|
for source in stages:
|
|
stage = deepcopy(source)
|
|
stage_id = identifier(stage.get("id"))
|
|
if stage_id in ids:
|
|
raise ValueError("Duplicate stage identifier")
|
|
ids.add(stage_id)
|
|
argv = stage.get("argv")
|
|
if (
|
|
not isinstance(argv, list)
|
|
or not argv
|
|
or len(argv) > 256
|
|
or any(
|
|
not isinstance(value, str) or "\0" in value or len(value) > 8192
|
|
for value in argv
|
|
)
|
|
):
|
|
raise ValueError(f"Invalid command for {stage_id}")
|
|
resolved = []
|
|
for value in argv:
|
|
for name, executable in tools.items():
|
|
value = value.replace("{" + name + "}", executable)
|
|
value = value.replace("{workspace}", str(workspace_root))
|
|
resolved.append(value)
|
|
cwd = Path(stage.get("cwd", str(workspace_root)))
|
|
if not cwd.is_absolute():
|
|
cwd = workspace_root / cwd
|
|
cwd = cwd.resolve()
|
|
if not cwd.is_relative_to(workspace_root.resolve()):
|
|
raise ValueError("Check working directory escapes the workspace")
|
|
timeout = stage.get("timeout_seconds", 3600)
|
|
if (
|
|
isinstance(timeout, bool)
|
|
or not isinstance(timeout, (int, float))
|
|
or not 0 < timeout <= 43200
|
|
):
|
|
raise ValueError("Check timeout must be between 0 and 43200 seconds")
|
|
for field in ("deps", "after", "resources"):
|
|
values = stage.get(field, [])
|
|
if not isinstance(values, list) or any(
|
|
not isinstance(value, str) or not value or len(value) > 256
|
|
for value in values
|
|
):
|
|
raise ValueError(f"Invalid stage {field}")
|
|
stage[field] = list(dict.fromkeys(values))
|
|
if stage.get("reuse", "verified") not in {"verified", "never"}:
|
|
raise ValueError("Stage reuse must be verified or never")
|
|
stage.setdefault("reuse", "verified")
|
|
stage.update(
|
|
id=stage_id,
|
|
argv=resolved,
|
|
cwd=str(cwd),
|
|
timeout_seconds=timeout,
|
|
title=str(stage.get("title", stage_id)),
|
|
reason=str(stage.get("reason", "Explicitly selected")),
|
|
)
|
|
result.append(stage)
|
|
for stage in result:
|
|
dependencies = set(stage["deps"]) | set(stage["after"])
|
|
if dependencies - ids or stage["id"] in dependencies:
|
|
raise ValueError("Stage has an unknown or self dependency")
|
|
if set(stage["deps"]) & set(stage["after"]):
|
|
raise ValueError(
|
|
"A dependency cannot be both data-dependent and order-only"
|
|
)
|
|
remaining = {item["id"]: set(item["deps"]) | set(item["after"]) for item in result}
|
|
while remaining:
|
|
ready = {key for key, deps in remaining.items() if not deps}
|
|
if not ready:
|
|
raise ValueError("Check dependencies contain a cycle")
|
|
remaining = {
|
|
key: deps - ready for key, deps in remaining.items() if key not in ready
|
|
}
|
|
return result
|
|
|
|
|
|
def _seal(receipt: dict) -> dict:
|
|
receipt["integrity_sha256"] = digest(
|
|
{key: value for key, value in receipt.items() if key != "integrity_sha256"}
|
|
)
|
|
return receipt
|
|
|
|
|
|
def read_receipt(workspace_root: Path, state_dir: Path | None, run_id: str) -> dict:
|
|
identifier(run_id)
|
|
path = state_root(workspace_root, state_dir) / "runs" / run_id / "receipt.json"
|
|
receipt = read_json(path)
|
|
if (
|
|
not isinstance(receipt, dict)
|
|
or receipt.get("schema_version") != 1
|
|
or receipt.get("run_id") != run_id
|
|
):
|
|
raise ValueError("Invalid check receipt")
|
|
if receipt.get("workspace_root") != str(workspace_root.resolve()):
|
|
raise ValueError("Check receipt belongs to a different workspace")
|
|
expected = digest(
|
|
{key: value for key, value in receipt.items() if key != "integrity_sha256"}
|
|
)
|
|
if receipt.get("integrity_sha256") != expected:
|
|
raise ValueError("Check receipt integrity mismatch")
|
|
if not isinstance(receipt.get("stages"), list) or len(receipt["stages"]) > 512:
|
|
raise ValueError("Invalid receipt stages")
|
|
if receipt.get("status") not in {
|
|
"running",
|
|
"passed",
|
|
"failed",
|
|
"interrupted",
|
|
"stale",
|
|
}:
|
|
raise ValueError("Invalid receipt run state")
|
|
if receipt["status"] == "passed" and (
|
|
receipt.get("snapshot_verified") is not True
|
|
or not receipt["stages"]
|
|
or any(
|
|
stage.get("status") != "passed"
|
|
for stage in receipt["stages"]
|
|
if isinstance(stage, dict)
|
|
)
|
|
):
|
|
raise ValueError(
|
|
"Passing receipt requires successful stages and a verified final snapshot"
|
|
)
|
|
if type(receipt.get("snapshot_verified")) is not bool:
|
|
raise ValueError("Receipt snapshot verification must be boolean")
|
|
stage_ids = set()
|
|
for stage in receipt["stages"]:
|
|
if not isinstance(stage, dict) or stage.get("status") not in TERMINAL | {
|
|
"pending",
|
|
"running",
|
|
}:
|
|
raise ValueError("Invalid receipt stage state")
|
|
identifier(stage.get("id"))
|
|
if stage["id"] in stage_ids:
|
|
raise ValueError("Duplicate receipt stage ID")
|
|
stage_ids.add(stage["id"])
|
|
if stage.get("exit_code") is not None and type(stage["exit_code"]) is not int:
|
|
raise ValueError("Check exit code must be an integer or null")
|
|
if stage["status"] == "passed" and stage.get("exit_code") != 0:
|
|
raise ValueError("Passing check receipt must have exit code zero")
|
|
validate_checkpoint_receipt(receipt)
|
|
return receipt
|
|
|
|
|
|
def _verified_log_bytes(stage: dict, root: Path) -> bytes:
|
|
raw = stage.get("log_path")
|
|
if not isinstance(raw, str) or not isinstance(stage.get("log_sha256"), str):
|
|
raise ValueError("Cached stage has no verifiable log; rerun without --resume")
|
|
path = Path(raw)
|
|
from .common import reject_symlinks
|
|
|
|
reject_symlinks(path)
|
|
if (
|
|
not path.is_absolute()
|
|
or not path.resolve().is_relative_to(root / "runs")
|
|
or not path.is_file()
|
|
):
|
|
raise ValueError("Cached log does not belong to this workspace's check runs")
|
|
encoded = read_bounded_bytes(path, MAX_LOG_BYTES + 65536)
|
|
if hashlib.sha256(encoded).hexdigest() != stage["log_sha256"]:
|
|
raise ValueError("Cached log integrity mismatch; rerun without --resume")
|
|
return encoded
|
|
|
|
|
|
def _verify_cached_log(stage: dict, root: Path) -> None:
|
|
_verified_log_bytes(stage, root)
|
|
|
|
|
|
def execute_stage(
|
|
stage: dict,
|
|
log_path: Path,
|
|
locks: Path,
|
|
env: dict[str, str],
|
|
cancelled: threading.Event,
|
|
checkpoints=None,
|
|
prior=None,
|
|
allow_reuse=True,
|
|
on_checkpoint=None,
|
|
verify_log=None,
|
|
) -> dict:
|
|
try:
|
|
with ExitStack() as stack:
|
|
for resource in sorted(stage["resources"]):
|
|
directory = (
|
|
locks.parent.parent / "host-resource-locks"
|
|
if resource.startswith(("browser:", "port:", "host:"))
|
|
else locks
|
|
)
|
|
stack.enter_context(resource_lock(directory, resource))
|
|
if cancelled.is_set():
|
|
return {
|
|
"status": "interrupted",
|
|
"exit_code": None,
|
|
"duration_seconds": 0,
|
|
"log_path": None,
|
|
"checkpoint_verified": False,
|
|
}
|
|
before = None
|
|
if checkpoints:
|
|
before, cached = checkpoints.prepare(
|
|
stage, prior, allow_reuse, verify_log
|
|
)
|
|
if cached:
|
|
cached["reused_from"] = stage["candidate_run_id"]
|
|
on_checkpoint(cached)
|
|
return cached
|
|
result = _execute_stage_command(stage, log_path, env, cancelled)
|
|
if checkpoints:
|
|
try:
|
|
result = checkpoints.finish(stage, before, result)
|
|
except InterruptedError as exc:
|
|
result.update(
|
|
status="interrupted",
|
|
checkpoint_verified=False,
|
|
error=redact(str(exc)),
|
|
)
|
|
except (
|
|
OSError,
|
|
ValueError,
|
|
RuntimeError,
|
|
subprocess.SubprocessError,
|
|
) as exc:
|
|
# Keep the actual completed command/log visible even when
|
|
# its post-execution input probe cannot certify a checkpoint.
|
|
result.update(
|
|
status="stale",
|
|
checkpoint_verified=False,
|
|
error="Stage checkpoint could not be verified: "
|
|
+ redact(str(exc)),
|
|
)
|
|
if result.get("checkpoint_verified"):
|
|
verify_log(result)
|
|
# Persist while the resource locks are still held, before another
|
|
# phase/process can consume or replace this phase's state.
|
|
on_checkpoint(result)
|
|
return result
|
|
except InterruptedError:
|
|
return {
|
|
"status": "interrupted",
|
|
"exit_code": None,
|
|
"duration_seconds": 0,
|
|
"log_path": None,
|
|
"checkpoint_verified": False,
|
|
}
|
|
except RuntimeError as exc:
|
|
return {
|
|
"status": "blocked",
|
|
"exit_code": None,
|
|
"error": redact(str(exc)),
|
|
"checkpoint_verified": False,
|
|
"log_path": None,
|
|
}
|
|
|
|
|
|
def _execute_stage_command(stage, log_path, env, cancelled) -> dict:
|
|
from .process import run_captured
|
|
|
|
started = time.monotonic()
|
|
state, code, truncated, output = "failed", None, False, b""
|
|
omitted_bytes, rendered = 0, None
|
|
try:
|
|
result = run_captured(
|
|
stage["argv"],
|
|
cwd=stage["cwd"],
|
|
env=env,
|
|
timeout=stage["timeout_seconds"],
|
|
max_stdout=MAX_LOG_BYTES,
|
|
cancelled=cancelled,
|
|
merge_stderr=True,
|
|
terminate_on_limit=False,
|
|
capture_mode="head_tail",
|
|
on_output=lambda snapshot: write_live(
|
|
log_path, stage["id"], snapshot, started
|
|
),
|
|
)
|
|
state = result.status if result.status in TERMINAL else "failed"
|
|
code, truncated, output = result.returncode, result.truncated, result.stdout
|
|
omitted_bytes = result.omitted_stdout_bytes
|
|
rendered = capture_text(result.snapshot())
|
|
if result.status == "leaked_process":
|
|
rendered += "\nA descendant outlived the command; the owned process group was terminated.\n"
|
|
except RuntimeError as exc:
|
|
state, output = "blocked", str(exc).encode()
|
|
except (OSError, subprocess.SubprocessError, ValueError) as exc:
|
|
output = f"{type(exc).__name__}: {exc}".encode()
|
|
text = (
|
|
rendered
|
|
if rendered is not None
|
|
else redact(output.decode("utf-8", errors="replace"))
|
|
)
|
|
encoded = text.encode("utf-8")
|
|
if len(encoded) > MAX_LOG_BYTES:
|
|
truncated = True
|
|
text = bounded_display(text, MAX_LOG_BYTES)
|
|
if truncated:
|
|
text += "\n[Output truncated at configured log bound; exit status is independent.]\n"
|
|
atomic_text(log_path, text, max_bytes=MAX_LOG_BYTES + 65536)
|
|
return {
|
|
"status": state,
|
|
"exit_code": code,
|
|
"duration_seconds": round(time.monotonic() - started, 3),
|
|
"log_path": str(log_path),
|
|
"output_truncated": truncated,
|
|
"omitted_output_bytes": omitted_bytes,
|
|
"log_sha256": hashlib.sha256(log_path.read_bytes()).hexdigest(),
|
|
}
|
|
|
|
|
|
def summarize(receipt: dict) -> dict:
|
|
counts = {}
|
|
for stage in receipt["stages"]:
|
|
counts[stage["status"]] = counts.get(stage["status"], 0) + 1
|
|
lines = [
|
|
f"Check run {receipt['run_id']}: {receipt['status']}",
|
|
", ".join(f"{count} {state}" for state, count in sorted(counts.items()))
|
|
or "No checks selected",
|
|
]
|
|
elapsed = elapsed_seconds(receipt)
|
|
lines.insert(
|
|
1,
|
|
f"Phase: {receipt.get('phase', 'checking' if receipt['status'] == 'running' else 'finished')}; elapsed: {elapsed}s",
|
|
)
|
|
if receipt.get("fingerprint_version") == FINGERPRINT_VERSION:
|
|
reused = sum(bool(stage.get("reused_from")) for stage in receipt["stages"])
|
|
verified = sum(
|
|
stage.get("checkpoint_verified") is True for stage in receipt["stages"]
|
|
)
|
|
lines.append(f"Verified phase checkpoints: {verified}; reused: {reused}")
|
|
if receipt.get("error") or receipt.get("invalidated_reason"):
|
|
lines.append(redact(str(receipt.get("error") or receipt["invalidated_reason"])))
|
|
for stage in receipt["stages"]:
|
|
if stage["status"] not in {"passed", "pending"}:
|
|
lines.append(
|
|
f"{stage['id']}: {stage['status']}"
|
|
+ (f" — {stage['log_path']}" if stage.get("log_path") else "")
|
|
)
|
|
notes = list(
|
|
dict.fromkeys(
|
|
note
|
|
for stage in receipt["stages"]
|
|
for note in stage.get("coverage_notes", [])
|
|
)
|
|
)
|
|
lines.extend(notes[:8])
|
|
if len(notes) > 8:
|
|
lines.append(
|
|
f"{len(notes) - 8} additional coverage notes are in the JSON receipt."
|
|
)
|
|
lines.append(
|
|
"Receipts describe local checks, not a signed security certification or approval to publish."
|
|
)
|
|
return {
|
|
**receipt,
|
|
"counts": counts,
|
|
"elapsed_seconds": elapsed,
|
|
"summary": lines,
|
|
"_exit_code": 0 if receipt["status"] in {"passed", "running", "planned"} else 1,
|
|
}
|
|
|
|
|
|
def input_scope_label(stage: dict) -> str:
|
|
if "inputs" not in stage:
|
|
return "whole workspace (conservative fallback)"
|
|
repos = stage["inputs"]["repos"]
|
|
if len(repos) <= 4:
|
|
return ", ".join(repos)
|
|
return f"{len(repos)} declared repositories (complete list in --json)"
|
|
|
|
|
|
def run_checks(args, stages: list[dict]) -> dict:
|
|
workspace_root = Path(args.workspace_root).resolve()
|
|
project = load_project(workspace_root, getattr(args, "project", None))
|
|
tools = resolve_tools(workspace_root, project)
|
|
env = execution_environment(workspace_root, project, tools)
|
|
plan = validate_stages(stages, workspace_root, tools)
|
|
input_validator = InputSnapshotter(project, workspace_root=workspace_root)
|
|
for stage in plan:
|
|
input_validator.scope(stage)
|
|
if getattr(args, "dry_run", False):
|
|
notes = list(
|
|
dict.fromkeys(
|
|
note for stage in plan for note in stage.get("coverage_notes", [])
|
|
)
|
|
)
|
|
return {
|
|
"status": "planned",
|
|
"stages": plan,
|
|
"coverage": getattr(args, "check_coverage", None),
|
|
"summary": (
|
|
[
|
|
f"{stage['id']}: {stage['title']} — {stage['reason']}"
|
|
+ "; inputs: "
|
|
+ input_scope_label(stage)
|
|
for stage in plan
|
|
]
|
|
or ["No checks selected"]
|
|
)
|
|
+ notes[:8],
|
|
}
|
|
if not plan:
|
|
return {
|
|
"status": "not_run",
|
|
"stages": [],
|
|
"summary": ["No checks selected; this is not a passing verification."],
|
|
"_exit_code": 0,
|
|
}
|
|
jobs = getattr(args, "jobs", 2)
|
|
if not 1 <= jobs <= 8:
|
|
raise ValueError("jobs must be between 1 and 8")
|
|
source = environment = None
|
|
base = state_root(workspace_root, getattr(args, "state_dir", None))
|
|
run_id = (
|
|
now().replace(":", "").replace("+", "-").replace(".", "-")
|
|
+ "-"
|
|
+ uuid.uuid4().hex[:8]
|
|
)
|
|
previous_id = getattr(args, "resume", None)
|
|
previous = (
|
|
read_receipt(workspace_root, getattr(args, "state_dir", None), previous_id)
|
|
if previous_id
|
|
else None
|
|
)
|
|
if previous:
|
|
if previous["status"] == "running":
|
|
raise ValueError(
|
|
"Run still reports running; use status to inspect it and explicitly recover interruption first"
|
|
)
|
|
run_dir = base / "runs" / run_id
|
|
private_directory(run_dir)
|
|
stages_state = []
|
|
previous_stages = (
|
|
{stage["id"]: stage for stage in previous["stages"]}
|
|
if previous and previous.get("fingerprint_version") == FINGERPRINT_VERSION
|
|
else {}
|
|
)
|
|
for stage in plan:
|
|
state = {
|
|
**stage,
|
|
"status": "pending",
|
|
"exit_code": None,
|
|
"duration_seconds": None,
|
|
"log_path": None,
|
|
"checkpoint_verified": False,
|
|
"candidate_run_id": previous_id,
|
|
}
|
|
stages_state.append(state)
|
|
receipt = {
|
|
"schema_version": 1,
|
|
"fingerprint_version": FINGERPRINT_VERSION,
|
|
"run_id": run_id,
|
|
"workspace_root": str(workspace_root),
|
|
"project_file": str(args.project.resolve())
|
|
if getattr(args, "project", None)
|
|
else None,
|
|
"source_fingerprint": source,
|
|
"environment_fingerprint": environment,
|
|
"plan_fingerprint": digest(plan),
|
|
"status": "running",
|
|
"phase": "preparing",
|
|
"stages": stages_state,
|
|
"generated_at": now(),
|
|
"finished_at": None,
|
|
"owner_pid": os.getpid(),
|
|
"resumed_from": previous_id,
|
|
"profile": getattr(args, "profile", None),
|
|
"selection": {
|
|
"repos": getattr(args, "repo", []),
|
|
"changed": getattr(args, "changed", False),
|
|
},
|
|
"snapshot_verified": False,
|
|
"coverage": getattr(args, "check_coverage", None),
|
|
}
|
|
saved_digest = None
|
|
receipt_lock = threading.RLock()
|
|
last_progress = 0.0
|
|
last_progress_key = None
|
|
|
|
def progress(force=False):
|
|
nonlocal last_progress, last_progress_key
|
|
callback = getattr(args, "on_progress", None)
|
|
if callback is None:
|
|
return
|
|
key = (receipt["phase"], tuple(stage["status"] for stage in stages_state))
|
|
elapsed = time.monotonic() - last_progress
|
|
if not force and (elapsed < 1 or (key == last_progress_key and elapsed < 15)):
|
|
return
|
|
counts = {}
|
|
for stage in stages_state:
|
|
counts[stage["status"]] = counts.get(stage["status"], 0) + 1
|
|
callback(
|
|
{
|
|
"event": "check_progress",
|
|
"run_id": run_id,
|
|
"phase": receipt["phase"],
|
|
"status": receipt["status"],
|
|
"counts": counts,
|
|
"total_stages": len(stages_state),
|
|
"elapsed_seconds": elapsed_seconds(receipt),
|
|
"active_stages": [
|
|
stage["id"]
|
|
for stage in stages_state
|
|
if stage["status"] == "running"
|
|
],
|
|
"receipt_path": str(run_dir / "receipt.json"),
|
|
}
|
|
)
|
|
last_progress, last_progress_key = time.monotonic(), key
|
|
|
|
def save():
|
|
with receipt_lock:
|
|
save_locked()
|
|
|
|
def save_locked():
|
|
nonlocal saved_digest
|
|
# Execution retains argv in memory; receipts never persist known credentials.
|
|
public = deepcopy(receipt)
|
|
for item in public["stages"]:
|
|
item["argv"] = redact_argv(item["argv"])
|
|
for field in ("title", "reason", "error"):
|
|
if field in item:
|
|
item[field] = redact(str(item[field]))
|
|
if "error" in public:
|
|
public["error"] = redact(str(public["error"]))
|
|
_seal(public)
|
|
if saved_digest != public["integrity_sha256"]:
|
|
atomic_json(run_dir / "receipt.json", public)
|
|
saved_digest = public["integrity_sha256"]
|
|
|
|
cancelled = threading.Event()
|
|
checkpoints = Checkpoints(
|
|
project,
|
|
workspace_root,
|
|
plan,
|
|
lambda: environment_fingerprint(workspace_root, project, tools, env),
|
|
cancelled,
|
|
)
|
|
|
|
def persist_checkpoint(stage, result):
|
|
with receipt_lock:
|
|
before = dict(stage)
|
|
try:
|
|
stage.update(result)
|
|
save()
|
|
except BaseException:
|
|
stage.clear()
|
|
stage.update(before)
|
|
raise
|
|
|
|
old_handlers = {}
|
|
if threading.current_thread() is threading.main_thread():
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
old_handlers[sig] = signal.signal(sig, lambda *_: cancelled.set())
|
|
try:
|
|
with resource_lock(base / "locks", "run:" + run_id):
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
try:
|
|
save()
|
|
progress(force=True)
|
|
snapshot, environment = checkpoints.initialize()
|
|
source = snapshot["observed_source_fingerprint"]
|
|
receipt.update(
|
|
source_fingerprint=source,
|
|
environment_fingerprint=environment,
|
|
source_scope={
|
|
"repos": sorted(snapshot["repository_fingerprints"]),
|
|
"complete_workspace": snapshot["complete_workspace"],
|
|
},
|
|
input_scan_stats=snapshot["scan_stats"],
|
|
)
|
|
receipt["phase"] = "checking"
|
|
save()
|
|
progress(force=True)
|
|
active = {}
|
|
while True:
|
|
by_id = {stage["id"]: stage for stage in stages_state}
|
|
active_ids = {stage["id"] for stage in active.values()}
|
|
active_resources = {
|
|
resource
|
|
for stage in active.values()
|
|
for resource in stage["resources"]
|
|
}
|
|
for stage in stages_state:
|
|
if stage["status"] != "pending":
|
|
continue
|
|
deps = [
|
|
by_id[dep]["status"]
|
|
for dep in stage["deps"] + stage["after"]
|
|
]
|
|
if cancelled.is_set() or any(
|
|
state in TERMINAL - {"passed"} for state in deps
|
|
):
|
|
with receipt_lock:
|
|
stage["status"] = (
|
|
"interrupted"
|
|
if cancelled.is_set()
|
|
else "skipped"
|
|
)
|
|
elif (
|
|
len(active) < jobs
|
|
and all(state == "passed" for state in deps)
|
|
# A checkpoint callback may be publishing its
|
|
# result. Consumers wait for the future to
|
|
# complete, including successful persistence.
|
|
and not active_ids.intersection(
|
|
stage["deps"] + stage["after"]
|
|
)
|
|
and not active_resources.intersection(
|
|
stage["resources"]
|
|
)
|
|
):
|
|
with receipt_lock:
|
|
stage["status"] = "running"
|
|
stage["started_at"] = now()
|
|
# Locks remain shared across different evidence directories.
|
|
future = pool.submit(
|
|
execute_stage,
|
|
stage,
|
|
run_dir / (stage["id"] + ".log"),
|
|
state_root(workspace_root) / "resource-locks",
|
|
env,
|
|
cancelled,
|
|
checkpoints,
|
|
previous_stages.get(stage["id"]),
|
|
all(
|
|
by_id[dep].get("reused_from")
|
|
for dep in stage["deps"]
|
|
),
|
|
lambda result, current=stage: (
|
|
persist_checkpoint(current, result)
|
|
),
|
|
lambda result: _verify_cached_log(result, base),
|
|
)
|
|
active[future] = stage
|
|
active_ids.add(stage["id"])
|
|
active_resources.update(stage["resources"])
|
|
save()
|
|
progress()
|
|
if not active:
|
|
break
|
|
done, _ = wait(
|
|
active, timeout=0.5, return_when=FIRST_COMPLETED
|
|
)
|
|
for future in done:
|
|
stage = active.pop(future)
|
|
try:
|
|
with receipt_lock:
|
|
stage.update(future.result())
|
|
except Exception as exc:
|
|
with receipt_lock:
|
|
stage.update(
|
|
status="failed",
|
|
checkpoint_verified=False,
|
|
error=f"{type(exc).__name__}: {redact(str(exc))}",
|
|
)
|
|
terminal_status = (
|
|
"interrupted"
|
|
if cancelled.is_set()
|
|
else "passed"
|
|
if all(
|
|
stage["status"] == "passed" for stage in stages_state
|
|
)
|
|
else "failed"
|
|
)
|
|
receipt["phase"] = "finalizing"
|
|
save()
|
|
progress(force=True)
|
|
try:
|
|
matches, final_snapshot, _ = checkpoints.finalize(
|
|
stages_state
|
|
)
|
|
receipt["final_input_scan_stats"] = final_snapshot[
|
|
"scan_stats"
|
|
]
|
|
except (
|
|
OSError,
|
|
ValueError,
|
|
RuntimeError,
|
|
subprocess.SubprocessError,
|
|
) as exc:
|
|
matches = False
|
|
receipt["invalidated_reason"] = (
|
|
"Final snapshot could not be verified: "
|
|
+ redact(str(exc))
|
|
)
|
|
if cancelled.is_set():
|
|
receipt.update(
|
|
status="interrupted", snapshot_verified=matches
|
|
)
|
|
elif matches:
|
|
receipt.update(
|
|
status=terminal_status, snapshot_verified=True
|
|
)
|
|
else:
|
|
receipt["status"] = "stale"
|
|
receipt.setdefault(
|
|
"invalidated_reason",
|
|
"Source or environment changed during verification; rerun checks against the final state",
|
|
)
|
|
finally:
|
|
# Signal owned processes before the executor waits for them, even on persistence errors.
|
|
cancelled.set()
|
|
except InterruptedError as exc:
|
|
receipt.update(status="interrupted", error=redact(str(exc)))
|
|
except Exception as exc:
|
|
receipt.update(
|
|
status="failed", error=f"{type(exc).__name__}: {redact(str(exc))}"
|
|
)
|
|
raise
|
|
finally:
|
|
if receipt["status"] == "running":
|
|
receipt["status"] = "interrupted"
|
|
for stage in stages_state:
|
|
if stage["status"] in {"pending", "running"}:
|
|
stage["status"] = "interrupted"
|
|
receipt["finished_at"] = now()
|
|
receipt["phase"] = "finished"
|
|
save() # The run lock is held through final persistence and cleanup.
|
|
progress(force=True)
|
|
finally:
|
|
for sig, handler in old_handlers.items():
|
|
signal.signal(sig, handler)
|
|
result = summarize(
|
|
read_receipt(workspace_root, getattr(args, "state_dir", None), run_id)
|
|
)
|
|
result["receipt_path"] = str(run_dir / "receipt.json")
|
|
result["summary"].append("Receipt: " + result["receipt_path"])
|
|
return result
|
|
|
|
|
|
def register(subparsers):
|
|
check = subparsers.add_parser(
|
|
"check", help="Run registered checks, keeping bounded logs and evidence"
|
|
)
|
|
check.add_argument(
|
|
"--profile", choices=("quick", "ui", "backend", "full"), default="quick"
|
|
)
|
|
check.add_argument("--repo", action="append", default=[])
|
|
check.add_argument("--changed", action="store_true")
|
|
check.add_argument("--jobs", type=int, default=2)
|
|
check.add_argument("--dry-run", action="store_true")
|
|
check.add_argument(
|
|
"--resume",
|
|
metavar="RUN",
|
|
help="Reuse verified phase checkpoints whose declared inputs and environment still match",
|
|
)
|
|
|
|
def handle(args):
|
|
from .catalog import build_stages, build_coverage
|
|
|
|
stages = build_stages(
|
|
args.workspace_root, args.profile, args.repo, args.changed, args.project
|
|
)
|
|
args.check_coverage = build_coverage(
|
|
args.workspace_root,
|
|
args.profile,
|
|
args.repo,
|
|
args.changed,
|
|
args.project,
|
|
stages=stages,
|
|
)
|
|
if stages:
|
|
stages[0]["coverage_notes"] = list(
|
|
dict.fromkeys(
|
|
[
|
|
*stages[0].get("coverage_notes", []),
|
|
*args.check_coverage["notes"],
|
|
]
|
|
)
|
|
)
|
|
return run_checks(args, stages)
|
|
|
|
check.set_defaults(handler=handle)
|
|
coverage = subparsers.add_parser(
|
|
"coverage",
|
|
help="Inventory declared suites, planned coverage, exclusions and unsupported commands",
|
|
)
|
|
coverage.add_argument(
|
|
"--profile", choices=("quick", "ui", "backend", "full"), default="quick"
|
|
)
|
|
coverage.add_argument("--repo", action="append", default=[])
|
|
coverage.add_argument("--changed", action="store_true")
|
|
|
|
def coverage_handler(args):
|
|
from .catalog import build_coverage
|
|
|
|
result = build_coverage(
|
|
args.workspace_root, args.profile, args.repo, args.changed, args.project
|
|
)
|
|
omitted = [
|
|
item
|
|
for item in result["suites"]
|
|
if item["disposition"] in {"excluded", "unsupported"}
|
|
]
|
|
result["summary"] = [
|
|
"Declared-suite coverage for "
|
|
+ args.profile
|
|
+ " (plan only, not execution evidence)",
|
|
", ".join(
|
|
f"{count} {name}" for name, count in sorted(result["counts"].items())
|
|
),
|
|
*result["notes"],
|
|
*[
|
|
f"{item['repo']} / {item['name']}: {item['disposition']} — {item['reason']}"
|
|
for item in omitted[:12]
|
|
],
|
|
]
|
|
if len(omitted) > 12:
|
|
result["summary"].append(
|
|
f"{len(omitted) - 12} additional exclusions/unsupported suites are listed with --json."
|
|
)
|
|
return result
|
|
|
|
coverage.set_defaults(handler=coverage_handler)
|
|
runs = subparsers.add_parser(
|
|
"runs", help="List recent local runs with bounded cursor pagination"
|
|
)
|
|
runs.add_argument("--limit", type=int, default=10)
|
|
runs.add_argument(
|
|
"--before", help="Exclusive run-ID cursor from the preceding page"
|
|
)
|
|
runs.set_defaults(handler=list_runs)
|
|
latest = subparsers.add_parser(
|
|
"latest", help="Read the newest matching run; never substitute an older pass"
|
|
)
|
|
latest.set_defaults(handler=latest_run)
|
|
for name in ("status", "summary"):
|
|
parser = subparsers.add_parser(
|
|
name, help="Read a check run without reading or replaying its full logs"
|
|
)
|
|
parser.add_argument("run_id")
|
|
parser.set_defaults(
|
|
handler=lambda args: summarize(
|
|
read_receipt(args.workspace_root, args.state_dir, args.run_id)
|
|
)
|
|
)
|
|
resume = subparsers.add_parser(
|
|
"resume",
|
|
help="Replan a previous check selection and reuse only verified unchanged results",
|
|
)
|
|
resume.add_argument("run_id")
|
|
resume.add_argument("--jobs", type=int, default=2)
|
|
resume.add_argument("--dry-run", action="store_true")
|
|
|
|
def resume_handler(args):
|
|
previous = read_receipt(args.workspace_root, args.state_dir, args.run_id)
|
|
if previous.get("profile") not in {"quick", "ui", "backend", "full"}:
|
|
raise ValueError(
|
|
"This run has no reusable check profile; repeat its owning audit command"
|
|
)
|
|
previous_project = previous.get("project_file")
|
|
if args.project and str(args.project.resolve()) != previous_project:
|
|
raise ValueError("Resume project differs from the recorded project")
|
|
args.project = Path(previous_project) if previous_project else None
|
|
args.profile = previous["profile"]
|
|
args.repo = previous.get("selection", {}).get("repos", [])
|
|
args.changed = previous.get("selection", {}).get("changed", False)
|
|
args.resume = args.run_id
|
|
return handle(args)
|
|
|
|
resume.set_defaults(handler=resume_handler)
|
|
recover = subparsers.add_parser(
|
|
"recover", help="Mark an abandoned check run interrupted; never rerun a process"
|
|
)
|
|
recover.add_argument("run_id")
|
|
recover.add_argument("--apply", action="store_true")
|
|
recover.add_argument(
|
|
"--confirm-processes-stopped",
|
|
action="store_true",
|
|
help="Attest that surviving commands from the abandoned run were manually checked and stopped",
|
|
)
|
|
|
|
def recover_handler(args):
|
|
base = state_root(args.workspace_root, args.state_dir)
|
|
previous = read_receipt(args.workspace_root, args.state_dir, args.run_id)
|
|
if previous["status"] != "running":
|
|
return {
|
|
"status": "unchanged",
|
|
"summary": ["Run does not require interrupted-owner recovery."],
|
|
}
|
|
if not args.apply:
|
|
return {
|
|
"status": "planned",
|
|
"summary": [
|
|
"Abandoned run owner PID: " + str(previous.get("owner_pid")),
|
|
"A free run lock does not prove its child processes stopped after a hard kill. Inspect the recorded commands and stop any survivors before using --apply --confirm-processes-stopped. No checks will execute during recovery.",
|
|
],
|
|
}
|
|
if not args.confirm_processes_stopped:
|
|
raise ValueError(
|
|
"Hard-crash recovery requires manual verification that the recorded commands and surviving child processes stopped; then use --apply --confirm-processes-stopped"
|
|
)
|
|
with resource_lock(base / "locks", "run:" + args.run_id):
|
|
previous = read_receipt(args.workspace_root, args.state_dir, args.run_id)
|
|
if previous["status"] != "running":
|
|
return {
|
|
"status": "unchanged",
|
|
"summary": [
|
|
"Run completed before recovery acquired its lock; no state was changed."
|
|
],
|
|
}
|
|
previous.update(
|
|
status="interrupted",
|
|
phase="finished",
|
|
finished_at=now(),
|
|
snapshot_verified=False,
|
|
)
|
|
for item in previous["stages"]:
|
|
if item["status"] in {"running", "pending"}:
|
|
item["status"] = "interrupted"
|
|
atomic_json(base / "runs" / args.run_id / "receipt.json", _seal(previous))
|
|
result = summarize(previous)
|
|
result["summary"].append(
|
|
"The interrupted aggregate is not passing evidence. Resume may reuse independently verified matching phase checkpoints; other stages run again."
|
|
)
|
|
return result
|
|
|
|
recover.set_defaults(handler=recover_handler)
|
|
logs = subparsers.add_parser(
|
|
"logs",
|
|
help="Read bounded provisional output or a hash-verified final stage log",
|
|
)
|
|
logs.add_argument("run_id")
|
|
logs.add_argument("--stage", required=True)
|
|
logs.add_argument("--tail", type=int, default=40)
|
|
logs.add_argument(
|
|
"--final-only",
|
|
action="store_true",
|
|
help="Require a finalized, hash-verified stage log",
|
|
)
|
|
|
|
def logs_handler(args):
|
|
if not 1 <= args.tail <= 200:
|
|
raise ValueError("Log tail must be between 1 and 200 lines")
|
|
receipt = read_receipt(args.workspace_root, args.state_dir, args.run_id)
|
|
selected = next(
|
|
(stage for stage in receipt["stages"] if stage["id"] == args.stage), None
|
|
)
|
|
if not selected:
|
|
raise ValueError("Stage does not belong to this run")
|
|
if selected.get("log_sha256"):
|
|
text = _verified_log_bytes(
|
|
selected, state_root(args.workspace_root, args.state_dir)
|
|
).decode("utf-8", errors="replace")
|
|
details = {"provisional": False, "log_verified": True}
|
|
else:
|
|
if args.final_only:
|
|
raise ValueError(
|
|
"No finalized stage log is available yet; live output is provisional"
|
|
)
|
|
details = read_live(
|
|
args.workspace_root, args.state_dir, args.run_id, args.stage
|
|
)
|
|
text = details.pop("excerpt")
|
|
details["log_verified"] = False
|
|
excerpt = bounded_display(
|
|
redact("\n".join(text.splitlines()[-args.tail :])), 16384, tail_only=True
|
|
)
|
|
label = (
|
|
"Provisional live output — not verified evidence."
|
|
if details["provisional"]
|
|
else "Final stage log — hash verified; consult run status for overall verification."
|
|
)
|
|
return {
|
|
**details,
|
|
"run_id": args.run_id,
|
|
"stage": args.stage,
|
|
"run_status": receipt["status"],
|
|
"snapshot_verified": receipt["snapshot_verified"],
|
|
"excerpt": excerpt,
|
|
"summary": [label, excerpt],
|
|
}
|
|
|
|
logs.set_defaults(handler=logs_handler)
|