"""Resource-aware checks with bounded logs and resumable, source-bound receipts.""" from __future__ import annotations from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED from contextlib import ExitStack from copy import deepcopy import os import hashlib from pathlib import Path import signal import subprocess import threading import time import uuid from .common import ( atomic_json, atomic_text, digest, identifier, now, private_directory, read_bounded_bytes, read_json, redact, redact_argv, resource_lock, state_root, ) from .environment import environment_fingerprint, execution_environment, resolve_tools from .workspace import load_project from .checkpoints import Checkpoints, FINGERPRINT_VERSION, validate_checkpoint_receipt from .inputs import InputSnapshotter from .monitoring import ( bounded_display, capture_text, elapsed_seconds, latest_run, list_runs, read_live, write_live, ) MAX_LOG_BYTES = 8 * 1024 * 1024 TERMINAL = { "passed", "failed", "timed_out", "blocked", "skipped", "interrupted", "stale", } def validate_stages( stages: list[dict], workspace_root: Path, tools: dict[str, str] ) -> list[dict]: if not isinstance(stages, list) or len(stages) > 512: raise ValueError("A check plan must contain at most 512 stages") result, ids = [], set() for source in stages: stage = deepcopy(source) stage_id = identifier(stage.get("id")) if stage_id in ids: raise ValueError("Duplicate stage identifier") ids.add(stage_id) argv = stage.get("argv") if ( not isinstance(argv, list) or not argv or len(argv) > 256 or any( not isinstance(value, str) or "\0" in value or len(value) > 8192 for value in argv ) ): raise ValueError(f"Invalid command for {stage_id}") resolved = [] for value in argv: for name, executable in tools.items(): value = value.replace("{" + name + "}", executable) value = value.replace("{workspace}", str(workspace_root)) resolved.append(value) cwd = Path(stage.get("cwd", str(workspace_root))) if not cwd.is_absolute(): cwd = workspace_root / cwd cwd = cwd.resolve() if not cwd.is_relative_to(workspace_root.resolve()): raise ValueError("Check working directory escapes the workspace") timeout = stage.get("timeout_seconds", 3600) if ( isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0 < timeout <= 43200 ): raise ValueError("Check timeout must be between 0 and 43200 seconds") for field in ("deps", "after", "resources"): values = stage.get(field, []) if not isinstance(values, list) or any( not isinstance(value, str) or not value or len(value) > 256 for value in values ): raise ValueError(f"Invalid stage {field}") stage[field] = list(dict.fromkeys(values)) if stage.get("reuse", "verified") not in {"verified", "never"}: raise ValueError("Stage reuse must be verified or never") stage.setdefault("reuse", "verified") stage.update( id=stage_id, argv=resolved, cwd=str(cwd), timeout_seconds=timeout, title=str(stage.get("title", stage_id)), reason=str(stage.get("reason", "Explicitly selected")), ) result.append(stage) for stage in result: dependencies = set(stage["deps"]) | set(stage["after"]) if dependencies - ids or stage["id"] in dependencies: raise ValueError("Stage has an unknown or self dependency") if set(stage["deps"]) & set(stage["after"]): raise ValueError( "A dependency cannot be both data-dependent and order-only" ) remaining = {item["id"]: set(item["deps"]) | set(item["after"]) for item in result} while remaining: ready = {key for key, deps in remaining.items() if not deps} if not ready: raise ValueError("Check dependencies contain a cycle") remaining = { key: deps - ready for key, deps in remaining.items() if key not in ready } return result def _seal(receipt: dict) -> dict: receipt["integrity_sha256"] = digest( {key: value for key, value in receipt.items() if key != "integrity_sha256"} ) return receipt def read_receipt(workspace_root: Path, state_dir: Path | None, run_id: str) -> dict: identifier(run_id) path = state_root(workspace_root, state_dir) / "runs" / run_id / "receipt.json" receipt = read_json(path) if ( not isinstance(receipt, dict) or receipt.get("schema_version") != 1 or receipt.get("run_id") != run_id ): raise ValueError("Invalid check receipt") if receipt.get("workspace_root") != str(workspace_root.resolve()): raise ValueError("Check receipt belongs to a different workspace") expected = digest( {key: value for key, value in receipt.items() if key != "integrity_sha256"} ) if receipt.get("integrity_sha256") != expected: raise ValueError("Check receipt integrity mismatch") if not isinstance(receipt.get("stages"), list) or len(receipt["stages"]) > 512: raise ValueError("Invalid receipt stages") if receipt.get("status") not in { "running", "passed", "failed", "interrupted", "stale", }: raise ValueError("Invalid receipt run state") if receipt["status"] == "passed" and ( receipt.get("snapshot_verified") is not True or not receipt["stages"] or any( stage.get("status") != "passed" for stage in receipt["stages"] if isinstance(stage, dict) ) ): raise ValueError( "Passing receipt requires successful stages and a verified final snapshot" ) if type(receipt.get("snapshot_verified")) is not bool: raise ValueError("Receipt snapshot verification must be boolean") stage_ids = set() for stage in receipt["stages"]: if not isinstance(stage, dict) or stage.get("status") not in TERMINAL | { "pending", "running", }: raise ValueError("Invalid receipt stage state") identifier(stage.get("id")) if stage["id"] in stage_ids: raise ValueError("Duplicate receipt stage ID") stage_ids.add(stage["id"]) if stage.get("exit_code") is not None and type(stage["exit_code"]) is not int: raise ValueError("Check exit code must be an integer or null") if stage["status"] == "passed" and stage.get("exit_code") != 0: raise ValueError("Passing check receipt must have exit code zero") validate_checkpoint_receipt(receipt) return receipt def _verified_log_bytes(stage: dict, root: Path) -> bytes: raw = stage.get("log_path") if not isinstance(raw, str) or not isinstance(stage.get("log_sha256"), str): raise ValueError("Cached stage has no verifiable log; rerun without --resume") path = Path(raw) from .common import reject_symlinks reject_symlinks(path) if ( not path.is_absolute() or not path.resolve().is_relative_to(root / "runs") or not path.is_file() ): raise ValueError("Cached log does not belong to this workspace's check runs") encoded = read_bounded_bytes(path, MAX_LOG_BYTES + 65536) if hashlib.sha256(encoded).hexdigest() != stage["log_sha256"]: raise ValueError("Cached log integrity mismatch; rerun without --resume") return encoded def _verify_cached_log(stage: dict, root: Path) -> None: _verified_log_bytes(stage, root) def execute_stage( stage: dict, log_path: Path, locks: Path, env: dict[str, str], cancelled: threading.Event, checkpoints=None, prior=None, allow_reuse=True, on_checkpoint=None, verify_log=None, ) -> dict: try: with ExitStack() as stack: for resource in sorted(stage["resources"]): directory = ( locks.parent.parent / "host-resource-locks" if resource.startswith(("browser:", "port:", "host:")) else locks ) stack.enter_context(resource_lock(directory, resource)) if cancelled.is_set(): return { "status": "interrupted", "exit_code": None, "duration_seconds": 0, "log_path": None, "checkpoint_verified": False, } before = None if checkpoints: before, cached = checkpoints.prepare( stage, prior, allow_reuse, verify_log ) if cached: cached["reused_from"] = stage["candidate_run_id"] on_checkpoint(cached) return cached result = _execute_stage_command(stage, log_path, env, cancelled) if checkpoints: try: result = checkpoints.finish(stage, before, result) except InterruptedError as exc: result.update( status="interrupted", checkpoint_verified=False, error=redact(str(exc)), ) except ( OSError, ValueError, RuntimeError, subprocess.SubprocessError, ) as exc: # Keep the actual completed command/log visible even when # its post-execution input probe cannot certify a checkpoint. result.update( status="stale", checkpoint_verified=False, error="Stage checkpoint could not be verified: " + redact(str(exc)), ) if result.get("checkpoint_verified"): verify_log(result) # Persist while the resource locks are still held, before another # phase/process can consume or replace this phase's state. on_checkpoint(result) return result except InterruptedError: return { "status": "interrupted", "exit_code": None, "duration_seconds": 0, "log_path": None, "checkpoint_verified": False, } except RuntimeError as exc: return { "status": "blocked", "exit_code": None, "error": redact(str(exc)), "checkpoint_verified": False, "log_path": None, } def _execute_stage_command(stage, log_path, env, cancelled) -> dict: from .process import run_captured started = time.monotonic() state, code, truncated, output = "failed", None, False, b"" omitted_bytes, rendered = 0, None try: result = run_captured( stage["argv"], cwd=stage["cwd"], env=env, timeout=stage["timeout_seconds"], max_stdout=MAX_LOG_BYTES, cancelled=cancelled, merge_stderr=True, terminate_on_limit=False, capture_mode="head_tail", on_output=lambda snapshot: write_live( log_path, stage["id"], snapshot, started ), ) state = result.status if result.status in TERMINAL else "failed" code, truncated, output = result.returncode, result.truncated, result.stdout omitted_bytes = result.omitted_stdout_bytes rendered = capture_text(result.snapshot()) if result.status == "leaked_process": rendered += "\nA descendant outlived the command; the owned process group was terminated.\n" except RuntimeError as exc: state, output = "blocked", str(exc).encode() except (OSError, subprocess.SubprocessError, ValueError) as exc: output = f"{type(exc).__name__}: {exc}".encode() text = ( rendered if rendered is not None else redact(output.decode("utf-8", errors="replace")) ) encoded = text.encode("utf-8") if len(encoded) > MAX_LOG_BYTES: truncated = True text = bounded_display(text, MAX_LOG_BYTES) if truncated: text += "\n[Output truncated at configured log bound; exit status is independent.]\n" atomic_text(log_path, text, max_bytes=MAX_LOG_BYTES + 65536) return { "status": state, "exit_code": code, "duration_seconds": round(time.monotonic() - started, 3), "log_path": str(log_path), "output_truncated": truncated, "omitted_output_bytes": omitted_bytes, "log_sha256": hashlib.sha256(log_path.read_bytes()).hexdigest(), } def summarize(receipt: dict) -> dict: counts = {} for stage in receipt["stages"]: counts[stage["status"]] = counts.get(stage["status"], 0) + 1 lines = [ f"Check run {receipt['run_id']}: {receipt['status']}", ", ".join(f"{count} {state}" for state, count in sorted(counts.items())) or "No checks selected", ] elapsed = elapsed_seconds(receipt) lines.insert( 1, f"Phase: {receipt.get('phase', 'checking' if receipt['status'] == 'running' else 'finished')}; elapsed: {elapsed}s", ) if receipt.get("fingerprint_version") == FINGERPRINT_VERSION: reused = sum(bool(stage.get("reused_from")) for stage in receipt["stages"]) verified = sum( stage.get("checkpoint_verified") is True for stage in receipt["stages"] ) lines.append(f"Verified phase checkpoints: {verified}; reused: {reused}") if receipt.get("error") or receipt.get("invalidated_reason"): lines.append(redact(str(receipt.get("error") or receipt["invalidated_reason"]))) for stage in receipt["stages"]: if stage["status"] not in {"passed", "pending"}: lines.append( f"{stage['id']}: {stage['status']}" + (f" — {stage['log_path']}" if stage.get("log_path") else "") ) notes = list( dict.fromkeys( note for stage in receipt["stages"] for note in stage.get("coverage_notes", []) ) ) lines.extend(notes[:8]) if len(notes) > 8: lines.append( f"{len(notes) - 8} additional coverage notes are in the JSON receipt." ) lines.append( "Receipts describe local checks, not a signed security certification or approval to publish." ) return { **receipt, "counts": counts, "elapsed_seconds": elapsed, "summary": lines, "_exit_code": 0 if receipt["status"] in {"passed", "running", "planned"} else 1, } def input_scope_label(stage: dict) -> str: if "inputs" not in stage: return "whole workspace (conservative fallback)" repos = stage["inputs"]["repos"] if len(repos) <= 4: return ", ".join(repos) return f"{len(repos)} declared repositories (complete list in --json)" def run_checks(args, stages: list[dict]) -> dict: workspace_root = Path(args.workspace_root).resolve() project = load_project(workspace_root, getattr(args, "project", None)) tools = resolve_tools(workspace_root, project) env = execution_environment(workspace_root, project, tools) plan = validate_stages(stages, workspace_root, tools) input_validator = InputSnapshotter(project, workspace_root=workspace_root) for stage in plan: input_validator.scope(stage) if getattr(args, "dry_run", False): notes = list( dict.fromkeys( note for stage in plan for note in stage.get("coverage_notes", []) ) ) return { "status": "planned", "stages": plan, "coverage": getattr(args, "check_coverage", None), "summary": ( [ f"{stage['id']}: {stage['title']} — {stage['reason']}" + "; inputs: " + input_scope_label(stage) for stage in plan ] or ["No checks selected"] ) + notes[:8], } if not plan: return { "status": "not_run", "stages": [], "summary": ["No checks selected; this is not a passing verification."], "_exit_code": 0, } jobs = getattr(args, "jobs", 2) if not 1 <= jobs <= 8: raise ValueError("jobs must be between 1 and 8") source = environment = None base = state_root(workspace_root, getattr(args, "state_dir", None)) run_id = ( now().replace(":", "").replace("+", "-").replace(".", "-") + "-" + uuid.uuid4().hex[:8] ) previous_id = getattr(args, "resume", None) previous = ( read_receipt(workspace_root, getattr(args, "state_dir", None), previous_id) if previous_id else None ) if previous: if previous["status"] == "running": raise ValueError( "Run still reports running; use status to inspect it and explicitly recover interruption first" ) run_dir = base / "runs" / run_id private_directory(run_dir) stages_state = [] previous_stages = ( {stage["id"]: stage for stage in previous["stages"]} if previous and previous.get("fingerprint_version") == FINGERPRINT_VERSION else {} ) for stage in plan: state = { **stage, "status": "pending", "exit_code": None, "duration_seconds": None, "log_path": None, "checkpoint_verified": False, "candidate_run_id": previous_id, } stages_state.append(state) receipt = { "schema_version": 1, "fingerprint_version": FINGERPRINT_VERSION, "run_id": run_id, "workspace_root": str(workspace_root), "project_file": str(args.project.resolve()) if getattr(args, "project", None) else None, "source_fingerprint": source, "environment_fingerprint": environment, "plan_fingerprint": digest(plan), "status": "running", "phase": "preparing", "stages": stages_state, "generated_at": now(), "finished_at": None, "owner_pid": os.getpid(), "resumed_from": previous_id, "profile": getattr(args, "profile", None), "selection": { "repos": getattr(args, "repo", []), "changed": getattr(args, "changed", False), }, "snapshot_verified": False, "coverage": getattr(args, "check_coverage", None), } saved_digest = None receipt_lock = threading.RLock() last_progress = 0.0 last_progress_key = None def progress(force=False): nonlocal last_progress, last_progress_key callback = getattr(args, "on_progress", None) if callback is None: return key = (receipt["phase"], tuple(stage["status"] for stage in stages_state)) elapsed = time.monotonic() - last_progress if not force and (elapsed < 1 or (key == last_progress_key and elapsed < 15)): return counts = {} for stage in stages_state: counts[stage["status"]] = counts.get(stage["status"], 0) + 1 callback( { "event": "check_progress", "run_id": run_id, "phase": receipt["phase"], "status": receipt["status"], "counts": counts, "total_stages": len(stages_state), "elapsed_seconds": elapsed_seconds(receipt), "active_stages": [ stage["id"] for stage in stages_state if stage["status"] == "running" ], "receipt_path": str(run_dir / "receipt.json"), } ) last_progress, last_progress_key = time.monotonic(), key def save(): with receipt_lock: save_locked() def save_locked(): nonlocal saved_digest # Execution retains argv in memory; receipts never persist known credentials. public = deepcopy(receipt) for item in public["stages"]: item["argv"] = redact_argv(item["argv"]) for field in ("title", "reason", "error"): if field in item: item[field] = redact(str(item[field])) if "error" in public: public["error"] = redact(str(public["error"])) _seal(public) if saved_digest != public["integrity_sha256"]: atomic_json(run_dir / "receipt.json", public) saved_digest = public["integrity_sha256"] cancelled = threading.Event() checkpoints = Checkpoints( project, workspace_root, plan, lambda: environment_fingerprint(workspace_root, project, tools, env), cancelled, ) def persist_checkpoint(stage, result): with receipt_lock: before = dict(stage) try: stage.update(result) save() except BaseException: stage.clear() stage.update(before) raise old_handlers = {} if threading.current_thread() is threading.main_thread(): for sig in (signal.SIGINT, signal.SIGTERM): old_handlers[sig] = signal.signal(sig, lambda *_: cancelled.set()) try: with resource_lock(base / "locks", "run:" + run_id): try: with ThreadPoolExecutor(max_workers=jobs) as pool: try: save() progress(force=True) snapshot, environment = checkpoints.initialize() source = snapshot["observed_source_fingerprint"] receipt.update( source_fingerprint=source, environment_fingerprint=environment, source_scope={ "repos": sorted(snapshot["repository_fingerprints"]), "complete_workspace": snapshot["complete_workspace"], }, input_scan_stats=snapshot["scan_stats"], ) receipt["phase"] = "checking" save() progress(force=True) active = {} while True: by_id = {stage["id"]: stage for stage in stages_state} active_ids = {stage["id"] for stage in active.values()} active_resources = { resource for stage in active.values() for resource in stage["resources"] } for stage in stages_state: if stage["status"] != "pending": continue deps = [ by_id[dep]["status"] for dep in stage["deps"] + stage["after"] ] if cancelled.is_set() or any( state in TERMINAL - {"passed"} for state in deps ): with receipt_lock: stage["status"] = ( "interrupted" if cancelled.is_set() else "skipped" ) elif ( len(active) < jobs and all(state == "passed" for state in deps) # A checkpoint callback may be publishing its # result. Consumers wait for the future to # complete, including successful persistence. and not active_ids.intersection( stage["deps"] + stage["after"] ) and not active_resources.intersection( stage["resources"] ) ): with receipt_lock: stage["status"] = "running" stage["started_at"] = now() # Locks remain shared across different evidence directories. future = pool.submit( execute_stage, stage, run_dir / (stage["id"] + ".log"), state_root(workspace_root) / "resource-locks", env, cancelled, checkpoints, previous_stages.get(stage["id"]), all( by_id[dep].get("reused_from") for dep in stage["deps"] ), lambda result, current=stage: ( persist_checkpoint(current, result) ), lambda result: _verify_cached_log(result, base), ) active[future] = stage active_ids.add(stage["id"]) active_resources.update(stage["resources"]) save() progress() if not active: break done, _ = wait( active, timeout=0.5, return_when=FIRST_COMPLETED ) for future in done: stage = active.pop(future) try: with receipt_lock: stage.update(future.result()) except Exception as exc: with receipt_lock: stage.update( status="failed", checkpoint_verified=False, error=f"{type(exc).__name__}: {redact(str(exc))}", ) terminal_status = ( "interrupted" if cancelled.is_set() else "passed" if all( stage["status"] == "passed" for stage in stages_state ) else "failed" ) receipt["phase"] = "finalizing" save() progress(force=True) try: matches, final_snapshot, _ = checkpoints.finalize( stages_state ) receipt["final_input_scan_stats"] = final_snapshot[ "scan_stats" ] except ( OSError, ValueError, RuntimeError, subprocess.SubprocessError, ) as exc: matches = False receipt["invalidated_reason"] = ( "Final snapshot could not be verified: " + redact(str(exc)) ) if cancelled.is_set(): receipt.update( status="interrupted", snapshot_verified=matches ) elif matches: receipt.update( status=terminal_status, snapshot_verified=True ) else: receipt["status"] = "stale" receipt.setdefault( "invalidated_reason", "Source or environment changed during verification; rerun checks against the final state", ) finally: # Signal owned processes before the executor waits for them, even on persistence errors. cancelled.set() except InterruptedError as exc: receipt.update(status="interrupted", error=redact(str(exc))) except Exception as exc: receipt.update( status="failed", error=f"{type(exc).__name__}: {redact(str(exc))}" ) raise finally: if receipt["status"] == "running": receipt["status"] = "interrupted" for stage in stages_state: if stage["status"] in {"pending", "running"}: stage["status"] = "interrupted" receipt["finished_at"] = now() receipt["phase"] = "finished" save() # The run lock is held through final persistence and cleanup. progress(force=True) finally: for sig, handler in old_handlers.items(): signal.signal(sig, handler) result = summarize( read_receipt(workspace_root, getattr(args, "state_dir", None), run_id) ) result["receipt_path"] = str(run_dir / "receipt.json") result["summary"].append("Receipt: " + result["receipt_path"]) return result def register(subparsers): check = subparsers.add_parser( "check", help="Run registered checks, keeping bounded logs and evidence" ) check.add_argument( "--profile", choices=("quick", "ui", "backend", "full"), default="quick" ) check.add_argument("--repo", action="append", default=[]) check.add_argument("--changed", action="store_true") check.add_argument("--jobs", type=int, default=2) check.add_argument("--dry-run", action="store_true") check.add_argument( "--resume", metavar="RUN", help="Reuse verified phase checkpoints whose declared inputs and environment still match", ) def handle(args): from .catalog import build_stages, build_coverage stages = build_stages( args.workspace_root, args.profile, args.repo, args.changed, args.project ) args.check_coverage = build_coverage( args.workspace_root, args.profile, args.repo, args.changed, args.project, stages=stages, ) if stages: stages[0]["coverage_notes"] = list( dict.fromkeys( [ *stages[0].get("coverage_notes", []), *args.check_coverage["notes"], ] ) ) return run_checks(args, stages) check.set_defaults(handler=handle) coverage = subparsers.add_parser( "coverage", help="Inventory declared suites, planned coverage, exclusions and unsupported commands", ) coverage.add_argument( "--profile", choices=("quick", "ui", "backend", "full"), default="quick" ) coverage.add_argument("--repo", action="append", default=[]) coverage.add_argument("--changed", action="store_true") def coverage_handler(args): from .catalog import build_coverage result = build_coverage( args.workspace_root, args.profile, args.repo, args.changed, args.project ) omitted = [ item for item in result["suites"] if item["disposition"] in {"excluded", "unsupported"} ] result["summary"] = [ "Declared-suite coverage for " + args.profile + " (plan only, not execution evidence)", ", ".join( f"{count} {name}" for name, count in sorted(result["counts"].items()) ), *result["notes"], *[ f"{item['repo']} / {item['name']}: {item['disposition']} — {item['reason']}" for item in omitted[:12] ], ] if len(omitted) > 12: result["summary"].append( f"{len(omitted) - 12} additional exclusions/unsupported suites are listed with --json." ) return result coverage.set_defaults(handler=coverage_handler) runs = subparsers.add_parser( "runs", help="List recent local runs with bounded cursor pagination" ) runs.add_argument("--limit", type=int, default=10) runs.add_argument( "--before", help="Exclusive run-ID cursor from the preceding page" ) runs.set_defaults(handler=list_runs) latest = subparsers.add_parser( "latest", help="Read the newest matching run; never substitute an older pass" ) latest.set_defaults(handler=latest_run) for name in ("status", "summary"): parser = subparsers.add_parser( name, help="Read a check run without reading or replaying its full logs" ) parser.add_argument("run_id") parser.set_defaults( handler=lambda args: summarize( read_receipt(args.workspace_root, args.state_dir, args.run_id) ) ) resume = subparsers.add_parser( "resume", help="Replan a previous check selection and reuse only verified unchanged results", ) resume.add_argument("run_id") resume.add_argument("--jobs", type=int, default=2) resume.add_argument("--dry-run", action="store_true") def resume_handler(args): previous = read_receipt(args.workspace_root, args.state_dir, args.run_id) if previous.get("profile") not in {"quick", "ui", "backend", "full"}: raise ValueError( "This run has no reusable check profile; repeat its owning audit command" ) previous_project = previous.get("project_file") if args.project and str(args.project.resolve()) != previous_project: raise ValueError("Resume project differs from the recorded project") args.project = Path(previous_project) if previous_project else None args.profile = previous["profile"] args.repo = previous.get("selection", {}).get("repos", []) args.changed = previous.get("selection", {}).get("changed", False) args.resume = args.run_id return handle(args) resume.set_defaults(handler=resume_handler) recover = subparsers.add_parser( "recover", help="Mark an abandoned check run interrupted; never rerun a process" ) recover.add_argument("run_id") recover.add_argument("--apply", action="store_true") recover.add_argument( "--confirm-processes-stopped", action="store_true", help="Attest that surviving commands from the abandoned run were manually checked and stopped", ) def recover_handler(args): base = state_root(args.workspace_root, args.state_dir) previous = read_receipt(args.workspace_root, args.state_dir, args.run_id) if previous["status"] != "running": return { "status": "unchanged", "summary": ["Run does not require interrupted-owner recovery."], } if not args.apply: return { "status": "planned", "summary": [ "Abandoned run owner PID: " + str(previous.get("owner_pid")), "A free run lock does not prove its child processes stopped after a hard kill. Inspect the recorded commands and stop any survivors before using --apply --confirm-processes-stopped. No checks will execute during recovery.", ], } if not args.confirm_processes_stopped: raise ValueError( "Hard-crash recovery requires manual verification that the recorded commands and surviving child processes stopped; then use --apply --confirm-processes-stopped" ) with resource_lock(base / "locks", "run:" + args.run_id): previous = read_receipt(args.workspace_root, args.state_dir, args.run_id) if previous["status"] != "running": return { "status": "unchanged", "summary": [ "Run completed before recovery acquired its lock; no state was changed." ], } previous.update( status="interrupted", phase="finished", finished_at=now(), snapshot_verified=False, ) for item in previous["stages"]: if item["status"] in {"running", "pending"}: item["status"] = "interrupted" atomic_json(base / "runs" / args.run_id / "receipt.json", _seal(previous)) result = summarize(previous) result["summary"].append( "The interrupted aggregate is not passing evidence. Resume may reuse independently verified matching phase checkpoints; other stages run again." ) return result recover.set_defaults(handler=recover_handler) logs = subparsers.add_parser( "logs", help="Read bounded provisional output or a hash-verified final stage log", ) logs.add_argument("run_id") logs.add_argument("--stage", required=True) logs.add_argument("--tail", type=int, default=40) logs.add_argument( "--final-only", action="store_true", help="Require a finalized, hash-verified stage log", ) def logs_handler(args): if not 1 <= args.tail <= 200: raise ValueError("Log tail must be between 1 and 200 lines") receipt = read_receipt(args.workspace_root, args.state_dir, args.run_id) selected = next( (stage for stage in receipt["stages"] if stage["id"] == args.stage), None ) if not selected: raise ValueError("Stage does not belong to this run") if selected.get("log_sha256"): text = _verified_log_bytes( selected, state_root(args.workspace_root, args.state_dir) ).decode("utf-8", errors="replace") details = {"provisional": False, "log_verified": True} else: if args.final_only: raise ValueError( "No finalized stage log is available yet; live output is provisional" ) details = read_live( args.workspace_root, args.state_dir, args.run_id, args.stage ) text = details.pop("excerpt") details["log_verified"] = False excerpt = bounded_display( redact("\n".join(text.splitlines()[-args.tail :])), 16384, tail_only=True ) label = ( "Provisional live output — not verified evidence." if details["provisional"] else "Final stage log — hash verified; consult run status for overall verification." ) return { **details, "run_id": args.run_id, "stage": args.stage, "run_status": receipt["status"], "snapshot_verified": receipt["snapshot_verified"], "excerpt": excerpt, "summary": [label, excerpt], } logs.set_defaults(handler=logs_handler)