From 08426ea177a38c74082e5d885c317e05c4b93ca1 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Tue, 30 Jun 2026 13:39:31 +0200 Subject: [PATCH] Initial commit, very early alpha --- .env.example | 2 + .gitignore | 3 + README.md | 66 ++- app.py | 467 ++++++++++++++++++ requirements.txt | 4 + ...026-06-28T115642755+0000_bca98569dc74.json | 63 +++ ...026-06-28T120226219+0000_bca98569dc74.json | 251 ++++++++++ scripts/launch-dev.sh | 108 ++++ verify_bundle.py | 72 +++ 9 files changed, 1035 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 app.py create mode 100644 requirements.txt create mode 100644 runs/run_2026-06-28T115642755+0000_bca98569dc74.json create mode 100644 runs/run_2026-06-28T120226219+0000_bca98569dc74.json create mode 100644 scripts/launch-dev.sh create mode 100644 verify_bundle.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d583282 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# Copy to .env or export this variable in your shell. +OPENAI_API_KEY=sk-your-key-here diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e497ef5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +.venv/ +.vscode/ diff --git a/README.md b/README.md index 863593f..82ab6eb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,66 @@ -# llm-repro-ui +# LLM Reproducibility Harness +A local Streamlit UI for testing how repeatable OpenAI Chat Completions are when you hold the prompt, context, model, seed, and generation parameters constant. + +The goal is to demonstrate operational reproducibility: identical request payloads, identical outputs, and identical backend metadata where available. It is not a proof that the hosted model is mathematically deterministic. + +## What it records + +Each run writes a JSON bundle under `runs/` containing: + +- exact request payload sent to `v1/chat/completions` +- SHA-256 hash of the canonicalized request payload +- repeated trial outputs +- SHA-256 hash of each output string +- `system_fingerprint` returned by the API, when available +- returned model id, finish reason, usage, response id, and raw response + +The API key is not stored in bundles. + +## Setup + +```bash +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +cp .env.example .env +# Edit .env and set OPENAI_API_KEY, or export it in your shell. +streamlit run app.py +``` + +## Recommended first experiment + +Use the default prompt and these settings: + +- model: a dated snapshot, for example `gpt-4.1-mini-2025-04-14` +- repetitions: `3` or `5` +- seed: `42` +- temperature: `0` +- top_p: `1` +- max_completion_tokens: keep modest, for example `200` +- streaming: off; tools: not used by this harness + +Then change only the seed and run again. You should usually see a different output hash while the request hash changes by exactly the seed field. + +## Baseline comparison + +After a run is saved, select it in the sidebar as a baseline and run the same request again. The comparison checks: + +- whether the canonical request hash is identical +- whether output hashes are identical in the same order +- whether the set of returned system fingerprints is identical + +## Offline verification + +To recompute bundle and output hashes without calling the API: + +```bash +python verify_bundle.py runs/run_*.json +``` + +## Caveats + +- Hosted LLM reproducibility is best-effort. Matching `seed`, prompt, parameters, and `system_fingerprint` improves repeatability but does not guarantee identical text forever. +- Rolling model aliases can change. Use snapshot model identifiers where your account and model family support them. +- Longer completions provide more opportunities for divergence. +- Tool calls, retrieval, web access, time-dependent prompts, and streaming add more variables. This scaffold deliberately avoids them. diff --git a/app.py b/app.py new file mode 100644 index 0000000..1bff6b2 --- /dev/null +++ b/app.py @@ -0,0 +1,467 @@ +from __future__ import annotations + +import json +import os +import time +import hashlib +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional +import difflib + +import pandas as pd +import streamlit as st +from dotenv import load_dotenv +from openai import OpenAI + +load_dotenv() + +APP_SCHEMA_VERSION = "llm-repro-harness.v1" +DATA_DIR = Path("runs") +DATA_DIR.mkdir(exist_ok=True) + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="milliseconds") + + +def canonical_json(obj: Any) -> str: + """Stable JSON serialization used for request and bundle hashing.""" + return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) + + +def pretty_json(obj: Any) -> str: + return json.dumps(obj, ensure_ascii=False, sort_keys=True, indent=2, default=str) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_json(obj: Any) -> str: + return sha256_text(canonical_json(obj)) + + +def dump_model(obj: Any) -> Any: + """Convert OpenAI SDK / Pydantic objects to JSON-compatible structures.""" + if obj is None: + return None + if hasattr(obj, "model_dump"): + return obj.model_dump(mode="json") + if hasattr(obj, "to_dict_recursive"): + return obj.to_dict_recursive() + if hasattr(obj, "dict"): + return obj.dict() + return obj + + +def parse_stop_sequences(raw: str) -> Optional[List[str]]: + values = [line for line in raw.splitlines() if line.strip()] + return values or None + + +def extract_message_text(message: Any) -> str: + """Extract text content from a chat completion message without assuming a single SDK shape.""" + content = getattr(message, "content", None) + if isinstance(content, str): + return content + if content is not None: + return canonical_json(content) + + refusal = getattr(message, "refusal", None) + if refusal: + return f"[refusal] {refusal}" + + return "" + + +def build_payload( + *, + model: str, + system_prompt: str, + user_prompt: str, + context: str, + include_seed: bool, + seed: int, + include_temperature: bool, + temperature: float, + include_top_p: bool, + top_p: float, + max_completion_tokens: int, + frequency_penalty: float, + presence_penalty: float, + stop_sequences: Optional[List[str]], + include_logprobs: bool, + top_logprobs: int, +) -> Dict[str, Any]: + messages: List[Dict[str, str]] = [] + if system_prompt.strip(): + messages.append({"role": "system", "content": system_prompt}) + if context.strip(): + messages.append({"role": "user", "content": f"Context:\n{context}"}) + messages.append({"role": "user", "content": user_prompt}) + + payload: Dict[str, Any] = { + "model": model, + "messages": messages, + "max_completion_tokens": max_completion_tokens, + "frequency_penalty": frequency_penalty, + "presence_penalty": presence_penalty, + "n": 1, + "stream": False, + } + + if include_seed: + payload["seed"] = int(seed) + if include_temperature: + payload["temperature"] = float(temperature) + if include_top_p: + payload["top_p"] = float(top_p) + if stop_sequences: + payload["stop"] = stop_sequences + if include_logprobs: + payload["logprobs"] = True + payload["top_logprobs"] = int(top_logprobs) + + return payload + + +def call_chat_completion(client: OpenAI, payload: Dict[str, Any], trial_index: int) -> Dict[str, Any]: + started_at = utc_now_iso() + start_time = time.perf_counter() + + response = client.chat.completions.create(**payload) + + elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2) + response_dict = dump_model(response) + choice = response.choices[0] + message = choice.message + content = extract_message_text(message) + + trial = { + "trial_index": trial_index, + "ok": True, + "started_at_utc": started_at, + "elapsed_ms": elapsed_ms, + "response_id": getattr(response, "id", None), + "response_model": getattr(response, "model", None), + "system_fingerprint": getattr(response, "system_fingerprint", None), + "finish_reason": getattr(choice, "finish_reason", None), + "content": content, + "content_sha256": sha256_text(content), + "usage": dump_model(getattr(response, "usage", None)), + "raw_response": response_dict, + } + return trial + + +def summarize_trials(trials: List[Dict[str, Any]]) -> Dict[str, Any]: + successful = [t for t in trials if t.get("ok")] + failed = [t for t in trials if not t.get("ok")] + + content_hashes = [t["content_sha256"] for t in successful] + fingerprints = [t.get("system_fingerprint") for t in successful] + response_models = [t.get("response_model") for t in successful] + + return { + "trial_count": len(trials), + "successful_trial_count": len(successful), + "failed_trial_count": len(failed), + "all_successful": len(failed) == 0, + "all_outputs_identical": len(set(content_hashes)) <= 1 if successful else False, + "all_system_fingerprints_identical": len(set(fingerprints)) <= 1 if successful else False, + "all_response_models_identical": len(set(response_models)) <= 1 if successful else False, + "unique_content_sha256": sorted(set(content_hashes)), + "unique_system_fingerprints": sorted({str(x) for x in fingerprints}), + "unique_response_models": sorted({str(x) for x in response_models}), + } + + +def make_bundle(payload: Dict[str, Any], trials: List[Dict[str, Any]]) -> Dict[str, Any]: + request_hash = sha256_json(payload) + bundle = { + "schema_version": APP_SCHEMA_VERSION, + "created_at_utc": utc_now_iso(), + "endpoint": "v1/chat/completions", + "request_payload": payload, + "request_payload_sha256": request_hash, + "trials": trials, + "summary": summarize_trials(trials), + } + bundle["bundle_sha256_without_self"] = sha256_json(bundle) + return bundle + + +def save_bundle(bundle: Dict[str, Any]) -> Path: + created = bundle["created_at_utc"].replace(":", "").replace(".", "") + request_hash_short = bundle["request_payload_sha256"][:12] + filename = f"run_{created}_{request_hash_short}.json" + path = DATA_DIR / filename + path.write_text(pretty_json(bundle), encoding="utf-8") + return path + + +def load_bundle(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def list_saved_runs() -> List[Path]: + return sorted(DATA_DIR.glob("run_*.json"), reverse=True) + + +def compare_bundles(baseline: Dict[str, Any], candidate: Dict[str, Any]) -> Dict[str, Any]: + b_hashes = [t.get("content_sha256") for t in baseline.get("trials", []) if t.get("ok")] + c_hashes = [t.get("content_sha256") for t in candidate.get("trials", []) if t.get("ok")] + return { + "same_request_payload_hash": baseline.get("request_payload_sha256") == candidate.get("request_payload_sha256"), + "same_ordered_output_hashes": b_hashes == c_hashes, + "same_output_hash_set": set(b_hashes) == set(c_hashes), + "baseline_output_hashes": b_hashes, + "candidate_output_hashes": c_hashes, + "baseline_system_fingerprints": baseline.get("summary", {}).get("unique_system_fingerprints", []), + "candidate_system_fingerprints": candidate.get("summary", {}).get("unique_system_fingerprints", []), + "same_system_fingerprint_set": set(baseline.get("summary", {}).get("unique_system_fingerprints", [])) + == set(candidate.get("summary", {}).get("unique_system_fingerprints", [])), + } + + +def render_summary(bundle: Dict[str, Any]) -> None: + summary = bundle["summary"] + col1, col2, col3, col4 = st.columns(4) + col1.metric("Trials", summary["trial_count"]) + col2.metric("Successful", summary["successful_trial_count"]) + col3.metric("Unique outputs", len(summary["unique_content_sha256"])) + col4.metric("Unique fingerprints", len(summary["unique_system_fingerprints"])) + + if summary["all_successful"] and summary["all_outputs_identical"] and summary["all_system_fingerprints_identical"]: + st.success("Within this run, the outputs were exactly identical and the system fingerprint was unchanged.") + elif summary["all_successful"] and summary["all_outputs_identical"]: + st.warning("Outputs were identical, but the system fingerprint varied or was unavailable.") + elif summary["all_successful"]: + st.warning("Outputs differed. Inspect the fingerprints, finish reasons, token counts, and diff below.") + else: + st.error("At least one trial failed. Inspect the error rows below.") + + rows = [] + for trial in bundle["trials"]: + usage = trial.get("usage") or {} + rows.append( + { + "trial": trial.get("trial_index"), + "ok": trial.get("ok"), + "content_sha256": trial.get("content_sha256", "")[:16], + "system_fingerprint": trial.get("system_fingerprint"), + "response_model": trial.get("response_model"), + "finish_reason": trial.get("finish_reason"), + "elapsed_ms": trial.get("elapsed_ms"), + "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": usage.get("completion_tokens"), + "total_tokens": usage.get("total_tokens"), + "error": trial.get("error"), + } + ) + st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True) + + successful = [t for t in bundle["trials"] if t.get("ok")] + if successful and not summary["all_outputs_identical"]: + first = successful[0] + different = next((t for t in successful[1:] if t["content_sha256"] != first["content_sha256"]), None) + if different: + st.subheader("First output diff") + diff = difflib.unified_diff( + first["content"].splitlines(), + different["content"].splitlines(), + fromfile=f"trial_{first['trial_index']}", + tofile=f"trial_{different['trial_index']}", + lineterm="", + n=3, + ) + st.code("\n".join(diff)[:12000], language="diff") + + st.subheader("Outputs") + for trial in bundle["trials"]: + label = f"Trial {trial.get('trial_index')}" + if trial.get("ok"): + label += f" | {trial.get('content_sha256', '')[:16]} | {trial.get('system_fingerprint')}" + with st.expander(label, expanded=False): + st.write(trial.get("content", "")) + else: + label += " | failed" + with st.expander(label, expanded=True): + st.code(trial.get("error", "")) + + +st.set_page_config(page_title="LLM Reproducibility Harness", layout="wide") +st.title("LLM Reproducibility Harness") +st.caption("Local UI for testing repeated Chat Completions with fixed request parameters, seed, and recorded backend metadata.") + +with st.sidebar: + st.header("Connection") + env_key_present = bool(os.environ.get("OPENAI_API_KEY")) + api_key_input = st.text_input( + "OpenAI API key", + type="password", + placeholder="Leave blank to use OPENAI_API_KEY" if env_key_present else "sk-...", + help="The key is used only for API calls and is not written to run bundles.", + ) + api_key = api_key_input.strip() or os.environ.get("OPENAI_API_KEY", "") + if env_key_present and not api_key_input: + st.caption("Using OPENAI_API_KEY from environment or .env.") + + st.header("Run controls") + repetitions = st.slider("Repetitions", min_value=1, max_value=12, value=3, help="Use at least 2 to demonstrate exact repeatability.") + delay_s = st.number_input("Delay between calls, seconds", min_value=0.0, max_value=10.0, value=0.0, step=0.25) + + st.header("Saved baselines") + saved = list_saved_runs() + saved_names = [""] + [p.name for p in saved] + selected_baseline_name = st.selectbox("Baseline bundle", saved_names, help="Compare a saved run with the latest run in this session.") + +with st.expander("Interpretation", expanded=True): + st.markdown( + """ +This harness tests *operational reproducibility*, not mathematical determinism. A strong run has: + +1. identical request payload hash, +2. identical output hashes, +3. identical `system_fingerprint`, +4. identical model snapshot or model identifier, +5. no streaming or tools. + +For stronger evidence, use a dated model snapshot rather than a rolling alias, keep `max_completion_tokens` modest, set `temperature` to `0`, set a fixed integer `seed`, and repeat the same saved bundle later. +""" + ) + +left, right = st.columns([2, 1]) + +with left: + st.subheader("Prompt and context") + system_prompt = st.text_area( + "System prompt", + value="You are a precise assistant. Answer in exactly three bullet points.", + height=100, + ) + context = st.text_area( + "Optional fixed context", + value="", + height=80, + help="Paste any external context here. It becomes part of the request payload and hash.", + ) + user_prompt = st.text_area( + "User prompt", + value="Explain why reproducibility matters in LLM experiments.", + height=120, + ) + +with right: + st.subheader("Model parameters") + model = st.text_input( + "Model", + value="gpt-4.1-mini-2025-04-14", + help="Prefer a dated snapshot model over a rolling alias for reproducibility checks.", + ) + max_completion_tokens = st.number_input("max_completion_tokens", min_value=1, max_value=4096, value=200, step=10) + + include_seed = st.checkbox("Send seed", value=True) + seed = st.number_input("seed", min_value=0, max_value=2_147_483_647, value=42, step=1, disabled=not include_seed) + + include_temperature = st.checkbox("Send temperature", value=True) + temperature = st.number_input("temperature", min_value=0.0, max_value=2.0, value=0.0, step=0.05, disabled=not include_temperature) + + include_top_p = st.checkbox("Send top_p", value=True) + top_p = st.number_input("top_p", min_value=0.0, max_value=1.0, value=1.0, step=0.05, disabled=not include_top_p) + + frequency_penalty = st.number_input("frequency_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1) + presence_penalty = st.number_input("presence_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1) + + stop_raw = st.text_area("Stop sequences, one per line", value="", height=70) + include_logprobs = st.checkbox("Request logprobs", value=False, help="Useful for inspection, but not all models support it.") + top_logprobs = st.number_input("top_logprobs", min_value=0, max_value=20, value=0, step=1, disabled=not include_logprobs) + +payload = build_payload( + model=model.strip(), + system_prompt=system_prompt, + user_prompt=user_prompt, + context=context, + include_seed=include_seed, + seed=int(seed), + include_temperature=include_temperature, + temperature=float(temperature), + include_top_p=include_top_p, + top_p=float(top_p), + max_completion_tokens=int(max_completion_tokens), + frequency_penalty=float(frequency_penalty), + presence_penalty=float(presence_penalty), + stop_sequences=parse_stop_sequences(stop_raw), + include_logprobs=include_logprobs, + top_logprobs=int(top_logprobs), +) + +request_hash = sha256_json(payload) + +st.subheader("Canonical request") +st.code(pretty_json(payload), language="json") +st.caption(f"Request payload SHA-256: `{request_hash}`") + +run_clicked = st.button("Run reproducibility test", type="primary", disabled=not bool(api_key)) +if not api_key: + st.info("Provide an API key in the sidebar or set OPENAI_API_KEY in your environment.") + +if run_clicked: + client = OpenAI(api_key=api_key) + trials: List[Dict[str, Any]] = [] + progress = st.progress(0, text="Starting calls") + + for i in range(1, repetitions + 1): + progress.progress((i - 1) / repetitions, text=f"Running trial {i} of {repetitions}") + try: + trial = call_chat_completion(client, payload, i) + except Exception as exc: + trial = { + "trial_index": i, + "ok": False, + "started_at_utc": utc_now_iso(), + "error_type": type(exc).__name__, + "error": str(exc), + } + trials.append(trial) + if delay_s and i < repetitions: + time.sleep(float(delay_s)) + + progress.progress(1.0, text="Complete") + bundle = make_bundle(payload, trials) + path = save_bundle(bundle) + st.session_state["latest_bundle"] = bundle + st.session_state["latest_bundle_path"] = str(path) + +if "latest_bundle" in st.session_state: + st.divider() + st.header("Latest run") + latest_bundle = st.session_state["latest_bundle"] + latest_path = st.session_state.get("latest_bundle_path") + st.caption(f"Saved locally as `{latest_path}`") + render_summary(latest_bundle) + + bundle_json = pretty_json(latest_bundle) + st.download_button( + "Download latest run bundle JSON", + data=bundle_json, + file_name=Path(latest_path).name if latest_path else "llm_repro_run.json", + mime="application/json", + ) + + if selected_baseline_name: + baseline_path = DATA_DIR / selected_baseline_name + if baseline_path.exists(): + baseline = load_bundle(baseline_path) + comparison = compare_bundles(baseline, latest_bundle) + st.subheader("Baseline comparison") + c1, c2, c3 = st.columns(3) + c1.metric("Same request hash", "yes" if comparison["same_request_payload_hash"] else "no") + c2.metric("Same ordered outputs", "yes" if comparison["same_ordered_output_hashes"] else "no") + c3.metric("Same fingerprint set", "yes" if comparison["same_system_fingerprint_set"] else "no") + st.code(pretty_json(comparison), language="json") +else: + st.divider() + st.header("No run yet") + st.write("Configure the prompt and parameters, then start a reproducibility test.") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fb0d619 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +openai>=1.55.0 +streamlit>=1.38.0 +pandas>=2.2.0 +python-dotenv>=1.0.1 diff --git a/runs/run_2026-06-28T115642755+0000_bca98569dc74.json b/runs/run_2026-06-28T115642755+0000_bca98569dc74.json new file mode 100644 index 0000000..4ac3d2a --- /dev/null +++ b/runs/run_2026-06-28T115642755+0000_bca98569dc74.json @@ -0,0 +1,63 @@ +{ + "bundle_sha256_without_self": "e53559b02ebca1ca73747638060673943c8436a1842c35145ae16e7876aa6135", + "created_at_utc": "2026-06-28T11:56:42.755+00:00", + "endpoint": "v1/chat/completions", + "request_payload": { + "frequency_penalty": 0.0, + "max_completion_tokens": 200, + "messages": [ + { + "content": "You are a precise assistant. Answer in exactly three bullet points.", + "role": "system" + }, + { + "content": "Explain why reproducibility matters in LLM experiments.", + "role": "user" + } + ], + "model": "gpt-4.1-mini-2025-04-14", + "n": 1, + "presence_penalty": 0.0, + "seed": 42, + "stream": false, + "temperature": 0.0, + "top_p": 1.0 + }, + "request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5", + "schema_version": "llm-repro-harness.v1", + "summary": { + "all_outputs_identical": false, + "all_response_models_identical": false, + "all_successful": false, + "all_system_fingerprints_identical": false, + "failed_trial_count": 3, + "successful_trial_count": 0, + "trial_count": 3, + "unique_content_sha256": [], + "unique_response_models": [], + "unique_system_fingerprints": [] + }, + "trials": [ + { + "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}", + "error_type": "RateLimitError", + "ok": false, + "started_at_utc": "2026-06-28T11:56:38.381+00:00", + "trial_index": 1 + }, + { + "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}", + "error_type": "RateLimitError", + "ok": false, + "started_at_utc": "2026-06-28T11:56:40.528+00:00", + "trial_index": 2 + }, + { + "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}", + "error_type": "RateLimitError", + "ok": false, + "started_at_utc": "2026-06-28T11:56:42.755+00:00", + "trial_index": 3 + } + ] +} \ No newline at end of file diff --git a/runs/run_2026-06-28T120226219+0000_bca98569dc74.json b/runs/run_2026-06-28T120226219+0000_bca98569dc74.json new file mode 100644 index 0000000..49f1072 --- /dev/null +++ b/runs/run_2026-06-28T120226219+0000_bca98569dc74.json @@ -0,0 +1,251 @@ +{ + "bundle_sha256_without_self": "01b5de5925b7dcdee3df4a6dcdd25226963b77f6b4ae98e3f0573e4533cded93", + "created_at_utc": "2026-06-28T12:02:26.219+00:00", + "endpoint": "v1/chat/completions", + "request_payload": { + "frequency_penalty": 0.0, + "max_completion_tokens": 200, + "messages": [ + { + "content": "You are a precise assistant. Answer in exactly three bullet points.", + "role": "system" + }, + { + "content": "Explain why reproducibility matters in LLM experiments.", + "role": "user" + } + ], + "model": "gpt-4.1-mini-2025-04-14", + "n": 1, + "presence_penalty": 0.0, + "seed": 42, + "stream": false, + "temperature": 0.0, + "top_p": 1.0 + }, + "request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5", + "schema_version": "llm-repro-harness.v1", + "summary": { + "all_outputs_identical": false, + "all_response_models_identical": true, + "all_successful": true, + "all_system_fingerprints_identical": true, + "failed_trial_count": 0, + "successful_trial_count": 3, + "trial_count": 3, + "unique_content_sha256": [ + "1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9", + "33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1", + "9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8" + ], + "unique_response_models": [ + "gpt-4.1-mini-2025-04-14" + ], + "unique_system_fingerprints": [ + "fp_8f3c0606ab" + ] + }, + "trials": [ + { + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development. \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.", + "content_sha256": "33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1", + "elapsed_ms": 2005.2, + "finish_reason": "stop", + "ok": true, + "raw_response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "audio": null, + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development. \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1782648142, + "id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G", + "model": "gpt-4.1-mini-2025-04-14", + "moderation": null, + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "fp_8f3c0606ab", + "usage": { + "completion_tokens": 92, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 126 + } + }, + "response_id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G", + "response_model": "gpt-4.1-mini-2025-04-14", + "started_at_utc": "2026-06-28T12:02:21.404+00:00", + "system_fingerprint": "fp_8f3c0606ab", + "trial_index": 1, + "usage": { + "completion_tokens": 92, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 126 + } + }, + { + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.", + "content_sha256": "9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8", + "elapsed_ms": 1477.74, + "finish_reason": "stop", + "ok": true, + "raw_response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "audio": null, + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1782648143, + "id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q", + "model": "gpt-4.1-mini-2025-04-14", + "moderation": null, + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "fp_8f3c0606ab", + "usage": { + "completion_tokens": 85, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 119 + } + }, + "response_id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q", + "response_model": "gpt-4.1-mini-2025-04-14", + "started_at_utc": "2026-06-28T12:02:23.410+00:00", + "system_fingerprint": "fp_8f3c0606ab", + "trial_index": 2, + "usage": { + "completion_tokens": 85, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 119 + } + }, + { + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.", + "content_sha256": "1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9", + "elapsed_ms": 1330.2, + "finish_reason": "stop", + "ok": true, + "raw_response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "audio": null, + "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1782648145, + "id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi", + "model": "gpt-4.1-mini-2025-04-14", + "moderation": null, + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "fp_8f3c0606ab", + "usage": { + "completion_tokens": 92, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 126 + } + }, + "response_id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi", + "response_model": "gpt-4.1-mini-2025-04-14", + "started_at_utc": "2026-06-28T12:02:24.888+00:00", + "system_fingerprint": "fp_8f3c0606ab", + "trial_index": 3, + "usage": { + "completion_tokens": 92, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 34, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 126 + } + } + ] +} \ No newline at end of file diff --git a/scripts/launch-dev.sh b/scripts/launch-dev.sh new file mode 100644 index 0000000..bb72779 --- /dev/null +++ b/scripts/launch-dev.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${LLM_REPRO_ROOT:-/mnt/DATA/git/llm-repro-ui}" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" +HOST="${LLM_REPRO_HOST:-127.0.0.1}" +PORT="${LLM_REPRO_PORT:-8501}" +OPEN_BROWSER="${OPEN_BROWSER:-1}" + +LOG_DIR="$ROOT/runs/dev-launcher" +SERVER_LOG="$LOG_DIR/streamlit.log" +URL="http://$HOST:$PORT" + +server_pid="" + +fail() { + printf 'launch-dev: %s\n' "$*" >&2 + exit 1 +} + +port_is_free() { + "$PYTHON" - "$1" "$2" <<'PY' +import socket +import sys + +host = sys.argv[1] +port = int(sys.argv[2]) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.bind((host, port)) + except OSError: + raise SystemExit(1) +PY +} + +wait_for_url() { + "$PYTHON" - "$1" <<'PY' +import sys +import time +import urllib.request + +url = sys.argv[1] +deadline = time.monotonic() + 60 +last_error = None +while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as response: + if 200 <= response.status < 500: + raise SystemExit(0) + except Exception as exc: # noqa: BLE001 - printed only on timeout. + last_error = exc + time.sleep(1) +print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr) +raise SystemExit(1) +PY +} + +cleanup() { + if [ -n "${server_pid:-}" ] && kill -0 "$server_pid" 2>/dev/null; then + kill "$server_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && python -m venv .venv && . .venv/bin/activate && pip install -r requirements.txt" + +mkdir -p "$LOG_DIR" +: > "$SERVER_LOG" + +port_is_free "$HOST" "$PORT" || fail "$URL is already in use" + +if [ -z "${OPENAI_API_KEY:-}" ] && { [ ! -f "$ROOT/.env" ] || ! grep -q '^OPENAI_API_KEY=sk-' "$ROOT/.env"; }; then + printf 'Warning: OPENAI_API_KEY is not set in the environment or .env. The UI will start, but API calls need a key.\n' >&2 +fi + +printf 'Starting LLM Reproducibility Harness at %s\n' "$URL" +( + cd "$ROOT" + "$PYTHON" -m streamlit run app.py \ + --server.address "$HOST" \ + --server.port "$PORT" \ + --server.headless true +) >"$SERVER_LOG" 2>&1 & +server_pid="$!" + +printf 'Waiting for %s\n' "$URL" +wait_for_url "$URL" || { + tail -n 80 "$SERVER_LOG" >&2 || true + fail "Streamlit did not become reachable" +} + +if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then + xdg-open "$URL" >/dev/null 2>&1 || true +fi + +cat < str: + return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_json(obj: Any) -> str: + return sha256_text(canonical_json(obj)) + + +def verify(path: Path) -> Dict[str, Any]: + bundle = json.loads(path.read_text(encoding="utf-8")) + payload = bundle.get("request_payload") + expected_payload_hash = bundle.get("request_payload_sha256") + actual_payload_hash = sha256_json(payload) + + trial_results: List[Dict[str, Any]] = [] + for trial in bundle.get("trials", []): + if not trial.get("ok"): + trial_results.append({"trial_index": trial.get("trial_index"), "ok": False, "reason": "trial_failed"}) + continue + expected = trial.get("content_sha256") + actual = sha256_text(trial.get("content", "")) + trial_results.append( + { + "trial_index": trial.get("trial_index"), + "ok": expected == actual, + "expected_content_sha256": expected, + "actual_content_sha256": actual, + } + ) + + successful_hashes = [t.get("content_sha256") for t in bundle.get("trials", []) if t.get("ok")] + fingerprints = [str(t.get("system_fingerprint")) for t in bundle.get("trials", []) if t.get("ok")] + + return { + "path": str(path), + "payload_hash_ok": expected_payload_hash == actual_payload_hash, + "expected_payload_sha256": expected_payload_hash, + "actual_payload_sha256": actual_payload_hash, + "trial_content_hashes_ok": all(t["ok"] for t in trial_results), + "all_successful_outputs_identical": len(set(successful_hashes)) <= 1 if successful_hashes else False, + "all_successful_fingerprints_identical": len(set(fingerprints)) <= 1 if fingerprints else False, + "unique_output_hashes": sorted(set(successful_hashes)), + "unique_system_fingerprints": sorted(set(fingerprints)), + "trials": trial_results, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Verify hashes inside LLM reproducibility run bundles.") + parser.add_argument("bundle", nargs="+", type=Path) + args = parser.parse_args() + + for path in args.bundle: + result = verify(path) + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main()