add-ideas/llm-repro-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit, very early alpha

Browse Source
This commit is contained in:
Albrecht Degering
2026-06-30 13:39:31 +02:00
parent e17611eda1
commit 08426ea177
9 changed files with 1035 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example Normal file
View File

@@ -0,0 +1,2 @@
# Copy to .env or export this variable in your shell.
OPENAI_API_KEY=sk-your-key-here

3
.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
.env
.venv/
.vscode/

66
README.md
View File

@@ -1,2 +1,66 @@
# llm-repro-ui
# LLM Reproducibility Harness
A local Streamlit UI for testing how repeatable OpenAI Chat Completions are when you hold the prompt, context, model, seed, and generation parameters constant.
The goal is to demonstrate operational reproducibility: identical request payloads, identical outputs, and identical backend metadata where available. It is not a proof that the hosted model is mathematically deterministic.
## What it records
Each run writes a JSON bundle under `runs/` containing:
- exact request payload sent to `v1/chat/completions`
- SHA-256 hash of the canonicalized request payload
- repeated trial outputs
- SHA-256 hash of each output string
- `system_fingerprint` returned by the API, when available
- returned model id, finish reason, usage, response id, and raw response
The API key is not stored in bundles.
## Setup
```bash
python -m venv .venv
source .venv/bin/activate # Windows: .venv\Scripts\activate
pip install -r requirements.txt
cp .env.example .env
# Edit .env and set OPENAI_API_KEY, or export it in your shell.
streamlit run app.py
```
## Recommended first experiment
Use the default prompt and these settings:
- model: a dated snapshot, for example `gpt-4.1-mini-2025-04-14`
- repetitions: `3` or `5`
- seed: `42`
- temperature: `0`
- top_p: `1`
- max_completion_tokens: keep modest, for example `200`
- streaming: off; tools: not used by this harness
Then change only the seed and run again. You should usually see a different output hash while the request hash changes by exactly the seed field.
## Baseline comparison
After a run is saved, select it in the sidebar as a baseline and run the same request again. The comparison checks:
- whether the canonical request hash is identical
- whether output hashes are identical in the same order
- whether the set of returned system fingerprints is identical
## Offline verification
To recompute bundle and output hashes without calling the API:
```bash
python verify_bundle.py runs/run_*.json
```
## Caveats
- Hosted LLM reproducibility is best-effort. Matching `seed`, prompt, parameters, and `system_fingerprint` improves repeatability but does not guarantee identical text forever.
- Rolling model aliases can change. Use snapshot model identifiers where your account and model family support them.
- Longer completions provide more opportunities for divergence.
- Tool calls, retrieval, web access, time-dependent prompts, and streaming add more variables. This scaffold deliberately avoids them.

467
app.py Normal file
View File

@@ -0,0 +1,467 @@
from __future__ import annotations
import json
import os
import time
import hashlib
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
import difflib
import pandas as pd
import streamlit as st
from dotenv import load_dotenv
from openai import OpenAI
load_dotenv()
APP_SCHEMA_VERSION = "llm-repro-harness.v1"
DATA_DIR = Path("runs")
DATA_DIR.mkdir(exist_ok=True)
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
def canonical_json(obj: Any) -> str:
"""Stable JSON serialization used for request and bundle hashing."""
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
def pretty_json(obj: Any) -> str:
return json.dumps(obj, ensure_ascii=False, sort_keys=True, indent=2, default=str)
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def sha256_json(obj: Any) -> str:
return sha256_text(canonical_json(obj))
def dump_model(obj: Any) -> Any:
"""Convert OpenAI SDK / Pydantic objects to JSON-compatible structures."""
if obj is None:
return None
if hasattr(obj, "model_dump"):
return obj.model_dump(mode="json")
if hasattr(obj, "to_dict_recursive"):
return obj.to_dict_recursive()
if hasattr(obj, "dict"):
return obj.dict()
return obj
def parse_stop_sequences(raw: str) -> Optional[List[str]]:
values = [line for line in raw.splitlines() if line.strip()]
return values or None
def extract_message_text(message: Any) -> str:
"""Extract text content from a chat completion message without assuming a single SDK shape."""
content = getattr(message, "content", None)
if isinstance(content, str):
return content
if content is not None:
return canonical_json(content)
refusal = getattr(message, "refusal", None)
if refusal:
return f"[refusal] {refusal}"
return ""
def build_payload(
*,
model: str,
system_prompt: str,
user_prompt: str,
context: str,
include_seed: bool,
seed: int,
include_temperature: bool,
temperature: float,
include_top_p: bool,
top_p: float,
max_completion_tokens: int,
frequency_penalty: float,
presence_penalty: float,
stop_sequences: Optional[List[str]],
include_logprobs: bool,
top_logprobs: int,
) -> Dict[str, Any]:
messages: List[Dict[str, str]] = []
if system_prompt.strip():
messages.append({"role": "system", "content": system_prompt})
if context.strip():
messages.append({"role": "user", "content": f"Context:\n{context}"})
messages.append({"role": "user", "content": user_prompt})
payload: Dict[str, Any] = {
"model": model,
"messages": messages,
"max_completion_tokens": max_completion_tokens,
"frequency_penalty": frequency_penalty,
"presence_penalty": presence_penalty,
"n": 1,
"stream": False,
}
if include_seed:
payload["seed"] = int(seed)
if include_temperature:
payload["temperature"] = float(temperature)
if include_top_p:
payload["top_p"] = float(top_p)
if stop_sequences:
payload["stop"] = stop_sequences
if include_logprobs:
payload["logprobs"] = True
payload["top_logprobs"] = int(top_logprobs)
return payload
def call_chat_completion(client: OpenAI, payload: Dict[str, Any], trial_index: int) -> Dict[str, Any]:
started_at = utc_now_iso()
start_time = time.perf_counter()
response = client.chat.completions.create(**payload)
elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2)
response_dict = dump_model(response)
choice = response.choices[0]
message = choice.message
content = extract_message_text(message)
trial = {
"trial_index": trial_index,
"ok": True,
"started_at_utc": started_at,
"elapsed_ms": elapsed_ms,
"response_id": getattr(response, "id", None),
"response_model": getattr(response, "model", None),
"system_fingerprint": getattr(response, "system_fingerprint", None),
"finish_reason": getattr(choice, "finish_reason", None),
"content": content,
"content_sha256": sha256_text(content),
"usage": dump_model(getattr(response, "usage", None)),
"raw_response": response_dict,
}
return trial
def summarize_trials(trials: List[Dict[str, Any]]) -> Dict[str, Any]:
successful = [t for t in trials if t.get("ok")]
failed = [t for t in trials if not t.get("ok")]
content_hashes = [t["content_sha256"] for t in successful]
fingerprints = [t.get("system_fingerprint") for t in successful]
response_models = [t.get("response_model") for t in successful]
return {
"trial_count": len(trials),
"successful_trial_count": len(successful),
"failed_trial_count": len(failed),
"all_successful": len(failed) == 0,
"all_outputs_identical": len(set(content_hashes)) <= 1 if successful else False,
"all_system_fingerprints_identical": len(set(fingerprints)) <= 1 if successful else False,
"all_response_models_identical": len(set(response_models)) <= 1 if successful else False,
"unique_content_sha256": sorted(set(content_hashes)),
"unique_system_fingerprints": sorted({str(x) for x in fingerprints}),
"unique_response_models": sorted({str(x) for x in response_models}),
}
def make_bundle(payload: Dict[str, Any], trials: List[Dict[str, Any]]) -> Dict[str, Any]:
request_hash = sha256_json(payload)
bundle = {
"schema_version": APP_SCHEMA_VERSION,
"created_at_utc": utc_now_iso(),
"endpoint": "v1/chat/completions",
"request_payload": payload,
"request_payload_sha256": request_hash,
"trials": trials,
"summary": summarize_trials(trials),
}
bundle["bundle_sha256_without_self"] = sha256_json(bundle)
return bundle
def save_bundle(bundle: Dict[str, Any]) -> Path:
created = bundle["created_at_utc"].replace(":", "").replace(".", "")
request_hash_short = bundle["request_payload_sha256"][:12]
filename = f"run_{created}_{request_hash_short}.json"
path = DATA_DIR / filename
path.write_text(pretty_json(bundle), encoding="utf-8")
return path
def load_bundle(path: Path) -> Dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def list_saved_runs() -> List[Path]:
return sorted(DATA_DIR.glob("run_*.json"), reverse=True)
def compare_bundles(baseline: Dict[str, Any], candidate: Dict[str, Any]) -> Dict[str, Any]:
b_hashes = [t.get("content_sha256") for t in baseline.get("trials", []) if t.get("ok")]
c_hashes = [t.get("content_sha256") for t in candidate.get("trials", []) if t.get("ok")]
return {
"same_request_payload_hash": baseline.get("request_payload_sha256") == candidate.get("request_payload_sha256"),
"same_ordered_output_hashes": b_hashes == c_hashes,
"same_output_hash_set": set(b_hashes) == set(c_hashes),
"baseline_output_hashes": b_hashes,
"candidate_output_hashes": c_hashes,
"baseline_system_fingerprints": baseline.get("summary", {}).get("unique_system_fingerprints", []),
"candidate_system_fingerprints": candidate.get("summary", {}).get("unique_system_fingerprints", []),
"same_system_fingerprint_set": set(baseline.get("summary", {}).get("unique_system_fingerprints", []))
== set(candidate.get("summary", {}).get("unique_system_fingerprints", [])),
}
def render_summary(bundle: Dict[str, Any]) -> None:
summary = bundle["summary"]
col1, col2, col3, col4 = st.columns(4)
col1.metric("Trials", summary["trial_count"])
col2.metric("Successful", summary["successful_trial_count"])
col3.metric("Unique outputs", len(summary["unique_content_sha256"]))
col4.metric("Unique fingerprints", len(summary["unique_system_fingerprints"]))
if summary["all_successful"] and summary["all_outputs_identical"] and summary["all_system_fingerprints_identical"]:
st.success("Within this run, the outputs were exactly identical and the system fingerprint was unchanged.")
elif summary["all_successful"] and summary["all_outputs_identical"]:
st.warning("Outputs were identical, but the system fingerprint varied or was unavailable.")
elif summary["all_successful"]:
st.warning("Outputs differed. Inspect the fingerprints, finish reasons, token counts, and diff below.")
else:
st.error("At least one trial failed. Inspect the error rows below.")
rows = []
for trial in bundle["trials"]:
usage = trial.get("usage") or {}
rows.append(
{
"trial": trial.get("trial_index"),
"ok": trial.get("ok"),
"content_sha256": trial.get("content_sha256", "")[:16],
"system_fingerprint": trial.get("system_fingerprint"),
"response_model": trial.get("response_model"),
"finish_reason": trial.get("finish_reason"),
"elapsed_ms": trial.get("elapsed_ms"),
"prompt_tokens": usage.get("prompt_tokens"),
"completion_tokens": usage.get("completion_tokens"),
"total_tokens": usage.get("total_tokens"),
"error": trial.get("error"),
}
)
st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)
successful = [t for t in bundle["trials"] if t.get("ok")]
if successful and not summary["all_outputs_identical"]:
first = successful[0]
different = next((t for t in successful[1:] if t["content_sha256"] != first["content_sha256"]), None)
if different:
st.subheader("First output diff")
diff = difflib.unified_diff(
first["content"].splitlines(),
different["content"].splitlines(),
fromfile=f"trial_{first['trial_index']}",
tofile=f"trial_{different['trial_index']}",
lineterm="",
n=3,
)
st.code("\n".join(diff)[:12000], language="diff")
st.subheader("Outputs")
for trial in bundle["trials"]:
label = f"Trial {trial.get('trial_index')}"
if trial.get("ok"):
label += f" | {trial.get('content_sha256', '')[:16]} | {trial.get('system_fingerprint')}"
with st.expander(label, expanded=False):
st.write(trial.get("content", ""))
else:
label += " | failed"
with st.expander(label, expanded=True):
st.code(trial.get("error", ""))
st.set_page_config(page_title="LLM Reproducibility Harness", layout="wide")
st.title("LLM Reproducibility Harness")
st.caption("Local UI for testing repeated Chat Completions with fixed request parameters, seed, and recorded backend metadata.")
with st.sidebar:
st.header("Connection")
env_key_present = bool(os.environ.get("OPENAI_API_KEY"))
api_key_input = st.text_input(
"OpenAI API key",
type="password",
placeholder="Leave blank to use OPENAI_API_KEY" if env_key_present else "sk-...",
help="The key is used only for API calls and is not written to run bundles.",
)
api_key = api_key_input.strip() or os.environ.get("OPENAI_API_KEY", "")
if env_key_present and not api_key_input:
st.caption("Using OPENAI_API_KEY from environment or .env.")
st.header("Run controls")
repetitions = st.slider("Repetitions", min_value=1, max_value=12, value=3, help="Use at least 2 to demonstrate exact repeatability.")
delay_s = st.number_input("Delay between calls, seconds", min_value=0.0, max_value=10.0, value=0.0, step=0.25)
st.header("Saved baselines")
saved = list_saved_runs()
saved_names = [""] + [p.name for p in saved]
selected_baseline_name = st.selectbox("Baseline bundle", saved_names, help="Compare a saved run with the latest run in this session.")
with st.expander("Interpretation", expanded=True):
st.markdown(
"""
This harness tests *operational reproducibility*, not mathematical determinism. A strong run has:
1. identical request payload hash,
2. identical output hashes,
3. identical `system_fingerprint`,
4. identical model snapshot or model identifier,
5. no streaming or tools.
For stronger evidence, use a dated model snapshot rather than a rolling alias, keep `max_completion_tokens` modest, set `temperature` to `0`, set a fixed integer `seed`, and repeat the same saved bundle later.
"""
)
left, right = st.columns([2, 1])
with left:
st.subheader("Prompt and context")
system_prompt = st.text_area(
"System prompt",
value="You are a precise assistant. Answer in exactly three bullet points.",
height=100,
)
context = st.text_area(
"Optional fixed context",
value="",
height=80,
help="Paste any external context here. It becomes part of the request payload and hash.",
)
user_prompt = st.text_area(
"User prompt",
value="Explain why reproducibility matters in LLM experiments.",
height=120,
)
with right:
st.subheader("Model parameters")
model = st.text_input(
"Model",
value="gpt-4.1-mini-2025-04-14",
help="Prefer a dated snapshot model over a rolling alias for reproducibility checks.",
)
max_completion_tokens = st.number_input("max_completion_tokens", min_value=1, max_value=4096, value=200, step=10)
include_seed = st.checkbox("Send seed", value=True)
seed = st.number_input("seed", min_value=0, max_value=2_147_483_647, value=42, step=1, disabled=not include_seed)
include_temperature = st.checkbox("Send temperature", value=True)
temperature = st.number_input("temperature", min_value=0.0, max_value=2.0, value=0.0, step=0.05, disabled=not include_temperature)
include_top_p = st.checkbox("Send top_p", value=True)
top_p = st.number_input("top_p", min_value=0.0, max_value=1.0, value=1.0, step=0.05, disabled=not include_top_p)
frequency_penalty = st.number_input("frequency_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
presence_penalty = st.number_input("presence_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
stop_raw = st.text_area("Stop sequences, one per line", value="", height=70)
include_logprobs = st.checkbox("Request logprobs", value=False, help="Useful for inspection, but not all models support it.")
top_logprobs = st.number_input("top_logprobs", min_value=0, max_value=20, value=0, step=1, disabled=not include_logprobs)
payload = build_payload(
model=model.strip(),
system_prompt=system_prompt,
user_prompt=user_prompt,
context=context,
include_seed=include_seed,
seed=int(seed),
include_temperature=include_temperature,
temperature=float(temperature),
include_top_p=include_top_p,
top_p=float(top_p),
max_completion_tokens=int(max_completion_tokens),
frequency_penalty=float(frequency_penalty),
presence_penalty=float(presence_penalty),
stop_sequences=parse_stop_sequences(stop_raw),
include_logprobs=include_logprobs,
top_logprobs=int(top_logprobs),
)
request_hash = sha256_json(payload)
st.subheader("Canonical request")
st.code(pretty_json(payload), language="json")
st.caption(f"Request payload SHA-256: `{request_hash}`")
run_clicked = st.button("Run reproducibility test", type="primary", disabled=not bool(api_key))
if not api_key:
st.info("Provide an API key in the sidebar or set OPENAI_API_KEY in your environment.")
if run_clicked:
client = OpenAI(api_key=api_key)
trials: List[Dict[str, Any]] = []
progress = st.progress(0, text="Starting calls")
for i in range(1, repetitions + 1):
progress.progress((i - 1) / repetitions, text=f"Running trial {i} of {repetitions}")
try:
trial = call_chat_completion(client, payload, i)
except Exception as exc:
trial = {
"trial_index": i,
"ok": False,
"started_at_utc": utc_now_iso(),
"error_type": type(exc).__name__,
"error": str(exc),
}
trials.append(trial)
if delay_s and i < repetitions:
time.sleep(float(delay_s))
progress.progress(1.0, text="Complete")
bundle = make_bundle(payload, trials)
path = save_bundle(bundle)
st.session_state["latest_bundle"] = bundle
st.session_state["latest_bundle_path"] = str(path)
if "latest_bundle" in st.session_state:
st.divider()
st.header("Latest run")
latest_bundle = st.session_state["latest_bundle"]
latest_path = st.session_state.get("latest_bundle_path")
st.caption(f"Saved locally as `{latest_path}`")
render_summary(latest_bundle)
bundle_json = pretty_json(latest_bundle)
st.download_button(
"Download latest run bundle JSON",
data=bundle_json,
file_name=Path(latest_path).name if latest_path else "llm_repro_run.json",
mime="application/json",
)
if selected_baseline_name:
baseline_path = DATA_DIR / selected_baseline_name
if baseline_path.exists():
baseline = load_bundle(baseline_path)
comparison = compare_bundles(baseline, latest_bundle)
st.subheader("Baseline comparison")
c1, c2, c3 = st.columns(3)
c1.metric("Same request hash", "yes" if comparison["same_request_payload_hash"] else "no")
c2.metric("Same ordered outputs", "yes" if comparison["same_ordered_output_hashes"] else "no")
c3.metric("Same fingerprint set", "yes" if comparison["same_system_fingerprint_set"] else "no")
st.code(pretty_json(comparison), language="json")
else:
st.divider()
st.header("No run yet")
st.write("Configure the prompt and parameters, then start a reproducibility test.")

4
requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
openai>=1.55.0
streamlit>=1.38.0
pandas>=2.2.0
python-dotenv>=1.0.1

63
runs/run_2026-06-28T115642755+0000_bca98569dc74.json Normal file
View File

@@ -0,0 +1,63 @@
{
"bundle_sha256_without_self": "e53559b02ebca1ca73747638060673943c8436a1842c35145ae16e7876aa6135",
"created_at_utc": "2026-06-28T11:56:42.755+00:00",
"endpoint": "v1/chat/completions",
"request_payload": {
"frequency_penalty": 0.0,
"max_completion_tokens": 200,
"messages": [
{
"content": "You are a precise assistant. Answer in exactly three bullet points.",
"role": "system"
},
{
"content": "Explain why reproducibility matters in LLM experiments.",
"role": "user"
}
],
"model": "gpt-4.1-mini-2025-04-14",
"n": 1,
"presence_penalty": 0.0,
"seed": 42,
"stream": false,
"temperature": 0.0,
"top_p": 1.0
},
"request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5",
"schema_version": "llm-repro-harness.v1",
"summary": {
"all_outputs_identical": false,
"all_response_models_identical": false,
"all_successful": false,
"all_system_fingerprints_identical": false,
"failed_trial_count": 3,
"successful_trial_count": 0,
"trial_count": 3,
"unique_content_sha256": [],
"unique_response_models": [],
"unique_system_fingerprints": []
},
"trials": [
{
"error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
"error_type": "RateLimitError",
"ok": false,
"started_at_utc": "2026-06-28T11:56:38.381+00:00",
"trial_index": 1
},
{
"error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
"error_type": "RateLimitError",
"ok": false,
"started_at_utc": "2026-06-28T11:56:40.528+00:00",
"trial_index": 2
},
{
"error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
"error_type": "RateLimitError",
"ok": false,
"started_at_utc": "2026-06-28T11:56:42.755+00:00",
"trial_index": 3
}
]
}

251
runs/run_2026-06-28T120226219+0000_bca98569dc74.json Normal file
View File

@@ -0,0 +1,251 @@
{
"bundle_sha256_without_self": "01b5de5925b7dcdee3df4a6dcdd25226963b77f6b4ae98e3f0573e4533cded93",
"created_at_utc": "2026-06-28T12:02:26.219+00:00",
"endpoint": "v1/chat/completions",
"request_payload": {
"frequency_penalty": 0.0,
"max_completion_tokens": 200,
"messages": [
{
"content": "You are a precise assistant. Answer in exactly three bullet points.",
"role": "system"
},
{
"content": "Explain why reproducibility matters in LLM experiments.",
"role": "user"
}
],
"model": "gpt-4.1-mini-2025-04-14",
"n": 1,
"presence_penalty": 0.0,
"seed": 42,
"stream": false,
"temperature": 0.0,
"top_p": 1.0
},
"request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5",
"schema_version": "llm-repro-harness.v1",
"summary": {
"all_outputs_identical": false,
"all_response_models_identical": true,
"all_successful": true,
"all_system_fingerprints_identical": true,
"failed_trial_count": 0,
"successful_trial_count": 3,
"trial_count": 3,
"unique_content_sha256": [
"1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9",
"33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1",
"9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8"
],
"unique_response_models": [
"gpt-4.1-mini-2025-04-14"
],
"unique_system_fingerprints": [
"fp_8f3c0606ab"
]
},
"trials": [
{
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development. \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.",
"content_sha256": "33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1",
"elapsed_ms": 2005.2,
"finish_reason": "stop",
"ok": true,
"raw_response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"message": {
"annotations": [],
"audio": null,
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development. \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.",
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": null
}
}
],
"created": 1782648142,
"id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G",
"model": "gpt-4.1-mini-2025-04-14",
"moderation": null,
"object": "chat.completion",
"service_tier": "default",
"system_fingerprint": "fp_8f3c0606ab",
"usage": {
"completion_tokens": 92,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 126
}
},
"response_id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G",
"response_model": "gpt-4.1-mini-2025-04-14",
"started_at_utc": "2026-06-28T12:02:21.404+00:00",
"system_fingerprint": "fp_8f3c0606ab",
"trial_index": 1,
"usage": {
"completion_tokens": 92,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 126
}
},
{
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.",
"content_sha256": "9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8",
"elapsed_ms": 1477.74,
"finish_reason": "stop",
"ok": true,
"raw_response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"message": {
"annotations": [],
"audio": null,
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.",
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": null
}
}
],
"created": 1782648143,
"id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q",
"model": "gpt-4.1-mini-2025-04-14",
"moderation": null,
"object": "chat.completion",
"service_tier": "default",
"system_fingerprint": "fp_8f3c0606ab",
"usage": {
"completion_tokens": 85,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 119
}
},
"response_id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q",
"response_model": "gpt-4.1-mini-2025-04-14",
"started_at_utc": "2026-06-28T12:02:23.410+00:00",
"system_fingerprint": "fp_8f3c0606ab",
"trial_index": 2,
"usage": {
"completion_tokens": 85,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 119
}
},
{
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.",
"content_sha256": "1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9",
"elapsed_ms": 1330.2,
"finish_reason": "stop",
"ok": true,
"raw_response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"message": {
"annotations": [],
"audio": null,
"content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors. \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications. \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.",
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": null
}
}
],
"created": 1782648145,
"id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi",
"model": "gpt-4.1-mini-2025-04-14",
"moderation": null,
"object": "chat.completion",
"service_tier": "default",
"system_fingerprint": "fp_8f3c0606ab",
"usage": {
"completion_tokens": 92,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 126
}
},
"response_id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi",
"response_model": "gpt-4.1-mini-2025-04-14",
"started_at_utc": "2026-06-28T12:02:24.888+00:00",
"system_fingerprint": "fp_8f3c0606ab",
"trial_index": 3,
"usage": {
"completion_tokens": 92,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens": 34,
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
},
"total_tokens": 126
}
}
]
}

108
scripts/launch-dev.sh Normal file
View File

@@ -0,0 +1,108 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="${LLM_REPRO_ROOT:-/mnt/DATA/git/llm-repro-ui}"
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
HOST="${LLM_REPRO_HOST:-127.0.0.1}"
PORT="${LLM_REPRO_PORT:-8501}"
OPEN_BROWSER="${OPEN_BROWSER:-1}"
LOG_DIR="$ROOT/runs/dev-launcher"
SERVER_LOG="$LOG_DIR/streamlit.log"
URL="http://$HOST:$PORT"
server_pid=""
fail() {
printf 'launch-dev: %s\n' "$*" >&2
exit 1
}
port_is_free() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
host = sys.argv[1]
port = int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((host, port))
except OSError:
raise SystemExit(1)
PY
}
wait_for_url() {
"$PYTHON" - "$1" <<'PY'
import sys
import time
import urllib.request
url = sys.argv[1]
deadline = time.monotonic() + 60
last_error = None
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if 200 <= response.status < 500:
raise SystemExit(0)
except Exception as exc: # noqa: BLE001 - printed only on timeout.
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
cleanup() {
if [ -n "${server_pid:-}" ] && kill -0 "$server_pid" 2>/dev/null; then
kill "$server_pid" 2>/dev/null || true
fi
}
trap cleanup EXIT INT TERM
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && python -m venv .venv && . .venv/bin/activate && pip install -r requirements.txt"
mkdir -p "$LOG_DIR"
: > "$SERVER_LOG"
port_is_free "$HOST" "$PORT" || fail "$URL is already in use"
if [ -z "${OPENAI_API_KEY:-}" ] && { [ ! -f "$ROOT/.env" ] || ! grep -q '^OPENAI_API_KEY=sk-' "$ROOT/.env"; }; then
printf 'Warning: OPENAI_API_KEY is not set in the environment or .env. The UI will start, but API calls need a key.\n' >&2
fi
printf 'Starting LLM Reproducibility Harness at %s\n' "$URL"
(
cd "$ROOT"
"$PYTHON" -m streamlit run app.py \
--server.address "$HOST" \
--server.port "$PORT" \
--server.headless true
) >"$SERVER_LOG" 2>&1 &
server_pid="$!"
printf 'Waiting for %s\n' "$URL"
wait_for_url "$URL" || {
tail -n 80 "$SERVER_LOG" >&2 || true
fail "Streamlit did not become reachable"
}
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
xdg-open "$URL" >/dev/null 2>&1 || true
fi
cat <<EOF
LLM Reproducibility Harness is running.
Web UI: $URL
Log:
$SERVER_LOG
Press Ctrl+C to stop the server.
EOF
wait "$server_pid"

72
verify_bundle.py Normal file
View File

@@ -0,0 +1,72 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any, Dict, List
def canonical_json(obj: Any) -> str:
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def sha256_json(obj: Any) -> str:
return sha256_text(canonical_json(obj))
def verify(path: Path) -> Dict[str, Any]:
bundle = json.loads(path.read_text(encoding="utf-8"))
payload = bundle.get("request_payload")
expected_payload_hash = bundle.get("request_payload_sha256")
actual_payload_hash = sha256_json(payload)
trial_results: List[Dict[str, Any]] = []
for trial in bundle.get("trials", []):
if not trial.get("ok"):
trial_results.append({"trial_index": trial.get("trial_index"), "ok": False, "reason": "trial_failed"})
continue
expected = trial.get("content_sha256")
actual = sha256_text(trial.get("content", ""))
trial_results.append(
{
"trial_index": trial.get("trial_index"),
"ok": expected == actual,
"expected_content_sha256": expected,
"actual_content_sha256": actual,
}
)
successful_hashes = [t.get("content_sha256") for t in bundle.get("trials", []) if t.get("ok")]
fingerprints = [str(t.get("system_fingerprint")) for t in bundle.get("trials", []) if t.get("ok")]
return {
"path": str(path),
"payload_hash_ok": expected_payload_hash == actual_payload_hash,
"expected_payload_sha256": expected_payload_hash,
"actual_payload_sha256": actual_payload_hash,
"trial_content_hashes_ok": all(t["ok"] for t in trial_results),
"all_successful_outputs_identical": len(set(successful_hashes)) <= 1 if successful_hashes else False,
"all_successful_fingerprints_identical": len(set(fingerprints)) <= 1 if fingerprints else False,
"unique_output_hashes": sorted(set(successful_hashes)),
"unique_system_fingerprints": sorted(set(fingerprints)),
"trials": trial_results,
}
def main() -> None:
parser = argparse.ArgumentParser(description="Verify hashes inside LLM reproducibility run bundles.")
parser.add_argument("bundle", nargs="+", type=Path)
args = parser.parse_args()
for path in args.bundle:
result = verify(path)
print(json.dumps(result, indent=2, sort_keys=True))
if __name__ == "__main__":
main()