Initial commit, very early alpha

.env.example

@@ -0,0 +1,2 @@
+# Copy to .env or export this variable in your shell.
+OPENAI_API_KEY=sk-your-key-here

.gitignore

@@ -0,0 +1,3 @@
+.env
+.venv/
+.vscode/

README.md

@@ -1,2 +1,66 @@
-# llm-repro-ui
+# LLM Reproducibility Harness
 
+A local Streamlit UI for testing how repeatable OpenAI Chat Completions are when you hold the prompt, context, model, seed, and generation parameters constant.
+
+The goal is to demonstrate operational reproducibility: identical request payloads, identical outputs, and identical backend metadata where available. It is not a proof that the hosted model is mathematically deterministic.
+
+## What it records
+
+Each run writes a JSON bundle under `runs/` containing:
+
+- exact request payload sent to `v1/chat/completions`
+- SHA-256 hash of the canonicalized request payload
+- repeated trial outputs
+- SHA-256 hash of each output string
+- `system_fingerprint` returned by the API, when available
+- returned model id, finish reason, usage, response id, and raw response
+
+The API key is not stored in bundles.
+
+## Setup
+
+```bash
+python -m venv .venv
+source .venv/bin/activate  # Windows: .venv\Scripts\activate
+pip install -r requirements.txt
+cp .env.example .env
+# Edit .env and set OPENAI_API_KEY, or export it in your shell.
+streamlit run app.py
+```
+
+## Recommended first experiment
+
+Use the default prompt and these settings:
+
+- model: a dated snapshot, for example `gpt-4.1-mini-2025-04-14`
+- repetitions: `3` or `5`
+- seed: `42`
+- temperature: `0`
+- top_p: `1`
+- max_completion_tokens: keep modest, for example `200`
+- streaming: off; tools: not used by this harness
+
+Then change only the seed and run again. You should usually see a different output hash while the request hash changes by exactly the seed field.
+
+## Baseline comparison
+
+After a run is saved, select it in the sidebar as a baseline and run the same request again. The comparison checks:
+
+- whether the canonical request hash is identical
+- whether output hashes are identical in the same order
+- whether the set of returned system fingerprints is identical
+
+## Offline verification
+
+To recompute bundle and output hashes without calling the API:
+
+```bash
+python verify_bundle.py runs/run_*.json
+```
+
+## Caveats
+
+- Hosted LLM reproducibility is best-effort. Matching `seed`, prompt, parameters, and `system_fingerprint` improves repeatability but does not guarantee identical text forever.
+- Rolling model aliases can change. Use snapshot model identifiers where your account and model family support them.
+- Longer completions provide more opportunities for divergence.
+- Tool calls, retrieval, web access, time-dependent prompts, and streaming add more variables. This scaffold deliberately avoids them.

app.py

@@ -0,0 +1,467 @@
+from __future__ import annotations
+
+import json
+import os
+import time
+import hashlib
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import difflib
+
+import pandas as pd
+import streamlit as st
+from dotenv import load_dotenv
+from openai import OpenAI
+
+load_dotenv()
+
+APP_SCHEMA_VERSION = "llm-repro-harness.v1"
+DATA_DIR = Path("runs")
+DATA_DIR.mkdir(exist_ok=True)
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
+
+
+def canonical_json(obj: Any) -> str:
+    """Stable JSON serialization used for request and bundle hashing."""
+    return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def pretty_json(obj: Any) -> str:
+    return json.dumps(obj, ensure_ascii=False, sort_keys=True, indent=2, default=str)
+
+
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def sha256_json(obj: Any) -> str:
+    return sha256_text(canonical_json(obj))
+
+
+def dump_model(obj: Any) -> Any:
+    """Convert OpenAI SDK / Pydantic objects to JSON-compatible structures."""
+    if obj is None:
+        return None
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump(mode="json")
+    if hasattr(obj, "to_dict_recursive"):
+        return obj.to_dict_recursive()
+    if hasattr(obj, "dict"):
+        return obj.dict()
+    return obj
+
+
+def parse_stop_sequences(raw: str) -> Optional[List[str]]:
+    values = [line for line in raw.splitlines() if line.strip()]
+    return values or None
+
+
+def extract_message_text(message: Any) -> str:
+    """Extract text content from a chat completion message without assuming a single SDK shape."""
+    content = getattr(message, "content", None)
+    if isinstance(content, str):
+        return content
+    if content is not None:
+        return canonical_json(content)
+
+    refusal = getattr(message, "refusal", None)
+    if refusal:
+        return f"[refusal] {refusal}"
+
+    return ""
+
+
+def build_payload(
+    *,
+    model: str,
+    system_prompt: str,
+    user_prompt: str,
+    context: str,
+    include_seed: bool,
+    seed: int,
+    include_temperature: bool,
+    temperature: float,
+    include_top_p: bool,
+    top_p: float,
+    max_completion_tokens: int,
+    frequency_penalty: float,
+    presence_penalty: float,
+    stop_sequences: Optional[List[str]],
+    include_logprobs: bool,
+    top_logprobs: int,
+) -> Dict[str, Any]:
+    messages: List[Dict[str, str]] = []
+    if system_prompt.strip():
+        messages.append({"role": "system", "content": system_prompt})
+    if context.strip():
+        messages.append({"role": "user", "content": f"Context:\n{context}"})
+    messages.append({"role": "user", "content": user_prompt})
+
+    payload: Dict[str, Any] = {
+        "model": model,
+        "messages": messages,
+        "max_completion_tokens": max_completion_tokens,
+        "frequency_penalty": frequency_penalty,
+        "presence_penalty": presence_penalty,
+        "n": 1,
+        "stream": False,
+    }
+
+    if include_seed:
+        payload["seed"] = int(seed)
+    if include_temperature:
+        payload["temperature"] = float(temperature)
+    if include_top_p:
+        payload["top_p"] = float(top_p)
+    if stop_sequences:
+        payload["stop"] = stop_sequences
+    if include_logprobs:
+        payload["logprobs"] = True
+        payload["top_logprobs"] = int(top_logprobs)
+
+    return payload
+
+
+def call_chat_completion(client: OpenAI, payload: Dict[str, Any], trial_index: int) -> Dict[str, Any]:
+    started_at = utc_now_iso()
+    start_time = time.perf_counter()
+
+    response = client.chat.completions.create(**payload)
+
+    elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2)
+    response_dict = dump_model(response)
+    choice = response.choices[0]
+    message = choice.message
+    content = extract_message_text(message)
+
+    trial = {
+        "trial_index": trial_index,
+        "ok": True,
+        "started_at_utc": started_at,
+        "elapsed_ms": elapsed_ms,
+        "response_id": getattr(response, "id", None),
+        "response_model": getattr(response, "model", None),
+        "system_fingerprint": getattr(response, "system_fingerprint", None),
+        "finish_reason": getattr(choice, "finish_reason", None),
+        "content": content,
+        "content_sha256": sha256_text(content),
+        "usage": dump_model(getattr(response, "usage", None)),
+        "raw_response": response_dict,
+    }
+    return trial
+
+
+def summarize_trials(trials: List[Dict[str, Any]]) -> Dict[str, Any]:
+    successful = [t for t in trials if t.get("ok")]
+    failed = [t for t in trials if not t.get("ok")]
+
+    content_hashes = [t["content_sha256"] for t in successful]
+    fingerprints = [t.get("system_fingerprint") for t in successful]
+    response_models = [t.get("response_model") for t in successful]
+
+    return {
+        "trial_count": len(trials),
+        "successful_trial_count": len(successful),
+        "failed_trial_count": len(failed),
+        "all_successful": len(failed) == 0,
+        "all_outputs_identical": len(set(content_hashes)) <= 1 if successful else False,
+        "all_system_fingerprints_identical": len(set(fingerprints)) <= 1 if successful else False,
+        "all_response_models_identical": len(set(response_models)) <= 1 if successful else False,
+        "unique_content_sha256": sorted(set(content_hashes)),
+        "unique_system_fingerprints": sorted({str(x) for x in fingerprints}),
+        "unique_response_models": sorted({str(x) for x in response_models}),
+    }
+
+
+def make_bundle(payload: Dict[str, Any], trials: List[Dict[str, Any]]) -> Dict[str, Any]:
+    request_hash = sha256_json(payload)
+    bundle = {
+        "schema_version": APP_SCHEMA_VERSION,
+        "created_at_utc": utc_now_iso(),
+        "endpoint": "v1/chat/completions",
+        "request_payload": payload,
+        "request_payload_sha256": request_hash,
+        "trials": trials,
+        "summary": summarize_trials(trials),
+    }
+    bundle["bundle_sha256_without_self"] = sha256_json(bundle)
+    return bundle
+
+
+def save_bundle(bundle: Dict[str, Any]) -> Path:
+    created = bundle["created_at_utc"].replace(":", "").replace(".", "")
+    request_hash_short = bundle["request_payload_sha256"][:12]
+    filename = f"run_{created}_{request_hash_short}.json"
+    path = DATA_DIR / filename
+    path.write_text(pretty_json(bundle), encoding="utf-8")
+    return path
+
+
+def load_bundle(path: Path) -> Dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def list_saved_runs() -> List[Path]:
+    return sorted(DATA_DIR.glob("run_*.json"), reverse=True)
+
+
+def compare_bundles(baseline: Dict[str, Any], candidate: Dict[str, Any]) -> Dict[str, Any]:
+    b_hashes = [t.get("content_sha256") for t in baseline.get("trials", []) if t.get("ok")]
+    c_hashes = [t.get("content_sha256") for t in candidate.get("trials", []) if t.get("ok")]
+    return {
+        "same_request_payload_hash": baseline.get("request_payload_sha256") == candidate.get("request_payload_sha256"),
+        "same_ordered_output_hashes": b_hashes == c_hashes,
+        "same_output_hash_set": set(b_hashes) == set(c_hashes),
+        "baseline_output_hashes": b_hashes,
+        "candidate_output_hashes": c_hashes,
+        "baseline_system_fingerprints": baseline.get("summary", {}).get("unique_system_fingerprints", []),
+        "candidate_system_fingerprints": candidate.get("summary", {}).get("unique_system_fingerprints", []),
+        "same_system_fingerprint_set": set(baseline.get("summary", {}).get("unique_system_fingerprints", []))
+        == set(candidate.get("summary", {}).get("unique_system_fingerprints", [])),
+    }
+
+
+def render_summary(bundle: Dict[str, Any]) -> None:
+    summary = bundle["summary"]
+    col1, col2, col3, col4 = st.columns(4)
+    col1.metric("Trials", summary["trial_count"])
+    col2.metric("Successful", summary["successful_trial_count"])
+    col3.metric("Unique outputs", len(summary["unique_content_sha256"]))
+    col4.metric("Unique fingerprints", len(summary["unique_system_fingerprints"]))
+
+    if summary["all_successful"] and summary["all_outputs_identical"] and summary["all_system_fingerprints_identical"]:
+        st.success("Within this run, the outputs were exactly identical and the system fingerprint was unchanged.")
+    elif summary["all_successful"] and summary["all_outputs_identical"]:
+        st.warning("Outputs were identical, but the system fingerprint varied or was unavailable.")
+    elif summary["all_successful"]:
+        st.warning("Outputs differed. Inspect the fingerprints, finish reasons, token counts, and diff below.")
+    else:
+        st.error("At least one trial failed. Inspect the error rows below.")
+
+    rows = []
+    for trial in bundle["trials"]:
+        usage = trial.get("usage") or {}
+        rows.append(
+            {
+                "trial": trial.get("trial_index"),
+                "ok": trial.get("ok"),
+                "content_sha256": trial.get("content_sha256", "")[:16],
+                "system_fingerprint": trial.get("system_fingerprint"),
+                "response_model": trial.get("response_model"),
+                "finish_reason": trial.get("finish_reason"),
+                "elapsed_ms": trial.get("elapsed_ms"),
+                "prompt_tokens": usage.get("prompt_tokens"),
+                "completion_tokens": usage.get("completion_tokens"),
+                "total_tokens": usage.get("total_tokens"),
+                "error": trial.get("error"),
+            }
+        )
+    st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)
+
+    successful = [t for t in bundle["trials"] if t.get("ok")]
+    if successful and not summary["all_outputs_identical"]:
+        first = successful[0]
+        different = next((t for t in successful[1:] if t["content_sha256"] != first["content_sha256"]), None)
+        if different:
+            st.subheader("First output diff")
+            diff = difflib.unified_diff(
+                first["content"].splitlines(),
+                different["content"].splitlines(),
+                fromfile=f"trial_{first['trial_index']}",
+                tofile=f"trial_{different['trial_index']}",
+                lineterm="",
+                n=3,
+            )
+            st.code("\n".join(diff)[:12000], language="diff")
+
+    st.subheader("Outputs")
+    for trial in bundle["trials"]:
+        label = f"Trial {trial.get('trial_index')}"
+        if trial.get("ok"):
+            label += f" | {trial.get('content_sha256', '')[:16]} | {trial.get('system_fingerprint')}"
+            with st.expander(label, expanded=False):
+                st.write(trial.get("content", ""))
+        else:
+            label += " | failed"
+            with st.expander(label, expanded=True):
+                st.code(trial.get("error", ""))
+
+
+st.set_page_config(page_title="LLM Reproducibility Harness", layout="wide")
+st.title("LLM Reproducibility Harness")
+st.caption("Local UI for testing repeated Chat Completions with fixed request parameters, seed, and recorded backend metadata.")
+
+with st.sidebar:
+    st.header("Connection")
+    env_key_present = bool(os.environ.get("OPENAI_API_KEY"))
+    api_key_input = st.text_input(
+        "OpenAI API key",
+        type="password",
+        placeholder="Leave blank to use OPENAI_API_KEY" if env_key_present else "sk-...",
+        help="The key is used only for API calls and is not written to run bundles.",
+    )
+    api_key = api_key_input.strip() or os.environ.get("OPENAI_API_KEY", "")
+    if env_key_present and not api_key_input:
+        st.caption("Using OPENAI_API_KEY from environment or .env.")
+
+    st.header("Run controls")
+    repetitions = st.slider("Repetitions", min_value=1, max_value=12, value=3, help="Use at least 2 to demonstrate exact repeatability.")
+    delay_s = st.number_input("Delay between calls, seconds", min_value=0.0, max_value=10.0, value=0.0, step=0.25)
+
+    st.header("Saved baselines")
+    saved = list_saved_runs()
+    saved_names = [""] + [p.name for p in saved]
+    selected_baseline_name = st.selectbox("Baseline bundle", saved_names, help="Compare a saved run with the latest run in this session.")
+
+with st.expander("Interpretation", expanded=True):
+    st.markdown(
+        """
+This harness tests *operational reproducibility*, not mathematical determinism. A strong run has:
+
+1. identical request payload hash,
+2. identical output hashes,
+3. identical `system_fingerprint`,
+4. identical model snapshot or model identifier,
+5. no streaming or tools.
+
+For stronger evidence, use a dated model snapshot rather than a rolling alias, keep `max_completion_tokens` modest, set `temperature` to `0`, set a fixed integer `seed`, and repeat the same saved bundle later.
+"""
+    )
+
+left, right = st.columns([2, 1])
+
+with left:
+    st.subheader("Prompt and context")
+    system_prompt = st.text_area(
+        "System prompt",
+        value="You are a precise assistant. Answer in exactly three bullet points.",
+        height=100,
+    )
+    context = st.text_area(
+        "Optional fixed context",
+        value="",
+        height=80,
+        help="Paste any external context here. It becomes part of the request payload and hash.",
+    )
+    user_prompt = st.text_area(
+        "User prompt",
+        value="Explain why reproducibility matters in LLM experiments.",
+        height=120,
+    )
+
+with right:
+    st.subheader("Model parameters")
+    model = st.text_input(
+        "Model",
+        value="gpt-4.1-mini-2025-04-14",
+        help="Prefer a dated snapshot model over a rolling alias for reproducibility checks.",
+    )
+    max_completion_tokens = st.number_input("max_completion_tokens", min_value=1, max_value=4096, value=200, step=10)
+
+    include_seed = st.checkbox("Send seed", value=True)
+    seed = st.number_input("seed", min_value=0, max_value=2_147_483_647, value=42, step=1, disabled=not include_seed)
+
+    include_temperature = st.checkbox("Send temperature", value=True)
+    temperature = st.number_input("temperature", min_value=0.0, max_value=2.0, value=0.0, step=0.05, disabled=not include_temperature)
+
+    include_top_p = st.checkbox("Send top_p", value=True)
+    top_p = st.number_input("top_p", min_value=0.0, max_value=1.0, value=1.0, step=0.05, disabled=not include_top_p)
+
+    frequency_penalty = st.number_input("frequency_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
+    presence_penalty = st.number_input("presence_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
+
+    stop_raw = st.text_area("Stop sequences, one per line", value="", height=70)
+    include_logprobs = st.checkbox("Request logprobs", value=False, help="Useful for inspection, but not all models support it.")
+    top_logprobs = st.number_input("top_logprobs", min_value=0, max_value=20, value=0, step=1, disabled=not include_logprobs)
+
+payload = build_payload(
+    model=model.strip(),
+    system_prompt=system_prompt,
+    user_prompt=user_prompt,
+    context=context,
+    include_seed=include_seed,
+    seed=int(seed),
+    include_temperature=include_temperature,
+    temperature=float(temperature),
+    include_top_p=include_top_p,
+    top_p=float(top_p),
+    max_completion_tokens=int(max_completion_tokens),
+    frequency_penalty=float(frequency_penalty),
+    presence_penalty=float(presence_penalty),
+    stop_sequences=parse_stop_sequences(stop_raw),
+    include_logprobs=include_logprobs,
+    top_logprobs=int(top_logprobs),
+)
+
+request_hash = sha256_json(payload)
+
+st.subheader("Canonical request")
+st.code(pretty_json(payload), language="json")
+st.caption(f"Request payload SHA-256: `{request_hash}`")
+
+run_clicked = st.button("Run reproducibility test", type="primary", disabled=not bool(api_key))
+if not api_key:
+    st.info("Provide an API key in the sidebar or set OPENAI_API_KEY in your environment.")
+
+if run_clicked:
+    client = OpenAI(api_key=api_key)
+    trials: List[Dict[str, Any]] = []
+    progress = st.progress(0, text="Starting calls")
+
+    for i in range(1, repetitions + 1):
+        progress.progress((i - 1) / repetitions, text=f"Running trial {i} of {repetitions}")
+        try:
+            trial = call_chat_completion(client, payload, i)
+        except Exception as exc:
+            trial = {
+                "trial_index": i,
+                "ok": False,
+                "started_at_utc": utc_now_iso(),
+                "error_type": type(exc).__name__,
+                "error": str(exc),
+            }
+        trials.append(trial)
+        if delay_s and i < repetitions:
+            time.sleep(float(delay_s))
+
+    progress.progress(1.0, text="Complete")
+    bundle = make_bundle(payload, trials)
+    path = save_bundle(bundle)
+    st.session_state["latest_bundle"] = bundle
+    st.session_state["latest_bundle_path"] = str(path)
+
+if "latest_bundle" in st.session_state:
+    st.divider()
+    st.header("Latest run")
+    latest_bundle = st.session_state["latest_bundle"]
+    latest_path = st.session_state.get("latest_bundle_path")
+    st.caption(f"Saved locally as `{latest_path}`")
+    render_summary(latest_bundle)
+
+    bundle_json = pretty_json(latest_bundle)
+    st.download_button(
+        "Download latest run bundle JSON",
+        data=bundle_json,
+        file_name=Path(latest_path).name if latest_path else "llm_repro_run.json",
+        mime="application/json",
+    )
+
+    if selected_baseline_name:
+        baseline_path = DATA_DIR / selected_baseline_name
+        if baseline_path.exists():
+            baseline = load_bundle(baseline_path)
+            comparison = compare_bundles(baseline, latest_bundle)
+            st.subheader("Baseline comparison")
+            c1, c2, c3 = st.columns(3)
+            c1.metric("Same request hash", "yes" if comparison["same_request_payload_hash"] else "no")
+            c2.metric("Same ordered outputs", "yes" if comparison["same_ordered_output_hashes"] else "no")
+            c3.metric("Same fingerprint set", "yes" if comparison["same_system_fingerprint_set"] else "no")
+            st.code(pretty_json(comparison), language="json")
+else:
+    st.divider()
+    st.header("No run yet")
+    st.write("Configure the prompt and parameters, then start a reproducibility test.")

requirements.txt

@@ -0,0 +1,4 @@
+openai>=1.55.0
+streamlit>=1.38.0
+pandas>=2.2.0
+python-dotenv>=1.0.1

runs/run_2026-06-28T115642755+0000_bca98569dc74.json

@@ -0,0 +1,63 @@
+{
+  "bundle_sha256_without_self": "e53559b02ebca1ca73747638060673943c8436a1842c35145ae16e7876aa6135",
+  "created_at_utc": "2026-06-28T11:56:42.755+00:00",
+  "endpoint": "v1/chat/completions",
+  "request_payload": {
+    "frequency_penalty": 0.0,
+    "max_completion_tokens": 200,
+    "messages": [
+      {
+        "content": "You are a precise assistant. Answer in exactly three bullet points.",
+        "role": "system"
+      },
+      {
+        "content": "Explain why reproducibility matters in LLM experiments.",
+        "role": "user"
+      }
+    ],
+    "model": "gpt-4.1-mini-2025-04-14",
+    "n": 1,
+    "presence_penalty": 0.0,
+    "seed": 42,
+    "stream": false,
+    "temperature": 0.0,
+    "top_p": 1.0
+  },
+  "request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5",
+  "schema_version": "llm-repro-harness.v1",
+  "summary": {
+    "all_outputs_identical": false,
+    "all_response_models_identical": false,
+    "all_successful": false,
+    "all_system_fingerprints_identical": false,
+    "failed_trial_count": 3,
+    "successful_trial_count": 0,
+    "trial_count": 3,
+    "unique_content_sha256": [],
+    "unique_response_models": [],
+    "unique_system_fingerprints": []
+  },
+  "trials": [
+    {
+      "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
+      "error_type": "RateLimitError",
+      "ok": false,
+      "started_at_utc": "2026-06-28T11:56:38.381+00:00",
+      "trial_index": 1
+    },
+    {
+      "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
+      "error_type": "RateLimitError",
+      "ok": false,
+      "started_at_utc": "2026-06-28T11:56:40.528+00:00",
+      "trial_index": 2
+    },
+    {
+      "error": "Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}",
+      "error_type": "RateLimitError",
+      "ok": false,
+      "started_at_utc": "2026-06-28T11:56:42.755+00:00",
+      "trial_index": 3
+    }
+  ]
+}

runs/run_2026-06-28T120226219+0000_bca98569dc74.json

@@ -0,0 +1,251 @@
+{
+  "bundle_sha256_without_self": "01b5de5925b7dcdee3df4a6dcdd25226963b77f6b4ae98e3f0573e4533cded93",
+  "created_at_utc": "2026-06-28T12:02:26.219+00:00",
+  "endpoint": "v1/chat/completions",
+  "request_payload": {
+    "frequency_penalty": 0.0,
+    "max_completion_tokens": 200,
+    "messages": [
+      {
+        "content": "You are a precise assistant. Answer in exactly three bullet points.",
+        "role": "system"
+      },
+      {
+        "content": "Explain why reproducibility matters in LLM experiments.",
+        "role": "user"
+      }
+    ],
+    "model": "gpt-4.1-mini-2025-04-14",
+    "n": 1,
+    "presence_penalty": 0.0,
+    "seed": 42,
+    "stream": false,
+    "temperature": 0.0,
+    "top_p": 1.0
+  },
+  "request_payload_sha256": "bca98569dc74b6ce719cccef6bd96397691b1656b62bdd646c32e7408546c1b5",
+  "schema_version": "llm-repro-harness.v1",
+  "summary": {
+    "all_outputs_identical": false,
+    "all_response_models_identical": true,
+    "all_successful": true,
+    "all_system_fingerprints_identical": true,
+    "failed_trial_count": 0,
+    "successful_trial_count": 3,
+    "trial_count": 3,
+    "unique_content_sha256": [
+      "1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9",
+      "33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1",
+      "9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8"
+    ],
+    "unique_response_models": [
+      "gpt-4.1-mini-2025-04-14"
+    ],
+    "unique_system_fingerprints": [
+      "fp_8f3c0606ab"
+    ]
+  },
+  "trials": [
+    {
+      "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development.  \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.",
+      "content_sha256": "33b675b86a0f7ff5206d75f0d81763c9928aec97614de5758100ed6bd1285db1",
+      "elapsed_ms": 2005.2,
+      "finish_reason": "stop",
+      "ok": true,
+      "raw_response": {
+        "choices": [
+          {
+            "finish_reason": "stop",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+              "annotations": [],
+              "audio": null,
+              "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: By providing clear methodologies and data, reproducibility enables others to build upon existing work, accelerating advancements in LLM development.  \n- Enhances transparency: Reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases in LLM models.",
+              "function_call": null,
+              "refusal": null,
+              "role": "assistant",
+              "tool_calls": null
+            }
+          }
+        ],
+        "created": 1782648142,
+        "id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G",
+        "model": "gpt-4.1-mini-2025-04-14",
+        "moderation": null,
+        "object": "chat.completion",
+        "service_tier": "default",
+        "system_fingerprint": "fp_8f3c0606ab",
+        "usage": {
+          "completion_tokens": 92,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens": 34,
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          },
+          "total_tokens": 126
+        }
+      },
+      "response_id": "chatcmpl-DviVKgREgVmCI5v8UdAcIGdQxIo4G",
+      "response_model": "gpt-4.1-mini-2025-04-14",
+      "started_at_utc": "2026-06-28T12:02:21.404+00:00",
+      "system_fingerprint": "fp_8f3c0606ab",
+      "trial_index": 1,
+      "usage": {
+        "completion_tokens": 92,
+        "completion_tokens_details": {
+          "accepted_prediction_tokens": 0,
+          "audio_tokens": 0,
+          "reasoning_tokens": 0,
+          "rejected_prediction_tokens": 0
+        },
+        "prompt_tokens": 34,
+        "prompt_tokens_details": {
+          "audio_tokens": 0,
+          "cached_tokens": 0
+        },
+        "total_tokens": 126
+      }
+    },
+    {
+      "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development.  \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.",
+      "content_sha256": "9f9d595b9bea32730625366fc01f1dc4d22854ad9cf583834106f86062d4e3d8",
+      "elapsed_ms": 1477.74,
+      "finish_reason": "stop",
+      "ok": true,
+      "raw_response": {
+        "choices": [
+          {
+            "finish_reason": "stop",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+              "annotations": [],
+              "audio": null,
+              "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: By enabling others to build upon validated work, reproducibility accelerates innovation and improvement in LLM development.  \n- Enhances transparency: Clear, reproducible experiments promote openness and trust in the research community, helping to identify limitations and biases.",
+              "function_call": null,
+              "refusal": null,
+              "role": "assistant",
+              "tool_calls": null
+            }
+          }
+        ],
+        "created": 1782648143,
+        "id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q",
+        "model": "gpt-4.1-mini-2025-04-14",
+        "moderation": null,
+        "object": "chat.completion",
+        "service_tier": "default",
+        "system_fingerprint": "fp_8f3c0606ab",
+        "usage": {
+          "completion_tokens": 85,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens": 34,
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          },
+          "total_tokens": 119
+        }
+      },
+      "response_id": "chatcmpl-DviVLp2D8EmQ3A6VCHfB8NCdCrp0q",
+      "response_model": "gpt-4.1-mini-2025-04-14",
+      "started_at_utc": "2026-06-28T12:02:23.410+00:00",
+      "system_fingerprint": "fp_8f3c0606ab",
+      "trial_index": 2,
+      "usage": {
+        "completion_tokens": 85,
+        "completion_tokens_details": {
+          "accepted_prediction_tokens": 0,
+          "audio_tokens": 0,
+          "reasoning_tokens": 0,
+          "rejected_prediction_tokens": 0
+        },
+        "prompt_tokens": 34,
+        "prompt_tokens_details": {
+          "audio_tokens": 0,
+          "cached_tokens": 0
+        },
+        "total_tokens": 119
+      }
+    },
+    {
+      "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications.  \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.",
+      "content_sha256": "1599a4c919502157127047d0fe3362d9f804da7bb48e5ea144ae3cfd86f58dc9",
+      "elapsed_ms": 1330.2,
+      "finish_reason": "stop",
+      "ok": true,
+      "raw_response": {
+        "choices": [
+          {
+            "finish_reason": "stop",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+              "annotations": [],
+              "audio": null,
+              "content": "- Ensures reliability: Reproducibility allows researchers to verify results by independently replicating experiments, confirming that findings are not due to chance or errors.  \n- Facilitates progress: It enables the research community to build upon previous work confidently, accelerating advancements in LLM development and applications.  \n- Enhances transparency: Clear, reproducible experiments promote openness and trust, making it easier to identify limitations, biases, and areas for improvement in LLM models.",
+              "function_call": null,
+              "refusal": null,
+              "role": "assistant",
+              "tool_calls": null
+            }
+          }
+        ],
+        "created": 1782648145,
+        "id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi",
+        "model": "gpt-4.1-mini-2025-04-14",
+        "moderation": null,
+        "object": "chat.completion",
+        "service_tier": "default",
+        "system_fingerprint": "fp_8f3c0606ab",
+        "usage": {
+          "completion_tokens": 92,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens": 34,
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          },
+          "total_tokens": 126
+        }
+      },
+      "response_id": "chatcmpl-DviVN4bVJuFvdj82Jcu2uQN4xEcfi",
+      "response_model": "gpt-4.1-mini-2025-04-14",
+      "started_at_utc": "2026-06-28T12:02:24.888+00:00",
+      "system_fingerprint": "fp_8f3c0606ab",
+      "trial_index": 3,
+      "usage": {
+        "completion_tokens": 92,
+        "completion_tokens_details": {
+          "accepted_prediction_tokens": 0,
+          "audio_tokens": 0,
+          "reasoning_tokens": 0,
+          "rejected_prediction_tokens": 0
+        },
+        "prompt_tokens": 34,
+        "prompt_tokens_details": {
+          "audio_tokens": 0,
+          "cached_tokens": 0
+        },
+        "total_tokens": 126
+      }
+    }
+  ]
+}

scripts/launch-dev.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${LLM_REPRO_ROOT:-/mnt/DATA/git/llm-repro-ui}"
+PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
+HOST="${LLM_REPRO_HOST:-127.0.0.1}"
+PORT="${LLM_REPRO_PORT:-8501}"
+OPEN_BROWSER="${OPEN_BROWSER:-1}"
+
+LOG_DIR="$ROOT/runs/dev-launcher"
+SERVER_LOG="$LOG_DIR/streamlit.log"
+URL="http://$HOST:$PORT"
+
+server_pid=""
+
+fail() {
+  printf 'launch-dev: %s\n' "$*" >&2
+  exit 1
+}
+
+port_is_free() {
+  "$PYTHON" - "$1" "$2" <<'PY'
+import socket
+import sys
+
+host = sys.argv[1]
+port = int(sys.argv[2])
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    try:
+        sock.bind((host, port))
+    except OSError:
+        raise SystemExit(1)
+PY
+}
+
+wait_for_url() {
+  "$PYTHON" - "$1" <<'PY'
+import sys
+import time
+import urllib.request
+
+url = sys.argv[1]
+deadline = time.monotonic() + 60
+last_error = None
+while time.monotonic() < deadline:
+    try:
+        with urllib.request.urlopen(url, timeout=2) as response:
+            if 200 <= response.status < 500:
+                raise SystemExit(0)
+    except Exception as exc:  # noqa: BLE001 - printed only on timeout.
+        last_error = exc
+    time.sleep(1)
+print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
+raise SystemExit(1)
+PY
+}
+
+cleanup() {
+  if [ -n "${server_pid:-}" ] && kill -0 "$server_pid" 2>/dev/null; then
+    kill "$server_pid" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && python -m venv .venv && . .venv/bin/activate && pip install -r requirements.txt"
+
+mkdir -p "$LOG_DIR"
+: > "$SERVER_LOG"
+
+port_is_free "$HOST" "$PORT" || fail "$URL is already in use"
+
+if [ -z "${OPENAI_API_KEY:-}" ] && { [ ! -f "$ROOT/.env" ] || ! grep -q '^OPENAI_API_KEY=sk-' "$ROOT/.env"; }; then
+  printf 'Warning: OPENAI_API_KEY is not set in the environment or .env. The UI will start, but API calls need a key.\n' >&2
+fi
+
+printf 'Starting LLM Reproducibility Harness at %s\n' "$URL"
+(
+  cd "$ROOT"
+  "$PYTHON" -m streamlit run app.py \
+    --server.address "$HOST" \
+    --server.port "$PORT" \
+    --server.headless true
+) >"$SERVER_LOG" 2>&1 &
+server_pid="$!"
+
+printf 'Waiting for %s\n' "$URL"
+wait_for_url "$URL" || {
+  tail -n 80 "$SERVER_LOG" >&2 || true
+  fail "Streamlit did not become reachable"
+}
+
+if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
+  xdg-open "$URL" >/dev/null 2>&1 || true
+fi
+
+cat <<EOF
+
+LLM Reproducibility Harness is running.
+Web UI: $URL
+
+Log:
+$SERVER_LOG
+
+Press Ctrl+C to stop the server.
+EOF
+
+wait "$server_pid"

verify_bundle.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+def canonical_json(obj: Any) -> str:
+    return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def sha256_json(obj: Any) -> str:
+    return sha256_text(canonical_json(obj))
+
+
+def verify(path: Path) -> Dict[str, Any]:
+    bundle = json.loads(path.read_text(encoding="utf-8"))
+    payload = bundle.get("request_payload")
+    expected_payload_hash = bundle.get("request_payload_sha256")
+    actual_payload_hash = sha256_json(payload)
+
+    trial_results: List[Dict[str, Any]] = []
+    for trial in bundle.get("trials", []):
+        if not trial.get("ok"):
+            trial_results.append({"trial_index": trial.get("trial_index"), "ok": False, "reason": "trial_failed"})
+            continue
+        expected = trial.get("content_sha256")
+        actual = sha256_text(trial.get("content", ""))
+        trial_results.append(
+            {
+                "trial_index": trial.get("trial_index"),
+                "ok": expected == actual,
+                "expected_content_sha256": expected,
+                "actual_content_sha256": actual,
+            }
+        )
+
+    successful_hashes = [t.get("content_sha256") for t in bundle.get("trials", []) if t.get("ok")]
+    fingerprints = [str(t.get("system_fingerprint")) for t in bundle.get("trials", []) if t.get("ok")]
+
+    return {
+        "path": str(path),
+        "payload_hash_ok": expected_payload_hash == actual_payload_hash,
+        "expected_payload_sha256": expected_payload_hash,
+        "actual_payload_sha256": actual_payload_hash,
+        "trial_content_hashes_ok": all(t["ok"] for t in trial_results),
+        "all_successful_outputs_identical": len(set(successful_hashes)) <= 1 if successful_hashes else False,
+        "all_successful_fingerprints_identical": len(set(fingerprints)) <= 1 if fingerprints else False,
+        "unique_output_hashes": sorted(set(successful_hashes)),
+        "unique_system_fingerprints": sorted(set(fingerprints)),
+        "trials": trial_results,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Verify hashes inside LLM reproducibility run bundles.")
+    parser.add_argument("bundle", nargs="+", type=Path)
+    args = parser.parse_args()
+
+    for path in args.bundle:
+        result = verify(path)
+        print(json.dumps(result, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()