Initial commit, very early alpha
This commit is contained in:
467
app.py
Normal file
467
app.py
Normal file
@@ -0,0 +1,467 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
import difflib
|
||||
|
||||
import pandas as pd
|
||||
import streamlit as st
|
||||
from dotenv import load_dotenv
|
||||
from openai import OpenAI
|
||||
|
||||
load_dotenv()
|
||||
|
||||
APP_SCHEMA_VERSION = "llm-repro-harness.v1"
|
||||
DATA_DIR = Path("runs")
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
||||
|
||||
|
||||
def canonical_json(obj: Any) -> str:
|
||||
"""Stable JSON serialization used for request and bundle hashing."""
|
||||
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def pretty_json(obj: Any) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False, sort_keys=True, indent=2, default=str)
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def sha256_json(obj: Any) -> str:
|
||||
return sha256_text(canonical_json(obj))
|
||||
|
||||
|
||||
def dump_model(obj: Any) -> Any:
|
||||
"""Convert OpenAI SDK / Pydantic objects to JSON-compatible structures."""
|
||||
if obj is None:
|
||||
return None
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump(mode="json")
|
||||
if hasattr(obj, "to_dict_recursive"):
|
||||
return obj.to_dict_recursive()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj
|
||||
|
||||
|
||||
def parse_stop_sequences(raw: str) -> Optional[List[str]]:
|
||||
values = [line for line in raw.splitlines() if line.strip()]
|
||||
return values or None
|
||||
|
||||
|
||||
def extract_message_text(message: Any) -> str:
|
||||
"""Extract text content from a chat completion message without assuming a single SDK shape."""
|
||||
content = getattr(message, "content", None)
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if content is not None:
|
||||
return canonical_json(content)
|
||||
|
||||
refusal = getattr(message, "refusal", None)
|
||||
if refusal:
|
||||
return f"[refusal] {refusal}"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def build_payload(
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
context: str,
|
||||
include_seed: bool,
|
||||
seed: int,
|
||||
include_temperature: bool,
|
||||
temperature: float,
|
||||
include_top_p: bool,
|
||||
top_p: float,
|
||||
max_completion_tokens: int,
|
||||
frequency_penalty: float,
|
||||
presence_penalty: float,
|
||||
stop_sequences: Optional[List[str]],
|
||||
include_logprobs: bool,
|
||||
top_logprobs: int,
|
||||
) -> Dict[str, Any]:
|
||||
messages: List[Dict[str, str]] = []
|
||||
if system_prompt.strip():
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
if context.strip():
|
||||
messages.append({"role": "user", "content": f"Context:\n{context}"})
|
||||
messages.append({"role": "user", "content": user_prompt})
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_completion_tokens": max_completion_tokens,
|
||||
"frequency_penalty": frequency_penalty,
|
||||
"presence_penalty": presence_penalty,
|
||||
"n": 1,
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
if include_seed:
|
||||
payload["seed"] = int(seed)
|
||||
if include_temperature:
|
||||
payload["temperature"] = float(temperature)
|
||||
if include_top_p:
|
||||
payload["top_p"] = float(top_p)
|
||||
if stop_sequences:
|
||||
payload["stop"] = stop_sequences
|
||||
if include_logprobs:
|
||||
payload["logprobs"] = True
|
||||
payload["top_logprobs"] = int(top_logprobs)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def call_chat_completion(client: OpenAI, payload: Dict[str, Any], trial_index: int) -> Dict[str, Any]:
|
||||
started_at = utc_now_iso()
|
||||
start_time = time.perf_counter()
|
||||
|
||||
response = client.chat.completions.create(**payload)
|
||||
|
||||
elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2)
|
||||
response_dict = dump_model(response)
|
||||
choice = response.choices[0]
|
||||
message = choice.message
|
||||
content = extract_message_text(message)
|
||||
|
||||
trial = {
|
||||
"trial_index": trial_index,
|
||||
"ok": True,
|
||||
"started_at_utc": started_at,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"response_id": getattr(response, "id", None),
|
||||
"response_model": getattr(response, "model", None),
|
||||
"system_fingerprint": getattr(response, "system_fingerprint", None),
|
||||
"finish_reason": getattr(choice, "finish_reason", None),
|
||||
"content": content,
|
||||
"content_sha256": sha256_text(content),
|
||||
"usage": dump_model(getattr(response, "usage", None)),
|
||||
"raw_response": response_dict,
|
||||
}
|
||||
return trial
|
||||
|
||||
|
||||
def summarize_trials(trials: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
successful = [t for t in trials if t.get("ok")]
|
||||
failed = [t for t in trials if not t.get("ok")]
|
||||
|
||||
content_hashes = [t["content_sha256"] for t in successful]
|
||||
fingerprints = [t.get("system_fingerprint") for t in successful]
|
||||
response_models = [t.get("response_model") for t in successful]
|
||||
|
||||
return {
|
||||
"trial_count": len(trials),
|
||||
"successful_trial_count": len(successful),
|
||||
"failed_trial_count": len(failed),
|
||||
"all_successful": len(failed) == 0,
|
||||
"all_outputs_identical": len(set(content_hashes)) <= 1 if successful else False,
|
||||
"all_system_fingerprints_identical": len(set(fingerprints)) <= 1 if successful else False,
|
||||
"all_response_models_identical": len(set(response_models)) <= 1 if successful else False,
|
||||
"unique_content_sha256": sorted(set(content_hashes)),
|
||||
"unique_system_fingerprints": sorted({str(x) for x in fingerprints}),
|
||||
"unique_response_models": sorted({str(x) for x in response_models}),
|
||||
}
|
||||
|
||||
|
||||
def make_bundle(payload: Dict[str, Any], trials: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
request_hash = sha256_json(payload)
|
||||
bundle = {
|
||||
"schema_version": APP_SCHEMA_VERSION,
|
||||
"created_at_utc": utc_now_iso(),
|
||||
"endpoint": "v1/chat/completions",
|
||||
"request_payload": payload,
|
||||
"request_payload_sha256": request_hash,
|
||||
"trials": trials,
|
||||
"summary": summarize_trials(trials),
|
||||
}
|
||||
bundle["bundle_sha256_without_self"] = sha256_json(bundle)
|
||||
return bundle
|
||||
|
||||
|
||||
def save_bundle(bundle: Dict[str, Any]) -> Path:
|
||||
created = bundle["created_at_utc"].replace(":", "").replace(".", "")
|
||||
request_hash_short = bundle["request_payload_sha256"][:12]
|
||||
filename = f"run_{created}_{request_hash_short}.json"
|
||||
path = DATA_DIR / filename
|
||||
path.write_text(pretty_json(bundle), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def load_bundle(path: Path) -> Dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def list_saved_runs() -> List[Path]:
|
||||
return sorted(DATA_DIR.glob("run_*.json"), reverse=True)
|
||||
|
||||
|
||||
def compare_bundles(baseline: Dict[str, Any], candidate: Dict[str, Any]) -> Dict[str, Any]:
|
||||
b_hashes = [t.get("content_sha256") for t in baseline.get("trials", []) if t.get("ok")]
|
||||
c_hashes = [t.get("content_sha256") for t in candidate.get("trials", []) if t.get("ok")]
|
||||
return {
|
||||
"same_request_payload_hash": baseline.get("request_payload_sha256") == candidate.get("request_payload_sha256"),
|
||||
"same_ordered_output_hashes": b_hashes == c_hashes,
|
||||
"same_output_hash_set": set(b_hashes) == set(c_hashes),
|
||||
"baseline_output_hashes": b_hashes,
|
||||
"candidate_output_hashes": c_hashes,
|
||||
"baseline_system_fingerprints": baseline.get("summary", {}).get("unique_system_fingerprints", []),
|
||||
"candidate_system_fingerprints": candidate.get("summary", {}).get("unique_system_fingerprints", []),
|
||||
"same_system_fingerprint_set": set(baseline.get("summary", {}).get("unique_system_fingerprints", []))
|
||||
== set(candidate.get("summary", {}).get("unique_system_fingerprints", [])),
|
||||
}
|
||||
|
||||
|
||||
def render_summary(bundle: Dict[str, Any]) -> None:
|
||||
summary = bundle["summary"]
|
||||
col1, col2, col3, col4 = st.columns(4)
|
||||
col1.metric("Trials", summary["trial_count"])
|
||||
col2.metric("Successful", summary["successful_trial_count"])
|
||||
col3.metric("Unique outputs", len(summary["unique_content_sha256"]))
|
||||
col4.metric("Unique fingerprints", len(summary["unique_system_fingerprints"]))
|
||||
|
||||
if summary["all_successful"] and summary["all_outputs_identical"] and summary["all_system_fingerprints_identical"]:
|
||||
st.success("Within this run, the outputs were exactly identical and the system fingerprint was unchanged.")
|
||||
elif summary["all_successful"] and summary["all_outputs_identical"]:
|
||||
st.warning("Outputs were identical, but the system fingerprint varied or was unavailable.")
|
||||
elif summary["all_successful"]:
|
||||
st.warning("Outputs differed. Inspect the fingerprints, finish reasons, token counts, and diff below.")
|
||||
else:
|
||||
st.error("At least one trial failed. Inspect the error rows below.")
|
||||
|
||||
rows = []
|
||||
for trial in bundle["trials"]:
|
||||
usage = trial.get("usage") or {}
|
||||
rows.append(
|
||||
{
|
||||
"trial": trial.get("trial_index"),
|
||||
"ok": trial.get("ok"),
|
||||
"content_sha256": trial.get("content_sha256", "")[:16],
|
||||
"system_fingerprint": trial.get("system_fingerprint"),
|
||||
"response_model": trial.get("response_model"),
|
||||
"finish_reason": trial.get("finish_reason"),
|
||||
"elapsed_ms": trial.get("elapsed_ms"),
|
||||
"prompt_tokens": usage.get("prompt_tokens"),
|
||||
"completion_tokens": usage.get("completion_tokens"),
|
||||
"total_tokens": usage.get("total_tokens"),
|
||||
"error": trial.get("error"),
|
||||
}
|
||||
)
|
||||
st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)
|
||||
|
||||
successful = [t for t in bundle["trials"] if t.get("ok")]
|
||||
if successful and not summary["all_outputs_identical"]:
|
||||
first = successful[0]
|
||||
different = next((t for t in successful[1:] if t["content_sha256"] != first["content_sha256"]), None)
|
||||
if different:
|
||||
st.subheader("First output diff")
|
||||
diff = difflib.unified_diff(
|
||||
first["content"].splitlines(),
|
||||
different["content"].splitlines(),
|
||||
fromfile=f"trial_{first['trial_index']}",
|
||||
tofile=f"trial_{different['trial_index']}",
|
||||
lineterm="",
|
||||
n=3,
|
||||
)
|
||||
st.code("\n".join(diff)[:12000], language="diff")
|
||||
|
||||
st.subheader("Outputs")
|
||||
for trial in bundle["trials"]:
|
||||
label = f"Trial {trial.get('trial_index')}"
|
||||
if trial.get("ok"):
|
||||
label += f" | {trial.get('content_sha256', '')[:16]} | {trial.get('system_fingerprint')}"
|
||||
with st.expander(label, expanded=False):
|
||||
st.write(trial.get("content", ""))
|
||||
else:
|
||||
label += " | failed"
|
||||
with st.expander(label, expanded=True):
|
||||
st.code(trial.get("error", ""))
|
||||
|
||||
|
||||
st.set_page_config(page_title="LLM Reproducibility Harness", layout="wide")
|
||||
st.title("LLM Reproducibility Harness")
|
||||
st.caption("Local UI for testing repeated Chat Completions with fixed request parameters, seed, and recorded backend metadata.")
|
||||
|
||||
with st.sidebar:
|
||||
st.header("Connection")
|
||||
env_key_present = bool(os.environ.get("OPENAI_API_KEY"))
|
||||
api_key_input = st.text_input(
|
||||
"OpenAI API key",
|
||||
type="password",
|
||||
placeholder="Leave blank to use OPENAI_API_KEY" if env_key_present else "sk-...",
|
||||
help="The key is used only for API calls and is not written to run bundles.",
|
||||
)
|
||||
api_key = api_key_input.strip() or os.environ.get("OPENAI_API_KEY", "")
|
||||
if env_key_present and not api_key_input:
|
||||
st.caption("Using OPENAI_API_KEY from environment or .env.")
|
||||
|
||||
st.header("Run controls")
|
||||
repetitions = st.slider("Repetitions", min_value=1, max_value=12, value=3, help="Use at least 2 to demonstrate exact repeatability.")
|
||||
delay_s = st.number_input("Delay between calls, seconds", min_value=0.0, max_value=10.0, value=0.0, step=0.25)
|
||||
|
||||
st.header("Saved baselines")
|
||||
saved = list_saved_runs()
|
||||
saved_names = [""] + [p.name for p in saved]
|
||||
selected_baseline_name = st.selectbox("Baseline bundle", saved_names, help="Compare a saved run with the latest run in this session.")
|
||||
|
||||
with st.expander("Interpretation", expanded=True):
|
||||
st.markdown(
|
||||
"""
|
||||
This harness tests *operational reproducibility*, not mathematical determinism. A strong run has:
|
||||
|
||||
1. identical request payload hash,
|
||||
2. identical output hashes,
|
||||
3. identical `system_fingerprint`,
|
||||
4. identical model snapshot or model identifier,
|
||||
5. no streaming or tools.
|
||||
|
||||
For stronger evidence, use a dated model snapshot rather than a rolling alias, keep `max_completion_tokens` modest, set `temperature` to `0`, set a fixed integer `seed`, and repeat the same saved bundle later.
|
||||
"""
|
||||
)
|
||||
|
||||
left, right = st.columns([2, 1])
|
||||
|
||||
with left:
|
||||
st.subheader("Prompt and context")
|
||||
system_prompt = st.text_area(
|
||||
"System prompt",
|
||||
value="You are a precise assistant. Answer in exactly three bullet points.",
|
||||
height=100,
|
||||
)
|
||||
context = st.text_area(
|
||||
"Optional fixed context",
|
||||
value="",
|
||||
height=80,
|
||||
help="Paste any external context here. It becomes part of the request payload and hash.",
|
||||
)
|
||||
user_prompt = st.text_area(
|
||||
"User prompt",
|
||||
value="Explain why reproducibility matters in LLM experiments.",
|
||||
height=120,
|
||||
)
|
||||
|
||||
with right:
|
||||
st.subheader("Model parameters")
|
||||
model = st.text_input(
|
||||
"Model",
|
||||
value="gpt-4.1-mini-2025-04-14",
|
||||
help="Prefer a dated snapshot model over a rolling alias for reproducibility checks.",
|
||||
)
|
||||
max_completion_tokens = st.number_input("max_completion_tokens", min_value=1, max_value=4096, value=200, step=10)
|
||||
|
||||
include_seed = st.checkbox("Send seed", value=True)
|
||||
seed = st.number_input("seed", min_value=0, max_value=2_147_483_647, value=42, step=1, disabled=not include_seed)
|
||||
|
||||
include_temperature = st.checkbox("Send temperature", value=True)
|
||||
temperature = st.number_input("temperature", min_value=0.0, max_value=2.0, value=0.0, step=0.05, disabled=not include_temperature)
|
||||
|
||||
include_top_p = st.checkbox("Send top_p", value=True)
|
||||
top_p = st.number_input("top_p", min_value=0.0, max_value=1.0, value=1.0, step=0.05, disabled=not include_top_p)
|
||||
|
||||
frequency_penalty = st.number_input("frequency_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
|
||||
presence_penalty = st.number_input("presence_penalty", min_value=-2.0, max_value=2.0, value=0.0, step=0.1)
|
||||
|
||||
stop_raw = st.text_area("Stop sequences, one per line", value="", height=70)
|
||||
include_logprobs = st.checkbox("Request logprobs", value=False, help="Useful for inspection, but not all models support it.")
|
||||
top_logprobs = st.number_input("top_logprobs", min_value=0, max_value=20, value=0, step=1, disabled=not include_logprobs)
|
||||
|
||||
payload = build_payload(
|
||||
model=model.strip(),
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
context=context,
|
||||
include_seed=include_seed,
|
||||
seed=int(seed),
|
||||
include_temperature=include_temperature,
|
||||
temperature=float(temperature),
|
||||
include_top_p=include_top_p,
|
||||
top_p=float(top_p),
|
||||
max_completion_tokens=int(max_completion_tokens),
|
||||
frequency_penalty=float(frequency_penalty),
|
||||
presence_penalty=float(presence_penalty),
|
||||
stop_sequences=parse_stop_sequences(stop_raw),
|
||||
include_logprobs=include_logprobs,
|
||||
top_logprobs=int(top_logprobs),
|
||||
)
|
||||
|
||||
request_hash = sha256_json(payload)
|
||||
|
||||
st.subheader("Canonical request")
|
||||
st.code(pretty_json(payload), language="json")
|
||||
st.caption(f"Request payload SHA-256: `{request_hash}`")
|
||||
|
||||
run_clicked = st.button("Run reproducibility test", type="primary", disabled=not bool(api_key))
|
||||
if not api_key:
|
||||
st.info("Provide an API key in the sidebar or set OPENAI_API_KEY in your environment.")
|
||||
|
||||
if run_clicked:
|
||||
client = OpenAI(api_key=api_key)
|
||||
trials: List[Dict[str, Any]] = []
|
||||
progress = st.progress(0, text="Starting calls")
|
||||
|
||||
for i in range(1, repetitions + 1):
|
||||
progress.progress((i - 1) / repetitions, text=f"Running trial {i} of {repetitions}")
|
||||
try:
|
||||
trial = call_chat_completion(client, payload, i)
|
||||
except Exception as exc:
|
||||
trial = {
|
||||
"trial_index": i,
|
||||
"ok": False,
|
||||
"started_at_utc": utc_now_iso(),
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
}
|
||||
trials.append(trial)
|
||||
if delay_s and i < repetitions:
|
||||
time.sleep(float(delay_s))
|
||||
|
||||
progress.progress(1.0, text="Complete")
|
||||
bundle = make_bundle(payload, trials)
|
||||
path = save_bundle(bundle)
|
||||
st.session_state["latest_bundle"] = bundle
|
||||
st.session_state["latest_bundle_path"] = str(path)
|
||||
|
||||
if "latest_bundle" in st.session_state:
|
||||
st.divider()
|
||||
st.header("Latest run")
|
||||
latest_bundle = st.session_state["latest_bundle"]
|
||||
latest_path = st.session_state.get("latest_bundle_path")
|
||||
st.caption(f"Saved locally as `{latest_path}`")
|
||||
render_summary(latest_bundle)
|
||||
|
||||
bundle_json = pretty_json(latest_bundle)
|
||||
st.download_button(
|
||||
"Download latest run bundle JSON",
|
||||
data=bundle_json,
|
||||
file_name=Path(latest_path).name if latest_path else "llm_repro_run.json",
|
||||
mime="application/json",
|
||||
)
|
||||
|
||||
if selected_baseline_name:
|
||||
baseline_path = DATA_DIR / selected_baseline_name
|
||||
if baseline_path.exists():
|
||||
baseline = load_bundle(baseline_path)
|
||||
comparison = compare_bundles(baseline, latest_bundle)
|
||||
st.subheader("Baseline comparison")
|
||||
c1, c2, c3 = st.columns(3)
|
||||
c1.metric("Same request hash", "yes" if comparison["same_request_payload_hash"] else "no")
|
||||
c2.metric("Same ordered outputs", "yes" if comparison["same_ordered_output_hashes"] else "no")
|
||||
c3.metric("Same fingerprint set", "yes" if comparison["same_system_fingerprint_set"] else "no")
|
||||
st.code(pretty_json(comparison), language="json")
|
||||
else:
|
||||
st.divider()
|
||||
st.header("No run yet")
|
||||
st.write("Configure the prompt and parameters, then start a reproducibility test.")
|
||||
Reference in New Issue
Block a user