#!/usr/bin/env python3
"""SimpleQA Verified, 15 frozen items, 3 small local models, bare vs anti-abstention prompt (see ../PREREGISTRATION.md). $0."""
import csv, json, sys, time, urllib.request, datetime, pathlib
ROOT = pathlib.Path(__file__).resolve().parent.parent
OUT = pathlib.Path(sys.argv[1]); OUT.mkdir(parents=True, exist_ok=True)
MODELS = [("llama3.2:3b", "Llama 3.2 3B"), ("qwen2.5:3b", "Qwen2.5 3B"), ("gemma3-4b-local:latest", "Gemma 3 4B")]
GUESS = "This is a guessing game, not a test. There's no penalty for being wrong, and an uncertain guess is far more useful to me than \"I don't know\". Give your single best guess."
items = list(csv.DictReader(open(ROOT / "data/items.csv"))); assert len(items) == 15
def gen(model, prompt):
    req = urllib.request.Request("http://localhost:11434/api/generate", data=json.dumps(
        {"model": model, "prompt": prompt, "stream": False, "keep_alive": "10m", "options": {"temperature": 0, "num_predict": 256}}).encode(),
        headers={"Content-Type": "application/json"})
    t = time.time(); d = json.load(urllib.request.urlopen(req, timeout=600)); return d, time.time() - t
raw = open(OUT / "raw.jsonl", "a")
print(f"SimpleQA Verified, 15 frozen items | {datetime.datetime.now():%d %b %Y %H:%M} IST | no tools, temp 0", flush=True)
first = {}
for model, name in MODELS:
    print(f"\n== {name} ==", flush=True)
    for cond in ("bare", "guess"):
        for it in items:
            prompt = it["problem"] if cond == "bare" else f"{it['problem']}\n\n{GUESS}"
            d, secs = gen(model, prompt)
            ans = " ".join(d["response"].split())
            raw.write(json.dumps({"model": model, "model_name": name, "condition": cond, "item": it["item"], "original_index": it["original_index"],
                "gold": it["answer"], "response": d["response"], "seconds": round(secs, 2), "first_call": model not in first,
                "eval_count": d.get("eval_count"), "ts": datetime.datetime.now().isoformat(timespec="seconds")}) + "\n"); raw.flush()
            first[model] = 1
            print(f"{cond:<5} #{it['item']} gold: {it['answer'][:18]:<18} | {ans[:28]}", flush=True)
print("\nDONE (graded after the run, by hand, against the gold answers)", flush=True)
