#!/usr/bin/env python3
"""Local judges vs reference labels (pre-registered in ../PREREGISTRATION.md). $0: local Ollama only."""
import csv, json, re, sys, time, urllib.request, datetime, pathlib
ROOT = pathlib.Path(__file__).resolve().parent.parent
OUT = pathlib.Path(sys.argv[1]); OUT.mkdir(parents=True, exist_ok=True)
MODELS = [("llama3.2:3b", "Llama 3.2 3B"), ("qwen2.5:3b", "Qwen2.5 3B"), ("gemma3-4b-local:latest", "Gemma 3 4B")]
RUBRIC = (ROOT / "data/rubric.txt").read_text().rstrip("\n")
items = {json.loads(l)["id"]: json.loads(l) for l in open(ROOT / "data/items.jsonl")}
labels = [r for r in csv.DictReader(open(ROOT / "data/labels.csv")) if r["ambiguous"] == "no"]
assert len(labels) == 22
def gen(model, prompt):
    req = urllib.request.Request("http://localhost:11434/api/generate", data=json.dumps(
        {"model": model, "prompt": prompt, "stream": False, "keep_alive": "10m", "options": {"num_predict": 200}}).encode(),
        headers={"Content-Type": "application/json"})
    t = time.time(); d = json.load(urllib.request.urlopen(req, timeout=600)); return d, time.time() - t
def parse(txt):
    v = set(m.upper() for m in re.findall(r"VERDICT:\s*(PASS|FAIL)", txt, re.I))
    return v.pop() if len(v) == 1 else "unparsed"
raw = open(OUT / "raw.jsonl", "a")
print(f"Local judges vs our labels | {datetime.datetime.now():%d %b %Y %H:%M} IST | 22 items x 3 runs", flush=True)
for model, name in MODELS:
    ok = n = 0; first = True
    print(f"\n== {name} ({model}) ==", flush=True)
    for run in (1, 2, 3):
        for lab in labels:
            it = items[lab["item_id"]]
            prompt = f"{RUBRIC}\n\nQuestion:\n{it['question']}\n\nAnswer to grade:\n{it['answer']}\n"
            d, secs = gen(model, prompt)
            v = parse(d["response"]); agree = v == lab["gold"]; ok += agree; n += 1
            raw.write(json.dumps({"model": model, "model_name": name, "run": run, "item_id": lab["item_id"], "gold": lab["gold"],
                "verdict": v, "agrees": agree, "seconds": round(secs, 2), "first_call": first, "response": d["response"],
                "eval_count": d.get("eval_count"), "prompt_eval_count": d.get("prompt_eval_count"),
                "ts": datetime.datetime.now().isoformat(timespec="seconds")}) + "\n"); raw.flush(); first = False
            print(f"r{run} {lab['item_id']:<9} gold {lab['gold']:<4} judge {v:<8} {'OK ' if agree else 'XX '} {secs:5.1f}s  agree {ok}/{n}", flush=True)
print("\nDONE", flush=True)
