{
 "models": [
  {
   "model": "Llama 3.2 3B",
   "ollama_tag": "llama3.2:3b",
   "items": 22,
   "calls": 66,
   "majority_agree": 21,
   "majority_ci95": [
    78.2,
    99.2
   ],
   "per_call_agree": 63,
   "false_pass": 0,
   "fail_calls": 36,
   "false_fail": 3,
   "pass_calls": 30,
   "unparsed": 0,
   "stable_items": 22,
   "median_s": 3.2,
   "p90_s": 4.6,
   "per_item": [
    {
     "item_id": "q01-good",
     "gold": "PASS",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": false,
     "stable": true
    },
    {
     "item_id": "q01-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q01-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q02-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q02-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q08-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q08-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q10-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q10-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    }
   ]
  },
  {
   "model": "Qwen2.5 3B",
   "ollama_tag": "qwen2.5:3b",
   "items": 22,
   "calls": 66,
   "majority_agree": 21,
   "majority_ci95": [
    78.2,
    99.2
   ],
   "per_call_agree": 61,
   "false_pass": 3,
   "fail_calls": 36,
   "false_fail": 2,
   "pass_calls": 30,
   "unparsed": 0,
   "stable_items": 20,
   "median_s": 3.5,
   "p90_s": 5.6,
   "per_item": [
    {
     "item_id": "q01-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q01-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q01-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q02-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "FAIL"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": false
    },
    {
     "item_id": "q02-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-bad",
     "gold": "FAIL",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": false,
     "stable": true
    },
    {
     "item_id": "q07-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q08-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q08-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q10-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "FAIL",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": false
    },
    {
     "item_id": "q10-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    }
   ]
  },
  {
   "model": "Gemma 3 4B",
   "ollama_tag": "gemma3-4b-local:latest",
   "items": 22,
   "calls": 66,
   "majority_agree": 20,
   "majority_ci95": [
    72.2,
    97.5
   ],
   "per_call_agree": 60,
   "false_pass": 6,
   "fail_calls": 36,
   "false_fail": 0,
   "pass_calls": 30,
   "unparsed": 0,
   "stable_items": 22,
   "median_s": 4.0,
   "p90_s": 4.8,
   "per_item": [
    {
     "item_id": "q01-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q01-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q01-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q02-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q02-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q05-bad",
     "gold": "FAIL",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": false,
     "stable": true
    },
    {
     "item_id": "q07-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-mid",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q07-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q08-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q03-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q04-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q06-bad",
     "gold": "FAIL",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": false,
     "stable": true
    },
    {
     "item_id": "q08-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q09-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q10-good",
     "gold": "PASS",
     "verdicts": [
      "PASS",
      "PASS",
      "PASS"
     ],
     "majority": "PASS",
     "majority_agrees": true,
     "stable": true
    },
    {
     "item_id": "q10-bad",
     "gold": "FAIL",
     "verdicts": [
      "FAIL",
      "FAIL",
      "FAIL"
     ],
     "majority": "FAIL",
     "majority_agrees": true,
     "stable": true
    }
   ]
  }
 ]
}