{
 "model": "llama3.2:3b",
 "groups": [
  {
   "method": "deepeval-geval",
   "rubric": "v1",
   "temp": "0",
   "items": 12,
   "repeats_per_item": 5,
   "calls": 60,
   "exact_agreement_pct": 58.3,
   "mean_sd": 0.43,
   "max_sd": 1.79,
   "verdict_flip_pct": 16.7,
   "verdict_flip_items": 2,
   "verdict_flip_by_tier": {
    "bad": 1,
    "mid": 1
   },
   "label_agreement_pct": 95.0,
   "mean_score_by_tier": {
    "good": 6.85,
    "mid": 3.05,
    "bad": 2.1
   },
   "unparsed_or_errors": 0,
   "mean_secs": 42.03
  },
  {
   "method": "deepeval-geval",
   "rubric": "v1",
   "temp": "0.8",
   "items": 12,
   "repeats_per_item": 5,
   "calls": 60,
   "exact_agreement_pct": 8.3,
   "mean_sd": 1.37,
   "max_sd": 2.9,
   "verdict_flip_pct": 58.3,
   "verdict_flip_items": 7,
   "verdict_flip_by_tier": {
    "good": 3,
    "mid": 2,
    "bad": 2
   },
   "label_agreement_pct": 80.0,
   "mean_score_by_tier": {
    "good": 6.2,
    "mid": 2.85,
    "bad": 2.6
   },
   "unparsed_or_errors": 0,
   "mean_secs": 46.38
  },
  {
   "method": "promptfoo-llm-rubric",
   "rubric": "v1",
   "temp": "0",
   "items": 30,
   "repeats_per_item": 5,
   "calls": 150,
   "exact_agreement_pct": 66.7,
   "mean_sd": 0.99,
   "max_sd": 4.9,
   "verdict_flip_pct": 20.0,
   "verdict_flip_items": 6,
   "verdict_flip_by_tier": {
    "good": 4,
    "mid": 2
   },
   "label_agreement_pct": 63.0,
   "mean_score_by_tier": {
    "good": 2.89,
    "mid": 0.47,
    "bad": 0.5
   },
   "unparsed_or_errors": 1,
   "mean_secs": 11.07
  },
  {
   "method": "promptfoo-llm-rubric",
   "rubric": "v1",
   "temp": "0.8",
   "items": 30,
   "repeats_per_item": 5,
   "calls": 150,
   "exact_agreement_pct": 36.7,
   "mean_sd": 1.93,
   "max_sd": 4.9,
   "verdict_flip_pct": 36.7,
   "verdict_flip_items": 11,
   "verdict_flip_by_tier": {
    "good": 9,
    "bad": 1,
    "mid": 1
   },
   "label_agreement_pct": 70.0,
   "mean_score_by_tier": {
    "good": 4.48,
    "mid": 0.33,
    "bad": 0.73
   },
   "unparsed_or_errors": 0,
   "mean_secs": 13.09
  },
  {
   "method": "raw",
   "rubric": "v1",
   "temp": "0",
   "items": 30,
   "repeats_per_item": 5,
   "calls": 150,
   "exact_agreement_pct": 93.3,
   "mean_sd": 0.04,
   "max_sd": 0.8,
   "verdict_flip_pct": 3.3,
   "verdict_flip_items": 1,
   "verdict_flip_by_tier": {
    "mid": 1
   },
   "label_agreement_pct": 95.0,
   "mean_score_by_tier": {
    "good": 7.9,
    "mid": 4.76,
    "bad": 3.68
   },
   "unparsed_or_errors": 0,
   "mean_secs": 1.04
  },
  {
   "method": "raw",
   "rubric": "v1",
   "temp": "default",
   "items": 30,
   "repeats_per_item": 5,
   "calls": 150,
   "exact_agreement_pct": 13.3,
   "mean_sd": 0.68,
   "max_sd": 1.74,
   "verdict_flip_pct": 30.0,
   "verdict_flip_items": 9,
   "verdict_flip_by_tier": {
    "mid": 5,
    "bad": 4
   },
   "label_agreement_pct": 94.0,
   "mean_score_by_tier": {
    "good": 7.96,
    "mid": 4.72,
    "bad": 3.68
   },
   "unparsed_or_errors": 0,
   "mean_secs": 1.06
  },
  {
   "method": "raw",
   "rubric": "v2",
   "temp": "0",
   "items": 30,
   "repeats_per_item": 3,
   "calls": 90,
   "exact_agreement_pct": 96.7,
   "mean_sd": 0.02,
   "max_sd": 0.47,
   "verdict_flip_pct": 0.0,
   "verdict_flip_items": 0,
   "verdict_flip_by_tier": {},
   "label_agreement_pct": 95.0,
   "mean_score_by_tier": {
    "good": 7.3,
    "mid": 4.8,
    "bad": 3.43
   },
   "unparsed_or_errors": 0,
   "mean_secs": 0.92
  },
  {
   "method": "raw",
   "rubric": "v2",
   "temp": "default",
   "items": 30,
   "repeats_per_item": 3,
   "calls": 90,
   "exact_agreement_pct": 23.3,
   "mean_sd": 0.74,
   "max_sd": 1.89,
   "verdict_flip_pct": 23.3,
   "verdict_flip_items": 7,
   "verdict_flip_by_tier": {
    "good": 2,
    "bad": 2,
    "mid": 3
   },
   "label_agreement_pct": 88.3,
   "mean_score_by_tier": {
    "good": 7.53,
    "mid": 4.53,
    "bad": 3.6
   },
   "unparsed_or_errors": 0,
   "mean_secs": 1.23
  }
 ],
 "rewording": [
  {
   "temp": "0",
   "items": 30,
   "score_changed_pct": 23.3,
   "mean_abs_change": 0.37,
   "mean_change": -0.3,
   "verdict_changed_pct": 6.7,
   "verdict_changed_items": 2
  },
  {
   "temp": "default",
   "items": 30,
   "score_changed_pct": 53.3,
   "mean_abs_change": 0.8,
   "mean_change": -0.33,
   "verdict_changed_pct": 13.3,
   "verdict_changed_items": 4
  }
 ],
 "pairwise": [
  {
   "kind": "all",
   "temp": "0",
   "pair_runs": 30,
   "calls": 60,
   "position_flip_pct": 10.0,
   "position_flip_runs": 3,
   "picked_better_pct": 91.7,
   "picked_A_pct": 45.0,
   "unparsed": 0
  },
  {
   "kind": "all",
   "temp": "default",
   "pair_runs": 90,
   "calls": 180,
   "position_flip_pct": 15.6,
   "position_flip_runs": 14,
   "picked_better_pct": 88.9,
   "picked_A_pct": 45.6,
   "unparsed": 0
  },
  {
   "kind": "good-vs-bad",
   "temp": "0",
   "pair_runs": 10,
   "calls": 20,
   "position_flip_pct": 0.0,
   "position_flip_runs": 0,
   "picked_better_pct": 100.0,
   "picked_A_pct": 50.0,
   "unparsed": 0
  },
  {
   "kind": "good-vs-bad",
   "temp": "default",
   "pair_runs": 30,
   "calls": 60,
   "position_flip_pct": 0.0,
   "position_flip_runs": 0,
   "picked_better_pct": 100.0,
   "picked_A_pct": 50.0,
   "unparsed": 0
  },
  {
   "kind": "good-vs-mid",
   "temp": "0",
   "pair_runs": 10,
   "calls": 20,
   "position_flip_pct": 0.0,
   "position_flip_runs": 0,
   "picked_better_pct": 100.0,
   "picked_A_pct": 50.0,
   "unparsed": 0
  },
  {
   "kind": "good-vs-mid",
   "temp": "default",
   "pair_runs": 30,
   "calls": 60,
   "position_flip_pct": 0.0,
   "position_flip_runs": 0,
   "picked_better_pct": 100.0,
   "picked_A_pct": 50.0,
   "unparsed": 0
  },
  {
   "kind": "mid-vs-bad",
   "temp": "0",
   "pair_runs": 10,
   "calls": 20,
   "position_flip_pct": 30.0,
   "position_flip_runs": 3,
   "picked_better_pct": 75.0,
   "picked_A_pct": 35.0,
   "unparsed": 0
  },
  {
   "kind": "mid-vs-bad",
   "temp": "default",
   "pair_runs": 30,
   "calls": 60,
   "position_flip_pct": 46.7,
   "position_flip_runs": 14,
   "picked_better_pct": 66.7,
   "picked_A_pct": 36.7,
   "unparsed": 0
  }
 ],
 "per_item": {
  "deepeval-geval|v1|0": {
   "q01-good": [
    6.0,
    6.0,
    6.0,
    6.0,
    6.0
   ],
   "q01-mid": [
    4.0,
    4.0,
    4.0,
    4.0,
    4.0
   ],
   "q01-bad": [
    0.0,
    0.0,
    0.0,
    0.0,
    0.0
   ],
   "q04-good": [
    9.0,
    8.0,
    6.0,
    6.0,
    8.0
   ],
   "q04-mid": [
    2.0,
    2.0,
    2.0,
    2.0,
    2.0
   ],
   "q04-bad": [
    0.0,
    0.0,
    0.0,
    0.0,
    0.0
   ],
   "q07-good": [
    8.0,
    8.0,
    8.0,
    8.0,
    8.0
   ],
   "q07-mid": [
    0.0,
    4.0,
    0.0,
    4.0,
    2.0
   ],
   "q07-bad": [
    4.0,
    6.0,
    6.0,
    4.0,
    4.0
   ],
   "q10-good": [
    6.0,
    6.0,
    6.0,
    6.0,
    6.0
   ],
   "q10-bad": [
    4.0,
    4.0,
    4.0,
    2.0,
    4.0
   ],
   "q10-mid": [
    4.0,
    4.0,
    4.0,
    5.0,
    4.0
   ]
  },
  "deepeval-geval|v1|0.8": {
   "q01-good": [
    6.0,
    6.0,
    6.0,
    7.0,
    4.0
   ],
   "q01-mid": [
    4.0,
    6.0,
    2.0,
    4.0,
    5.0
   ],
   "q01-bad": [
    0.0,
    2.0,
    0.0,
    4.0,
    0.0
   ],
   "q04-good": [
    6.0,
    5.0,
    4.0,
    4.0,
    6.0
   ],
   "q04-mid": [
    4.0,
    0.0,
    0.0,
    4.0,
    0.0
   ],
   "q04-bad": [
    0.0,
    0.0,
    0.0,
    0.0,
    0.0
   ],
   "q07-good": [
    6.0,
    8.0,
    8.0,
    7.0,
    8.0
   ],
   "q07-mid": [
    0.0,
    4.0,
    2.0,
    0.0,
    0.0
   ],
   "q07-bad": [
    9.0,
    3.0,
    8.0,
    2.0,
    8.0
   ],
   "q10-good": [
    4.0,
    6.0,
    8.0,
    7.0,
    8.0
   ],
   "q10-mid": [
    4.0,
    4.0,
    6.0,
    4.0,
    4.0
   ],
   "q10-bad": [
    6.0,
    4.0,
    0.0,
    4.0,
    2.0
   ]
  },
  "promptfoo-llm-rubric|v1|0": {
   "q01-good": [
    3.3,
    0,
    0,
    3.3,
    3.3
   ],
   "q01-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q01-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q02-good": [
    0,
    10,
    0,
    0,
    0
   ],
   "q02-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q02-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q03-good": [
    0,
    8.0,
    0,
    0,
    0
   ],
   "q03-mid": [
    0,
    10,
    0,
    0,
    0
   ],
   "q03-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q04-good": [
    10,
    10,
    10,
    10,
    10
   ],
   "q04-mid": [
    0,
    0,
    6.67,
    0,
    0
   ],
   "q04-bad": [
    3.3,
    3.3,
    3.3,
    3.3,
    3.3
   ],
   "q05-good": [
    0,
    10,
    0,
    10,
    0
   ],
   "q05-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q05-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q06-good": [
    0,
    0,
    0,
    0,
    0
   ],
   "q06-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q06-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q07-good": [
    0,
    10,
    10,
    10,
    10
   ],
   "q07-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q07-bad": [
    5.0,
    0,
    0,
    0,
    3.3
   ],
   "q08-good": [
    0,
    0,
    0,
    0,
    0
   ],
   "q08-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q08-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q09-good": [
    0,
    0,
    0,
    0,
    0
   ],
   "q09-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q09-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q10-good": [
    0,
    3.3,
    3.3,
    0,
    0
   ],
   "q10-mid": [
    null,
    0,
    3.3,
    0,
    3.3
   ],
   "q10-bad": [
    0,
    0,
    0,
    0,
    0
   ]
  },
  "promptfoo-llm-rubric|v1|0.8": {
   "q01-good": [
    3.3,
    0,
    0,
    3.3,
    0
   ],
   "q01-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q01-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q02-good": [
    0,
    10,
    0,
    10,
    0
   ],
   "q02-mid": [
    0,
    0,
    3.3,
    0,
    0
   ],
   "q02-bad": [
    3.33,
    3.3,
    0,
    0,
    0
   ],
   "q03-good": [
    10,
    3.3,
    10,
    0,
    10
   ],
   "q03-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q03-bad": [
    0,
    0,
    3.3,
    0,
    0
   ],
   "q04-good": [
    0,
    10,
    10,
    10,
    10
   ],
   "q04-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q04-bad": [
    3.3,
    0,
    0,
    0,
    0
   ],
   "q05-good": [
    0,
    10,
    0,
    10,
    8.0
   ],
   "q05-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q05-bad": [
    0,
    0,
    0,
    0,
    3.3
   ],
   "q06-good": [
    10,
    0,
    0,
    0,
    0
   ],
   "q06-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q06-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q07-good": [
    0,
    10,
    3.3,
    10,
    0
   ],
   "q07-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q07-bad": [
    3.3,
    10,
    0,
    0,
    0
   ],
   "q08-good": [
    3.3,
    0,
    0,
    0,
    10
   ],
   "q08-mid": [
    0,
    0,
    0,
    0,
    0
   ],
   "q08-bad": [
    0,
    0,
    0,
    0,
    0
   ],
   "q09-good": [
    0,
    10,
    8.0,
    0,
    0
   ],
   "q09-mid": [
    0,
    0,
    0,
    3.3,
    0
   ],
   "q09-bad": [
    0,
    0,
    3.3,
    0,
    3.3
   ],
   "q10-good": [
    8.0,
    3.3,
    0,
    10,
    10
   ],
   "q10-mid": [
    0,
    10,
    0,
    0,
    0
   ],
   "q10-bad": [
    0,
    0,
    0,
    0,
    0
   ]
  },
  "raw|v1|0": {
   "q01-good": [
    6,
    6,
    6,
    6,
    6
   ],
   "q01-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q01-bad": [
    2,
    2,
    2,
    2,
    2
   ],
   "q02-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q02-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q02-bad": [
    3,
    3,
    3,
    3,
    3
   ],
   "q03-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q03-mid": [
    8,
    8,
    8,
    8,
    8
   ],
   "q03-bad": [
    4,
    4,
    4,
    4,
    4
   ],
   "q04-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q04-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q04-bad": [
    4,
    4,
    4,
    4,
    4
   ],
   "q05-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q05-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q05-bad": [
    6,
    6,
    6,
    6,
    6
   ],
   "q06-good": [
    9,
    9,
    9,
    9,
    9
   ],
   "q06-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q06-bad": [
    4,
    4,
    4,
    4,
    4
   ],
   "q07-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q07-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q07-bad": [
    4,
    4,
    4,
    4,
    4
   ],
   "q08-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q08-mid": [
    6,
    6,
    6,
    4,
    6
   ],
   "q08-bad": [
    3,
    3,
    3,
    3,
    3
   ],
   "q09-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q09-mid": [
    6,
    6,
    6,
    6,
    6
   ],
   "q09-bad": [
    3,
    3,
    3,
    3,
    3
   ],
   "q10-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q10-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q10-bad": [
    4,
    3,
    4,
    4,
    4
   ]
  },
  "raw|v1|default": {
   "q01-good": [
    6,
    8,
    6,
    8,
    8
   ],
   "q01-mid": [
    4,
    4,
    3,
    4,
    3
   ],
   "q01-bad": [
    2,
    2,
    2,
    2,
    2
   ],
   "q02-good": [
    9,
    6,
    9,
    6,
    8
   ],
   "q02-mid": [
    4,
    4,
    4,
    6,
    3
   ],
   "q02-bad": [
    2,
    4,
    2,
    3,
    3
   ],
   "q03-good": [
    8,
    9,
    9,
    9,
    8
   ],
   "q03-mid": [
    8,
    8,
    8,
    8,
    8
   ],
   "q03-bad": [
    4,
    4,
    4,
    6,
    5
   ],
   "q04-good": [
    9,
    9,
    8,
    9,
    9
   ],
   "q04-mid": [
    4,
    4,
    4,
    4,
    4
   ],
   "q04-bad": [
    4,
    4,
    4,
    3,
    2
   ],
   "q05-good": [
    8,
    8,
    8,
    8,
    8
   ],
   "q05-mid": [
    6,
    4,
    4,
    4,
    4
   ],
   "q05-bad": [
    4,
    4,
    6,
    6,
    4
   ],
   "q06-good": [
    8,
    8,
    9,
    9,
    9
   ],
   "q06-mid": [
    3,
    3,
    4,
    4,
    4
   ],
   "q06-bad": [
    4,
    8,
    4,
    4,
    7
   ],
   "q07-good": [
    8,
    8,
    9,
    9,
    9
   ],
   "q07-mid": [
    3,
    4,
    3,
    4,
    4
   ],
   "q07-bad": [
    2,
    4,
    6,
    2,
    4
   ],
   "q08-good": [
    7,
    7,
    8,
    6,
    8
   ],
   "q08-mid": [
    6,
    4,
    4,
    6,
    6
   ],
   "q08-bad": [
    4,
    3,
    3,
    3,
    4
   ],
   "q09-good": [
    8,
    7,
    8,
    8,
    8
   ],
   "q09-mid": [
    6,
    6,
    6,
    6,
    4
   ],
   "q09-bad": [
    4,
    4,
    3,
    3,
    4
   ],
   "q10-good": [
    7,
    8,
    7,
    7,
    7
   ],
   "q10-mid": [
    6,
    4,
    3,
    6,
    6
   ],
   "q10-bad": [
    3,
    3,
    2,
    4,
    4
   ]
  },
  "raw|v2|0": {
   "q01-good": [
    4,
    4,
    4
   ],
   "q01-mid": [
    4,
    4,
    4
   ],
   "q01-bad": [
    2,
    1,
    1
   ],
   "q02-good": [
    8,
    8,
    8
   ],
   "q02-mid": [
    4,
    4,
    4
   ],
   "q02-bad": [
    3,
    3,
    3
   ],
   "q03-good": [
    8,
    8,
    8
   ],
   "q03-mid": [
    8,
    8,
    8
   ],
   "q03-bad": [
    4,
    4,
    4
   ],
   "q04-good": [
    8,
    8,
    8
   ],
   "q04-mid": [
    4,
    4,
    4
   ],
   "q04-bad": [
    4,
    4,
    4
   ],
   "q05-good": [
    8,
    8,
    8
   ],
   "q05-mid": [
    4,
    4,
    4
   ],
   "q05-bad": [
    4,
    4,
    4
   ],
   "q06-good": [
    9,
    9,
    9
   ],
   "q06-mid": [
    4,
    4,
    4
   ],
   "q06-bad": [
    4,
    4,
    4
   ],
   "q07-good": [
    8,
    8,
    8
   ],
   "q07-mid": [
    4,
    4,
    4
   ],
   "q07-bad": [
    4,
    4,
    4
   ],
   "q08-good": [
    6,
    6,
    6
   ],
   "q08-mid": [
    6,
    6,
    6
   ],
   "q08-bad": [
    2,
    2,
    2
   ],
   "q09-good": [
    8,
    8,
    8
   ],
   "q09-mid": [
    6,
    6,
    6
   ],
   "q09-bad": [
    4,
    4,
    4
   ],
   "q10-good": [
    6,
    6,
    6
   ],
   "q10-mid": [
    4,
    4,
    4
   ],
   "q10-bad": [
    4,
    4,
    4
   ]
  },
  "raw|v2|default": {
   "q01-good": [
    7,
    4,
    4
   ],
   "q01-mid": [
    3,
    3,
    3
   ],
   "q01-bad": [
    1,
    4,
    1
   ],
   "q02-good": [
    8,
    9,
    9
   ],
   "q02-mid": [
    3,
    3,
    3
   ],
   "q02-bad": [
    3,
    2,
    2
   ],
   "q03-good": [
    8,
    8,
    8
   ],
   "q03-mid": [
    8,
    8,
    8
   ],
   "q03-bad": [
    4,
    4,
    4
   ],
   "q04-good": [
    8,
    8,
    8
   ],
   "q04-mid": [
    4,
    4,
    2
   ],
   "q04-bad": [
    4,
    4,
    4
   ],
   "q05-good": [
    7,
    8,
    7
   ],
   "q05-mid": [
    4,
    4,
    3
   ],
   "q05-bad": [
    4,
    8,
    6
   ],
   "q06-good": [
    8,
    9,
    9
   ],
   "q06-mid": [
    4,
    2,
    6
   ],
   "q06-bad": [
    8,
    4,
    6
   ],
   "q07-good": [
    8,
    9,
    9
   ],
   "q07-mid": [
    4,
    3,
    3
   ],
   "q07-bad": [
    3,
    2,
    4
   ],
   "q08-good": [
    6,
    6,
    8
   ],
   "q08-mid": [
    8,
    4,
    4
   ],
   "q08-bad": [
    3,
    1,
    2
   ],
   "q09-good": [
    9,
    8,
    8
   ],
   "q09-mid": [
    6,
    7,
    7
   ],
   "q09-bad": [
    1,
    4,
    4
   ],
   "q10-good": [
    6,
    4,
    8
   ],
   "q10-mid": [
    7,
    4,
    4
   ],
   "q10-bad": [
    4,
    4,
    3
   ]
  }
 }
}