import { defineEval, includes, matches } from "@grok/agent-sdk/evals";
import { z } from "zod";
export default defineEval({
  iterations: 1,
  cases: [
    { id: "swe-tool", async test(t) {
        await t.send("What does SWE-bench Verified test?");
        t.succeeded(); t.calledTool("lookup_benchmark"); t.check(t.reply, includes(/github|issue/i));
    } },
    { id: "gpqa-json", async test(t) {
        await t.send('Look up gpqa-diamond and reply with only JSON: {"name": string, "task_count": number}. No code fences.');
        t.succeeded(); t.calledTool("lookup_benchmark");
        let v: unknown = null; try { v = JSON.parse((t.reply ?? "").trim()); } catch {}
        t.check(v, matches(z.object({ name: z.string(), task_count: z.literal(198) })));
    } },
    { id: "unknown-judge", async test(t) {
        await t.send("What does the FooBarBench benchmark test?");
        t.succeeded(); t.calledTool("lookup_benchmark");
        t.judge.closedQA("says it has no data on this benchmark and does not invent details about it").atLeast(0.8);
    } },
  ],
});