{
  "test": "human-votes-vs-physics-scores",
  "pre_registration": "tests/human-votes-vs-physics-scores/PREREGISTRATION.md",
  "pre_reg_commit": "eb28592375be5b963efedd46028740073748025f",
  "retrieved": {
    "aa-t2v": {
      "requested_url": "https://artificialanalysis.ai/video/leaderboard/text-to-video",
      "retrieved_at_ist": "2026-11-14T18:23:50,170909612+05:30",
      "retrieved_at_utc": "2026-11-14T12:53:50,172220479+00:00",
      "http_code": "200",
      "size_download": "151685",
      "url_effective": "https://artificialanalysis.ai/video/leaderboard/text-to-video",
      "content_type": "text/html; charset=utf-8"
    },
    "physics-iq-readme": {
      "requested_url": "https://raw.githubusercontent.com/google-deepmind/physics-IQ-benchmark/main/README.md",
      "retrieved_at_ist": "2026-11-14T18:23:32,410864445+05:30",
      "retrieved_at_utc": "2026-11-14T12:53:32,412112970+00:00",
      "http_code": "200",
      "size_download": "11380",
      "url_effective": "https://raw.githubusercontent.com/google-deepmind/physics-IQ-benchmark/main/README.md",
      "redirect_url": "",
      "content_type": "text/plain; charset=utf-8"
    },
    "videophy-2": {
      "requested_url": "https://videophy2.github.io/",
      "retrieved_at_ist": "2026-11-14T18:23:36,818583335+05:30",
      "retrieved_at_utc": "2026-11-14T12:53:36,819630523+00:00",
      "http_code": "200",
      "size_download": "7134",
      "url_effective": "https://videophy2.github.io/",
      "redirect_url": "",
      "content_type": "text/html; charset=utf-8"
    }
  },
  "claim": "We compared published leaderboards on the snapshot date. We did not run any video model and did not re-score any video in this test.",
  "counts": {
    "aa-video-t2v-v2.1": 34,
    "physics-iq-verified": 53,
    "physics-iq-original": 32,
    "videophy-2": 7
  },
  "comparisons": [
    {
      "human_board": "aa-video-t2v-v2.1",
      "physics_board": "physics-iq-verified",
      "n_human": 34,
      "n_physics": 53,
      "n_matched": 0,
      "matches": [],
      "n_unmatched_physics": 53,
      "spearman_rho": null,
      "kendall_tau_b": null,
      "note": "Fewer than 4 pre-registered matches, so no correlation was computed.",
      "unmatched_human_count": 34,
      "unmatched_human_with_same_family": [
        {
          "human_model": "Wan 3.0",
          "human_rank": 1,
          "close_physics_rows": [
            {
              "physics_model": "Wan 2.2 14B",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 32.2,
              "physics_rank": 45
            },
            {
              "physics_model": "Wan 2.2 5B",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 27.7,
              "physics_rank": 51
            }
          ]
        },
        {
          "human_model": "MiniMax H3 (768p)",
          "human_rank": 3,
          "close_physics_rows": [
            {
              "physics_model": "MiniMax H3 (FL2VA)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 39.8,
              "physics_rank": 36
            },
            {
              "physics_model": "MiniMax H3 Max",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 36.2,
              "physics_rank": 38
            }
          ]
        },
        {
          "human_model": "Gemini Omni Flash 1.2",
          "human_rank": 5,
          "close_physics_rows": [
            {
              "physics_model": "Gemini Omni 1.1 Flash",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 35.34,
              "physics_rank": 39
            },
            {
              "physics_model": "Gemini Omni Flash Preview",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 33.36,
              "physics_rank": 43
            }
          ]
        },
        {
          "human_model": "FLUX 3 Max",
          "human_rank": 6,
          "close_physics_rows": [
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts) + WMReward + consensus (BoN)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 64.35,
              "physics_rank": 4
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 61.11,
              "physics_rank": 8
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts) + WMReward + consensus (BoN)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 54.7,
              "physics_rank": 13
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 51.11,
              "physics_rank": 18
            }
          ]
        },
        {
          "human_model": "MiniMax H3 Max",
          "human_rank": 8,
          "close_physics_rows": [
            {
              "physics_model": "MiniMax H3 (FL2VA)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 39.8,
              "physics_rank": 36
            },
            {
              "physics_model": "MiniMax H3 Max",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 36.2,
              "physics_rank": 38
            }
          ]
        },
        {
          "human_model": "FLUX 3",
          "human_rank": 9,
          "close_physics_rows": [
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts) + WMReward + consensus (BoN)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 64.35,
              "physics_rank": 4
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 61.11,
              "physics_rank": 8
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts) + WMReward + consensus (BoN)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 54.7,
              "physics_rank": 13
            },
            {
              "physics_model": "[FLUX 3 [large]](https://bfl.ai/models/flux-3-video) (Claude Opus 5.5 prompts)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 51.11,
              "physics_rank": 18
            }
          ]
        },
        {
          "human_model": "P-Video-2-Pro",
          "human_rank": 12,
          "close_physics_rows": [
            {
              "physics_model": "P-Video",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 25.3,
              "physics_rank": 53
            }
          ]
        },
        {
          "human_model": "Gemini Omni Flash 1.1",
          "human_rank": 13,
          "close_physics_rows": [
            {
              "physics_model": "Gemini Omni 1.1 Flash",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 35.34,
              "physics_rank": 39
            },
            {
              "physics_model": "Gemini Omni Flash Preview",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 33.36,
              "physics_rank": 43
            }
          ]
        },
        {
          "human_model": "Grok Imagine Video 1.5",
          "human_rank": 16,
          "close_physics_rows": [
            {
              "physics_model": "Grok Imagine Video",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 34.8,
              "physics_rank": 40
            }
          ]
        },
        {
          "human_model": "MAGI-2 Preview",
          "human_rank": 18,
          "close_physics_rows": [
            {
              "physics_model": "Magi-1 24B + GeoPhys (BoN) (op)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 58.2,
              "physics_rank": 10
            },
            {
              "physics_model": "Magi-1 24B (op)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 48.4,
              "physics_rank": 25
            },
            {
              "physics_model": "Magi-1 24B + GeoPhys (BoN) (op)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 33.7,
              "physics_rank": 41
            },
            {
              "physics_model": "Magi-1 24B (op)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 30.2,
              "physics_rank": 49
            }
          ]
        },
        {
          "human_model": "Grok Imagine Video 1.5 Lite",
          "human_rank": 23,
          "close_physics_rows": [
            {
              "physics_model": "Grok Imagine Video",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 34.8,
              "physics_rank": 40
            }
          ]
        },
        {
          "human_model": "Veo 3.1",
          "human_rank": 25,
          "close_physics_rows": [
            {
              "physics_model": "Veo 3.1 Lite",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 31.83,
              "physics_rank": 46
            },
            {
              "physics_model": "Veo 3.1 Fast",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 29.96,
              "physics_rank": 50
            }
          ]
        },
        {
          "human_model": "Veo 3.1 Fast",
          "human_rank": 26,
          "close_physics_rows": [
            {
              "physics_model": "Veo 3.1 Lite",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 31.83,
              "physics_rank": 46
            },
            {
              "physics_model": "Veo 3.1 Fast",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 29.96,
              "physics_rank": 50
            }
          ]
        },
        {
          "human_model": "Veo 3.1 Lite",
          "human_rank": 27,
          "close_physics_rows": [
            {
              "physics_model": "Veo 3.1 Lite",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 31.83,
              "physics_rank": 46
            },
            {
              "physics_model": "Veo 3.1 Fast",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 29.96,
              "physics_rank": 50
            }
          ]
        }
      ]
    },
    {
      "human_board": "aa-video-t2v-v2.1",
      "physics_board": "physics-iq-original",
      "n_human": 34,
      "n_physics": 32,
      "n_matched": 0,
      "matches": [],
      "n_unmatched_physics": 32,
      "spearman_rho": null,
      "kendall_tau_b": null,
      "note": "Fewer than 4 pre-registered matches, so no correlation was computed.",
      "unmatched_human_count": 34,
      "unmatched_human_with_same_family": [
        {
          "human_model": "Pika H3 Flex",
          "human_rank": 14,
          "close_physics_rows": [
            {
              "physics_model": "Pika",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 13.0,
              "physics_rank": 31
            }
          ]
        },
        {
          "human_model": "MAGI-2 Preview",
          "human_rank": 18,
          "close_physics_rows": [
            {
              "physics_model": "Magi-1 + GeoPhys (BoN)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 64.5,
              "physics_rank": 1
            },
            {
              "physics_model": "Magi-1 + WMReward (BoN)",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 62.6,
              "physics_rank": 3
            },
            {
              "physics_model": "Magi-1",
              "physics_input": "multiframe (v2v)",
              "reason": "input-type mismatch: human='t2v' physics='multiframe (v2v)'",
              "physics_score": 56.0,
              "physics_rank": 6
            },
            {
              "physics_model": "Magi-1 + GeoPhys (BoN)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 38.6,
              "physics_rank": 15
            },
            {
              "physics_model": "Magi-1 + WMReward (BoN)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 36.9,
              "physics_rank": 17
            },
            {
              "physics_model": "Magi-1 4.5B + GeoPhys (BoN)",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 34.0,
              "physics_rank": 22
            },
            {
              "physics_model": "Magi-1",
              "physics_input": "i2v",
              "reason": "input-type mismatch: human='t2v' physics='i2v'",
              "physics_score": 30.2,
              "physics_rank": 24
            }
          ]
        }
      ]
    },
    {
      "human_board": "aa-video-t2v-v2.1",
      "physics_board": "videophy-2",
      "n_human": 34,
      "n_physics": 7,
      "n_matched": 0,
      "matches": [],
      "n_unmatched_physics": 7,
      "spearman_rho": null,
      "kendall_tau_b": null,
      "note": "Fewer than 4 pre-registered matches, so no correlation was computed.",
      "unmatched_human_count": 34,
      "unmatched_human_with_same_family": []
    }
  ],
  "blockers": {
    "lmarena_video": {
      "status": "unreadable",
      "detail": "HTTP 403 Cloudflare 'Just a moment...' challenge on arena.ai/leaderboard/video, /text-to-video, canary.arena.ai, the API path, and a curl User-Agent retry. No ranks taken from this board.",
      "meta": {
        "lmarena-video": {
          "requested_url": "https://arena.ai/leaderboard/video",
          "retrieved_at_ist": "2026-11-14T18:23:31,907087967+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:31,908153056+00:00",
          "http_code": "403",
          "size_download": "3334",
          "url_effective": "https://arena.ai/leaderboard/video",
          "redirect_url": "",
          "content_type": "text/html; charset=UTF-8"
        },
        "lmarena-t2v": {
          "requested_url": "https://arena.ai/leaderboard/text-to-video",
          "retrieved_at_ist": "2026-11-14T18:23:32,000402044+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:32,001883148+00:00",
          "http_code": "403",
          "size_download": "3360",
          "url_effective": "https://arena.ai/leaderboard/text-to-video",
          "redirect_url": "",
          "content_type": "text/html; charset=UTF-8"
        },
        "lmarena-t2v-canary": {
          "requested_url": "https://canary.arena.ai/leaderboard/text-to-video",
          "retrieved_at_ist": "2026-11-14T18:23:32,083632965+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:32,084847360+00:00",
          "http_code": "403",
          "size_download": "3380",
          "url_effective": "https://canary.arena.ai/leaderboard/text-to-video",
          "redirect_url": "",
          "content_type": "text/html; charset=UTF-8"
        },
        "lmarena-api": {
          "requested_url": "https://arena.ai/api/leaderboard/video",
          "retrieved_at_ist": "2026-11-14T18:23:32,174315097+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:32,175791075+00:00",
          "http_code": "403",
          "size_download": "3340",
          "url_effective": "https://arena.ai/api/leaderboard/video",
          "redirect_url": "",
          "content_type": "text/html; charset=UTF-8"
        },
        "lmarena-video-curlUA": {
          "requested_url": "https://arena.ai/leaderboard/video",
          "retrieved_at_ist": "2026-11-14T18:23:32,255458505+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:32,256753678+00:00",
          "http_code": "403",
          "size_download": "3183",
          "url_effective": "https://arena.ai/leaderboard/video",
          "redirect_url": "",
          "content_type": "text/html; charset=UTF-8"
        }
      }
    },
    "vbench-2.0": {
      "status": "no public numeric table",
      "detail": "The official project page has no numeric leaderboard table. The official Hugging Face Space Vchitect/VBench_Leaderboard is public, but its VBench-2.0 results CSV in dataset Vchitect/vbench_leaderboard_submission returned HTTP 401 without a token. We did not invent totals or use a third-party copy.",
      "meta": {
        "vbench2-project": {
          "requested_url": "https://vchitect.github.io/VBench-2.0-project/",
          "retrieved_at_ist": "2026-11-14T18:23:35,777831603+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:35,779312951+00:00",
          "http_code": "200",
          "size_download": "6157",
          "url_effective": "https://vchitect.github.io/VBench-2.0-project/",
          "redirect_url": "",
          "content_type": "text/html; charset=utf-8"
        },
        "vbench-leaderboard-space": {
          "requested_url": "https://huggingface.co/spaces/Vchitect/VBench_Leaderboard",
          "retrieved_at_ist": "2026-11-14T18:23:36,115092996+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:36,116397205+00:00",
          "http_code": "200",
          "size_download": "14681",
          "url_effective": "https://huggingface.co/spaces/Vchitect/VBench_Leaderboard",
          "redirect_url": "",
          "content_type": "text/html; charset=utf-8"
        },
        "vbench2-results-csv": {
          "requested_url": "https://huggingface.co/datasets/Vchitect/vbench_leaderboard_submission/resolve/main/results2_0.csv",
          "retrieved_at_ist": "2026-11-14T18:23:36,499267063+05:30",
          "retrieved_at_utc": "2026-11-14T12:53:36,500820139+00:00",
          "http_code": "401",
          "size_download": "29",
          "url_effective": "https://huggingface.co/datasets/Vchitect/vbench_leaderboard_submission/resolve/main/results2_0.csv",
          "redirect_url": "",
          "content_type": "text/plain; charset=utf-8"
        }
      }
    }
  },
  "written_at_ist": "2026-11-14T18:25:47.621607+05:30"
}
