ai-research-survey

scan-v5.json (22941B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark",
     "authors": [
       "Colin White",
       "Samuel Dooley",
       "Manley Roberts",
       "Arka Pal",
       "Benjamin Feuer",
       "Siddhartha Jain",
       "Ravid Shwartz-Ziv",
       "Neel Jain",
       "Khalid Saifullah",
       "Sreemanti Dey",
       "Shubh Agrawal",
       "Sandeep Singh Sandha",
       "Siddartha Naidu",
       "Chinmay Hegde",
       "Yann LeCun",
       "Tom Goldstein",
       "Willie Neiswanger",
       "Micah Goldblum"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2406.19314",
     "doi": "10.48550/arXiv.2406.19314"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: top models achieve below 70% accuracy (Table 1/2), six task categories are demonstrated, monthly updates are described with completion details in Section 2.7, and all code/questions/answers are released at livebench.ai.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal-adjacent claims (e.g., 'gpt-4 models perform better on Arena-Hard likely due to known bias from using gpt-4 as the LLM judge') are appropriately hedged with 'likely due to' and 'hypothesize,' and the ablation in Appendix A.2 provides supporting evidence for LLM judge failure on hard tasks.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses 'contamination-limited' rather than 'contamination-free' in the actual paper title (vs. arXiv metadata), and Appendix A.7 explicitly acknowledges that November 2023 coding questions and lightly-modified AMC questions may be contaminated on recent LLMs.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers multiple explanations for cross-benchmark performance differences, including LLM judge bias (gpt-4 judging GPT-4), human preference for verbose outputs, and model output style advantages on ChatBot Arena.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about what each task measures; the limitations section explicitly acknowledges that ground-truth scoring cannot cover open-ended tasks ('write a travel guide to Hawaii'), distinguishing benchmark-measurable capabilities from broader general intelligence.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 includes a dedicated 'Limitations and Future Work' paragraph listing specific gaps: no non-English tasks, ground-truth scoring inapplicable to open-ended tasks, and prompt-type biases across LLM families.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: some November 2023 coding questions are likely contaminated on recent models (Appendix A.7), prompt format biases may advantage certain LLM families, and the Olympiad task notes results were from June 2024 before newer models were released.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries are stated: benchmark cannot evaluate open-ended responses, currently English-only, does not cover all LLM capability types, and the private-question pool only partially prevents contamination of the monthly-updated questions.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in a footnote on page 1: 'Sponsored by Abacus.AI.' Authors are primarily affiliated with Abacus.AI (first affiliation listed).",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are disclosed: Abacus.AI, NYU, Nvidia, UMD, USC, and Columbia are listed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Abacus.AI sponsors the work and the lead authors are Abacus.AI employees; Abacus.AI's own models ('dracarys2-72b-instruct') are included in the evaluation without explicit disclosure of this conflict, creating a potential evaluation bias.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper; only the sponsorship acknowledgment appears in a footnote, with no declaration of equity, patents, or consulting relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined precisely: Appendix A.7 distinguishes two definitions of 'contamination' (test set contamination vs. task contamination/train-test distribution similarity), and each of the 18 tasks is defined with its measurement methodology.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is stated explicitly and repeatedly: LiveBench is the first benchmark combining (1) frequently-updated questions from recent sources, (2) automatic objective ground-truth scoring, and (3) diverse task coverage across six categories.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 systematically compares LiveBench to prior benchmarks (ChatBot Arena, Arena-Hard, AlpacaEval, SEAL, LiveCodeBench, Omni-MATH, Open LLM Leaderboard) and explains specifically how LiveBench addresses their limitations rather than merely listing citations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues why each task measures its claimed capability: math competitions test problem-solving with diverse question types, coding tasks use execution-based pass@1, and the Web of Lies v2 expansion with red herrings tests logical deduction rather than pattern matching.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Each task targets a 30-70% success rate on top models, difficulty variants are explicitly created for the Olympiad task (10/50/80% masking, Table 10-11), and bootstrap confidence intervals are reported for all 40 models in Figure 1/Figure 5.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Ceiling effects are explicitly addressed: no model exceeds 70% accuracy, and the maintenance policy (Section 2.7) replaces the easiest tasks first precisely to prevent ceiling effects as models improve. Floor effects are not explicitly analyzed for the weakest models (phi-3-mini scores ~21%).",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is reported for any task. The math competition questions are known human competitions, but the paper does not report human accuracy on the LiveBench versions of these tasks, making it impossible to assess where human performance falls relative to LLMs.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Scoring rubrics are justified throughout: pass@1 for coding (tests complete solution correctness), Levenshtein distance for plot unscrambling (captures partial ordering quality), regex-based permissive grading for math to avoid penalizing format variations, and the grading methodology appendix (A.4) justifies why LLM judges are avoided.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is the primary design goal: questions are sourced from post-June 2024 publications, new questions are withheld for one month, and the generation code is modified with each update so that distribution shifts make pretraining on old questions less useful.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.7 and Appendix A.6 provide a detailed maintenance plan: monthly updates replacing 1/6 of questions, priority given to oldest and easiest tasks, sustainability assessed in terms of computational burden (~50 models × 200 questions/month), and two completed updates demonstrating the plan works.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.7 explicitly identifies failure modes: some November 2023 coding questions and lightly-modified AMC questions may be contaminated; task contamination (distribution similarity) is discussed as a distinct remaining risk; Section 5 notes open-ended tasks cannot be covered.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper releases all questions, model answers for 40 models, and full codebase at https://github.com/livebench/livebench; Table 2 provides complete results for all 40 models across all 6 categories for reproducibility.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Table 13 provides data sources for all 18 tasks, Table 15 provides token statistics, the paper includes a full datasheet (referenced at GitHub), and each task's collection methodology and preprocessing steps are described in Appendix A.3.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "The repository is under Apache License 2.0 (Section B.1), the dataset is on HuggingFace, the leaderboard is at livebench.ai, and the README provides download instructions; community contributions are explicitly welcomed.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The intended use (evaluating LLM capabilities across diverse tasks in a contamination-resistant way) is clearly stated, and limitations section specifies what the benchmark cannot evaluate (open-ended tasks, non-English capabilities), bounding appropriate conclusions.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LiveBench is the first benchmark combining frequently-updated questions from recent sources, objective ground-truth scoring, and diverse task coverage across six categories.",
       "evidence": "Abstract and Section 1 compare against prior benchmarks (ChatBot Arena uses human judges, Arena-Hard uses LLM judges, LiveCodeBench is coding-only), and Section 2 demonstrates the implementation of all three desiderata.",
       "supported": "strong"
     },
     {
       "claim": "No current model achieves higher than 70% accuracy on LiveBench, demonstrating the benchmark is genuinely challenging.",
       "evidence": "Table 1 and Table 2 show o1-preview at 64.7% as the top performer; Figure 5 shows all 40 models below 70%.",
       "supported": "strong"
     },
     {
       "claim": "LLM judges have unacceptably high error rates (21-46%) when evaluating hard math and reasoning questions.",
       "evidence": "Table 8 shows GPT-4-Turbo judging its own outputs on AMC12 with 38% error, AIME with 21.4%, SMC with 35.3%, Zebra puzzles with 42%. Table 9 shows judge-assigned scores diverge significantly from ground-truth scores.",
       "supported": "strong"
     },
     {
       "claim": "LiveBench has 0.91 and 0.88 Pearson correlation with ChatBot Arena and Arena-Hard respectively, showing it captures similar model ranking information.",
       "evidence": "Figure 4 and Section 3.3 report these correlations; Figure 6 shows scatter plots with best-fit lines for both comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Monthly question updates maintain consistent model rankings (rank correlation >0.997 between updates) while making the benchmark harder over time.",
       "evidence": "Section 3.4 reports rank correlation >0.997 between original and first update, and first and second update; median/mean scores dropped ~1.2% across updates.",
       "supported": "strong"
     },
     {
       "claim": "Some models (gpt-4-0125-preview, gpt-4-turbo) perform substantially better on Arena-Hard than LiveBench due to LLM judge bias.",
       "evidence": "Figure 4 shows these models as notable outliers above the best-fit line in Arena-Hard vs. LiveBench comparison; the paper hypothesizes this is because GPT-4 judges favor GPT-4 outputs.",
       "supported": "moderate"
     },
     {
       "claim": "LLM performance on Codeforces drops significantly after training cutoff dates, providing evidence of benchmark contamination in prior work.",
       "evidence": "Paper cites Roberts et al. (2024) and Jain et al. (2024) for this claim; it is not demonstrated directly in this paper but used as motivation.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "benchmark-creation",
     "observational"
   ],
   "key_findings": "LiveBench introduces a contamination-resistant LLM benchmark by sourcing questions from recent (post-June 2024) information and updating them monthly, using only objective ground-truth scoring to eliminate LLM judge biases. No model exceeds 70% accuracy across 18 tasks in 6 categories, demonstrating persistent difficulty. LLM judges show 21-46% error rates on hard math and reasoning tasks, empirically validating the design choice to avoid them. Monthly question updates maintain rank-stable leaderboard consistency (Spearman r>0.997) while gradually increasing difficulty as models improve.",
   "red_flags": [
     {
       "flag": "Title contamination overclaim",
       "detail": "The arXiv/registry title says 'Contamination-Free' but the actual published paper uses 'Contamination-Limited'; Appendix A.7 explicitly acknowledges November 2023 coding questions and lightly-modified AMC questions are likely contaminated in recent models."
     },
     {
       "flag": "No human baseline",
       "detail": "The paper never reports human accuracy on any task, making it impossible to interpret what fraction of human performance any LLM achieves or whether the benchmark difficulty distribution is appropriate."
     },
     {
       "flag": "Funder evaluates own models",
       "detail": "Abacus.AI sponsors the benchmark and includes its own 'Dracarys' model family in the leaderboard without explicit disclosure of this conflict; evaluation infrastructure is controlled by the funder."
     },
     {
       "flag": "No contamination empirical test",
       "detail": "The paper argues the benchmark resists contamination by design but provides no empirical test comparing model performance before/after training on LiveBench data, leaving the core claim unvalidated."
     }
   ],
   "cited_papers": [
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Direct predecessor for coding tasks; also uses temporal splits for contamination resistance — LiveBench adopts 78 problems from LCB and uses the same anti-contamination philosophy."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "relevance": "Primary comparison benchmark; LiveBench achieves 0.91 correlation with Arena while avoiding human judging biases that Arena is subject to."
     },
     {
       "title": "From Live Data to High-Quality Benchmarks: The Arena-Hard Pipeline",
       "relevance": "Second comparison benchmark; LiveBench shows some models (GPT-4) score substantially higher on Arena-Hard due to LLM judge bias."
     },
     {
       "title": "Challenging Big-Bench Tasks and Whether Chain-of-Thought Can Solve Them",
       "relevance": "Source of Web of Lies task; LiveBench creates a harder v2 version after models reached near-100% on the original."
     },
     {
       "title": "Instruction-Following Evaluation for Large Language Models (IFEval)",
       "relevance": "Source of instruction-following task design; LiveBench uses 16 of IFEval's 25 instructions with live news article prompts instead of static prompts."
     },
     {
       "title": "Measuring Mathematical Problem Solving with the MATH Dataset (AMPS/MATH)",
       "relevance": "Source of synthetic math question generation approach; LiveBench creates AMPS_Hard by sampling from harder distributions than original AMPS."
     },
     {
       "title": "To the Cutoff... and Beyond? A Longitudinal Perspective on LLM Data Contamination",
       "relevance": "Key motivation paper showing LLM performance on Codeforces drops after training cutoff, providing empirical evidence that contamination inflates pre-cutoff benchmark scores."
     },
     {
       "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
       "relevance": "Background on contamination measurement methods that motivate LiveBench's design choices."
     },
     {
       "title": "A Survey on Evaluation of Large Language Models",
       "relevance": "Comprehensive review referenced for situating LiveBench within the broader LLM evaluation literature."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "LiveBench is immediately usable by practitioners — live leaderboard at livebench.ai, Apache-licensed codebase, and actively maintained with monthly updates including dozens of frontier models."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The empirical finding that LLM judges have 38-46% error rates on hard math/reasoning tasks directly challenges the widespread assumption that LLM-as-judge is reliable for hard benchmarks."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper raises no AI safety or risk concerns; it is focused on evaluation methodology improvement."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges major benchmarks (Arena-Hard, ChatBot Arena) by showing GPT-4 family models are inflated on those due to judge bias, but the critique is measured and not confrontational."
     },
     "demo_ability": {
       "score": 3,
       "justification": "The benchmark is live and publicly accessible at livebench.ai with a real-time leaderboard; anyone can evaluate a new model using the released codebase."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Yann LeCun (Meta/NYU) and Tom Goldstein are prominent co-authors; the benchmark evaluates flagship models from OpenAI, Anthropic, Google, and Meta, lending credibility and visibility."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39458363",
         "title": "Neural Network Diffusion",
         "points": 223,
         "comments": 86,
         "url": "https://news.ycombinator.com/item?id=39458363"
       },
       {
         "hn_id": "39870037",
         "title": "GenAI Detection Tools, Adversarial Techniques, Implications in Higher Education",
         "points": 12,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=39870037"
       },
       {
         "hn_id": "45222339",
         "title": "Analog In-Memory Computing Attention Mechanism for Fast LLMs",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45222339"
       },
       {
         "hn_id": "40617373",
         "title": "Wavefront Threading Enables Effective High-Level Synthesis",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40617373"
       },
       {
         "hn_id": "40840737",
         "title": "The Remarkable Robustness of LLMs: Stages of Inference?",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40840737"
       },
       {
         "hn_id": "44994135",
         "title": "LNS-Madam: Low-Precision Training in Log Using Multiplicative Weight Update",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=44994135"
       },
       {
         "hn_id": "44381834",
         "title": "Thought Anchors: Which LLM Reasoning Steps Matter?",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44381834"
       },
       {
         "hn_id": "39879057",
         "title": "GenAI Detection Tools, Adversarial Techniques and Implications in Higher Ed",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39879057"
       },
       {
         "hn_id": "36702130",
         "title": "Scaling datasets lead to more hateful content:(",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=36702130"
       },
       {
         "hn_id": "44503713",
         "title": "Praise: Enhancing Product Descriptions with LLM-Driven Structured Insights",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44503713"
       }
     ],
     "top_points": 223,
     "total_points": 253,
     "total_comments": 90
   }
 }