ai-research-survey

scan.json (33298B)

---

 {
   "paper": {
     "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark",
     "authors": [
       "Colin White",
       "Samuel Dooley",
       "Manley Roberts",
       "Arka Pal",
       "Benjamin Feuer",
       "Siddhartha Jain",
       "Ravid Shwartz-Ziv",
       "Neel Jain",
       "Khalid Saifullah",
       "Sreemanti Dey",
       "Shubh-Agrawal",
       "Sandeep Singh Sandha",
       "Siddartha Naidu",
       "Chinmay Hegde",
       "Yann LeCun",
       "Tom Goldstein",
       "Willie Neiswanger",
       "Micah Goldblum"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2406.19314",
     "doi": "10.48550/arXiv.2406.19314"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository at https://github.com/livebench/livebench and states in Section 6: 'we open-source the leaderboard, all questions, all code to run API and open-source models, all model outputs for 40 models, and all code to score the models.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All questions, model answers, and the leaderboard are publicly released. Section 6 states 'every part of the project is available publicly: https://livebench.ai/' and the benchmark is available on HuggingFace at https://huggingface.co/livebench. The only exception is 1/6 of questions withheld for one month."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using FastChat for chat templates and bfloat16 for open-source models, but does not provide requirements.txt, Dockerfile, or detailed dependency specifications. Section 3 mentions setup details but lacks reproducible environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Reproducibility Statement) states 'The readme in the above link gives instructions to download all parts of the project and to score new models.' The paper also describes experimental setup in detail (Section 3)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Figure 5 display 95% bootstrap confidence intervals for all models. However, the main results tables (Table 1, Table 2) report only point estimates without CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No significance tests are used for model comparisons. The paper makes claims like 'o1-preview-2024-09-12 substantially outperforms all other models' based on comparing point estimates without any formal statistical tests. Correlation analyses report Pearson coefficients with standard errors but no p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage differences with baseline context (e.g., 'o1-preview 6% better than all other models'), per-category scores, and Pearson correlation coefficients (0.91 and 0.88 with ChatBot Arena and Arena-Hard). Tables 5 and 6 report correlation coefficients with standard errors."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for why 40-100 questions per task or 1000 total questions is sufficient. Section 2 states the tasks 'loosely aim for an overall 30-70% success rate on the top models' but provides no power analysis or formal justification for these choices."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use temperature 0 for deterministic single-turn evaluation. No multiple runs, no seed sensitivity analysis. The bootstrap confidence intervals in Figure 1 capture question-sampling variance, not experimental run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 40 models including proprietary models (GPT-4, Claude, Gemini) and open-source models (Llama, Qwen, Mixtral, etc.). Additionally, Section 3.3 compares LiveBench against ChatBot Arena and Arena-Hard benchmarks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include state-of-the-art models at the time of publication: o1-preview, claude-3-5-sonnet, gemini-exp-1121, gpt-4o. Table 4 lists 80+ model versions across multiple evaluation rounds."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not ablate components of the benchmark design (e.g., removing task types, varying question difficulty distributions, comparing scoring methods). Correlation analyses in Section 3.2 show relationships between categories but do not constitute ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Different tasks use different appropriate metrics: pass@1 for coding, accuracy for multiple-choice, F1 for table join, Levenshtein distance-based score for plot unscrambling, edit distance for olympiad, and a two-component instruction-following score (prompt-level + instruction-level accuracy)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper's design explicitly avoids human evaluation: 'scores answers automatically according to objective ground-truth values, without the use of LLM judges' (Section 1). While manual inspection of scoring functions is mentioned (Appendix A.4), no human evaluation of model outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are evaluated zero-shot with no tuning on benchmark data. Additionally, 1/6 of questions are kept private each month. Section A.6 describes that 'the public leaderboard always has 1/6 questions that are private and completely contamination-free.' Model hyperparameters are taken from model cards, not tuned on LiveBench."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide per-category breakdowns (Coding, Data Analysis, Instruction Following, Language, Math, Reasoning) for all 40 models. Tables 5 and 6 provide per-category and per-task correlation analyses."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 shows each model's relative worst task. Section 3.2 and Figure 3 identify outlier models. Section 3.3 and Appendix A.2 discuss LLM judging failures. The paper discusses specific weaknesses (e.g., phi models strong in reasoning but weak in language)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 reports that LLM judges have high error rates (38-46%) on hard math and reasoning questions. Section 3.4 reports the benchmark is becoming harder over time (1.2% drop). Section A.7 acknowledges some questions may be contaminated."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: (1) frequently-updated questions from recent sources — demonstrated in Sections 2.1-2.6 and A.6; (2) automatic objective scoring — Section 2 and Appendix A.4; (3) diverse tasks — 18 tasks across 6 categories; (4) 'top models achieving below 70% accuracy' — Table 1 shows o1-preview at 64.7%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal-adjacent claims are appropriately hedged. 'Likely due to the known bias from using gpt-4 itself as the LLM judge' (Section 3.3) is hedged with 'likely.' 'We hypothesize that the strong performance of some models... may be due to having an output style that is preferred by humans' uses 'hypothesize' and 'may be.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to the specific tasks and models tested. Section 5 explicitly states limitations: no non-English tasks, ground-truth scoring cannot handle open-ended tasks, prompt formatting biases. The 'first benchmark' claim is qualified to require all three desiderata simultaneously."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses alternative explanations for benchmark score differences (LLM judge bias, output style preferences). Section A.7 distinguishes between test set contamination and task contamination. Section 3.2 discusses why certain categories correlate."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific task accuracies and reports them as such (pass@1, accuracy, F1). It does not overclaim beyond the measured metrics. Section 5 acknowledges that ground-truth scoring 'cannot be used for certain use cases, such as write a travel guide to Hawaii in which it is hard to define a ground truth.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "All 40+ models are identified with specific version strings throughout (e.g., 'gpt-4o-2024-08-06', 'claude-3-5-sonnet-20240620', 'o1-preview-2024-09-12'). Table 4 provides full model names with citations and version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts with actual text are provided for multiple tasks throughout the paper: Math Competitions (Section 2.1, Appendix A.3.1), Zebra Puzzles (Section 2.3), Typos (Section 2.6), Web of Lies (Appendix A.3.3), Connections (Appendix A.3.6), Spatial (Appendix A.3.3). Full code including prompts is released."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'we perform single-turn evaluation with temperature 0, unless otherwise noted in the model card. All models run with their respective templates from our updated version of FastChat. We run all open-source models with bfloat16.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a direct single-turn evaluation benchmark where models receive a question and produce an answer."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Each task's question generation and data processing pipeline is described in detail in Sections 2.1-2.6 and Appendix A.3. For example, math competition questions describe specific modifications made; coding completion describes how solutions are truncated; typos task describes how misspellings are injected."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Conclusions, Limitations, and Future Work' includes a dedicated paragraph discussing limitations: lack of non-English tasks, inability to handle open-ended tasks with ground-truth scoring, biases from prompt formatting across model families."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: 'there are still biases due to certain LLM families favoring certain prompt types.' Section A.7 discusses specific contamination risks for certain tasks. Appendix A.4 addresses the concern that automated grading adds an instruction-following component to non-IF tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states specific things the benchmark does NOT cover: 'we hope to add non-English language tasks,' 'ground truth scoring... cannot be used for certain use cases.' Section A.7 explicitly states what LiveBench does and does not guard against regarding contamination types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 6: 'we open-source the leaderboard, all questions, all code to run API and open-source models, all model outputs for 40 models, and all code to score the models.' All raw model outputs and questions are available at livebench.ai and HuggingFace."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each task's data collection is documented: math competitions from AMC12/AIME/SMC/USAMO/IMO with specific dates; coding from LiveCodeBench and LeetCode; instruction following from The Guardian API; language from ArXiv/NYT/IMDb; data analysis from Kaggle/Socrata. Sections 2.1-2.6 and Appendix A.3 provide details."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are publicly available competitions, news articles, datasets, and paper abstracts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from data sources to questions to evaluation to scoring is documented across Sections 2 and Appendix A.3-A.5. Table 13 summarizes number of questions and data sources per task. Table 15 provides token statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Sponsored by Abacus.AI' in the author footnote on page 1. Author affiliations are listed as '1 Abacus.AI, 2 NYU, 3 Nvidia, 4 UMD, 5 USC, 6 Columbia.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Abacus.AI, NYU, Nvidia, UMD, USC, Columbia. Table 4 identifies Dracarys models as from Abacus.AI (https://huggingface.co/abacusai/). The connection between sponsor and evaluated models is traceable."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The paper is sponsored by Abacus.AI, whose models (Dracarys2-72B-Instruct, Dracarys-72B-Instruct, Dracarys-Llama-3.1-70B-Instruct) are evaluated in the benchmark. Abacus.AI has a financial interest in their models performing well on the benchmark they sponsor."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No formal competing interests or conflict of interest statement is present in the paper. The Abacus.AI sponsorship is noted, but there is no declaration of whether authors hold equity, patents, or other financial interests related to the evaluated products."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses contamination extensively and designs around training cutoffs conceptually, it does not state the specific training data cutoff dates for each of the 40 evaluated models. Section 1 mentions 'after the training cutoff date of the LLM' generically."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 extensively discusses train/test overlap. Section A.7 provides a detailed analysis of two types of contamination (test set contamination vs. task contamination). The paper acknowledges that 'a limited fraction of LiveBench is likely contaminated on all recent LLMs' for some coding questions from November 2023."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination mitigation is the paper's core design principle: frequently-updated questions from recent sources, 1/6 of questions kept private each month, questions become harder over time. Section A.7 explicitly discusses which questions may be contaminated and the paper's mitigation strategies."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLMs on automated benchmark tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates LLMs on benchmark tasks with synthetic and publicly available data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 16 provides per-model API costs for running LiveBench (e.g., o1-preview ≈$47.87, claude-3-haiku ≈$0.90). Table 15 provides mean input/output token counts per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-model API costs are provided (Table 16) but the total computational budget is not stated. No GPU hours reported for running open-source models. Section 2.7 mentions costs are 'easily within the computational budgets of the authors' institutions' without quantifying."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is performed. All models are evaluated at temperature 0 (deterministic), producing a single output per question. No analysis of how results vary across seeds or sampling parameters."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'we perform single-turn evaluation with temperature 0' — this effectively specifies one deterministic run per question per model."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that model hyperparameters are taken from model cards and example code (Appendix A.5): 'we match this setup in LiveBench's code.' No hyperparameter search is performed, and this is clearly documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "No configuration selection is performed — default model configurations from model cards are used consistently (Appendix A.5). This eliminates selection bias."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not perform formal statistical hypothesis tests requiring multiple comparison correction. Model rankings are based on point estimates and correlation analyses, not significance tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is designed and sponsored by Abacus.AI, whose Dracarys models are evaluated (ranking 14th/40 at 50.1%). There is no discussion of potential bias from the benchmark authors evaluating models from their sponsoring organization."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not present performance as a function of compute budget or model size in a systematic way. While models range from 0.5B to 405B parameters, no compute-normalized comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 analyzes correlations between LiveBench categories and tasks to assess internal validity. Section 3.3 compares with ChatBot Arena and Arena-Hard to assess external validity. The paper discusses what ground-truth scoring can and cannot capture (Section 5)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated in direct single-turn mode with standardized inference via FastChat."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the paper's core concern. Questions are based on recently-released sources (math competitions, arXiv papers, news articles, datasets), and 1/6 are withheld privately each month. Section A.7 details the temporal anti-contamination strategy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation prompts or task formatting could leak answer information. For example, multiple-choice questions with modified answer orders could still provide structural hints. No analysis of prompt-level feature leakage is provided."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.7 distinguishes between test set contamination (data duplication) and task contamination (distribution similarity), noting that 'the data still ends up being highly similar to the kind of data likely seen in the pretraining set.' The paper acknowledges this as a separate concern from exact data overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses temporal splits as a leakage prevention method: questions from recently-released sources after model training cutoffs, plus 1/6 private questions at any time. Section A.7 describes their approach. They also reference Roberts et al. (2024) on contamination detection methods."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "LiveBench is the first benchmark combining frequently-updated questions, objective ground-truth scoring, and diverse challenging tasks.",
       "evidence": "The benchmark design is described in Section 2 with 18 tasks across 6 categories, all using objective scoring. LiveCodeBench is acknowledged as related but limited to coding only (Section 4).",
       "supported": "strong"
     },
     {
       "claim": "o1-preview-2024-09-12 achieves the highest overall LiveBench score at 64.7%, 6 percentage points ahead of all other models.",
       "evidence": "Table 1 shows o1-preview at 64.7%, with the next best models (claude-3-5-sonnet variants) at 58.5% and 58.2%.",
       "supported": "strong"
     },
     {
       "claim": "LiveBench has 0.91 and 0.88 Pearson correlation with ChatBot Arena and Arena-Hard respectively.",
       "evidence": "Section 3.3 reports these correlations based on models common to both benchmarks. Figure 4 and Figure 6 visualize the comparisons.",
       "supported": "strong"
     },
     {
       "claim": "LLM judges have high error rates (21-46%) on challenging math and reasoning questions.",
       "evidence": "Table 8 shows GPT-4-Turbo judging error rates: 38% on AMC12, 21% on AIME, 35% on SMC, 42-46% on Zebra Puzzles. However, the study is described as 'preliminary' and uses only one judge model (Appendix A.2).",
       "supported": "moderate"
     },
     {
       "claim": "Model rankings remain consistent across monthly benchmark updates with >0.997 rank correlation.",
       "evidence": "Section 3.4 states 'The rank correlation between the original and first update, and the first and second update, are both > 0.997.'",
       "supported": "strong"
     },
     {
       "claim": "The best open-source models (Llama-3.1-405b, Qwen2.5-72b) outperform GPT-4-Turbo.",
       "evidence": "Table 2 shows meta-llama-3.1-405b at 51.1% and qwen2.5-coder-32b at 45.0%, vs gpt-4-turbo at 49.6%. However, qwen2.5-72b-instruct is not in the displayed table (likely evaluated in another version).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LiveBench introduces a contamination-resistant LLM benchmark with 18 tasks across 6 categories (math, coding, reasoning, language, instruction following, data analysis) using objective ground-truth scoring rather than LLM or human judges. Among 40 evaluated models, o1-preview achieves the highest score (64.7%) while no model exceeds 70%. The benchmark correlates well with ChatBot Arena (r=0.91) and Arena-Hard (r=0.88) but reveals systematic differences attributable to judging biases. LLM judges are shown to have 21-46% error rates on hard math and reasoning tasks, supporting the case for ground-truth scoring.",
   "red_flags": [
     {
       "flag": "Sponsor conflict of interest",
       "detail": "The paper is sponsored by Abacus.AI, whose models (Dracarys2-72B-Instruct) are evaluated in the benchmark, ranking 14th out of 40 models at 50.1%. This conflict is not explicitly acknowledged despite the evaluated models bearing the sponsor's name."
     },
     {
       "flag": "No formal significance tests",
       "detail": "Model comparisons across 40 models are based entirely on point estimates. Claims like 'substantially outperforms' lack statistical tests. Bootstrap CIs are shown in figures but not used for formal pairwise comparisons."
     },
     {
       "flag": "Single deterministic run only",
       "detail": "All evaluations use temperature 0 with a single pass. While this ensures reproducibility, it means no analysis of how results might vary with different sampling parameters, and the bootstrap CIs capture only question-sampling variance."
     },
     {
       "flag": "Acknowledged partial contamination",
       "detail": "Section A.7 acknowledges that 'a limited fraction of LiveBench is likely contaminated on all recent LLMs' — specifically some coding questions from November 2023 and minimally-modified AMC questions. This undermines the contamination-free framing."
     }
   ],
   "cited_papers": [
     {
       "title": "To the cutoff... and beyond? A longitudinal perspective on LLM data contamination",
       "authors": ["Manley Roberts", "Himanshu Thakur", "Christine Herlihy", "Colin White", "Samuel Dooley"],
       "year": 2024,
       "relevance": "Key study on LLM data contamination showing performance drops after training cutoffs, directly motivating LiveBench's design."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Related contamination-free benchmark for coding tasks; LiveBench uses its coding questions and extends the approach to multiple domains."
     },
     {
       "title": "Chatbot Arena: An open platform for evaluating LLMs by human preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Major human-judged LLM benchmark that LiveBench compares against, finding r=0.91 correlation but also identifying bias differences."
     },
     {
       "title": "From live data to high-quality benchmarks: The Arena-Hard pipeline",
       "authors": ["Tianle Li", "Wei-Lin Chiang", "Evan Frick"],
       "year": 2024,
       "relevance": "LLM-judge benchmark that LiveBench compares against (r=0.88), with discussion of GPT-4 self-preference bias in judging."
     },
     {
       "title": "Challenging Big-Bench tasks and whether chain-of-thought can solve them",
       "authors": ["Mirac Suzgun", "Nathan Scales"],
       "year": 2023,
       "arxiv_id": "2210.09261",
       "relevance": "Big-Bench Hard benchmark from which LiveBench derives and creates harder versions of the Web of Lies task."
     },
     {
       "title": "Instruction-following evaluation for large language models",
       "authors": ["Jeffrey Zhou", "Tianjian Lu", "Swaroop Mishra"],
       "year": 2023,
       "arxiv_id": "2311.07911",
       "relevance": "IFEval benchmark that inspired LiveBench's instruction following tasks with verifiable constraints."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Study on data contamination in LLM evaluation, documenting how benchmark leakage inflates performance."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Demonstrates that several models have overfit to GSM8K benchmark, supporting the need for contamination-resistant benchmarks."
     },
     {
       "title": "Functional benchmarks for robust evaluation of reasoning performance, and the reasoning gap",
       "authors": ["Saurabh Srivastava"],
       "year": 2024,
       "arxiv_id": "2402.19450",
       "relevance": "Modifies the MATH dataset by changing problem numbers to test LLM generalization, finding performance declines."
     },
     {
       "title": "Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?",
       "authors": ["Aaditya K Singh"],
       "year": 2024,
       "arxiv_id": "2411.03923",
       "relevance": "Comprehensive study on measuring and understanding the impact of evaluation data contamination in LLMs."
     },
     {
       "title": "Proving test set contamination in black box language models",
       "authors": ["Yonatan Oren", "Nicole Meister"],
       "year": 2023,
       "arxiv_id": "2310.17623",
       "relevance": "Methods for detecting test set contamination in black-box LLMs, relevant to benchmark integrity assessment."
     },
     {
       "title": "Training on the test model: Contamination in ranking distillation",
       "authors": ["Vishakha Suresh Kalal", "Andrew Parry"],
       "year": 2024,
       "arxiv_id": "2411.02284",
       "relevance": "Demonstrates contamination effects in ranking model distillation, expanding the scope of contamination concerns."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Immediately usable benchmark with open-source code, public leaderboard at livebench.ai, and monthly updates — practitioners can evaluate their models today."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the widely-known problem of benchmark contamination rather than challenging conventional wisdom; the LLM judge error rate findings are modestly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper is about evaluation methodology."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit criticism of ChatBot Arena and Arena-Hard for judging biases, and suggests some model rankings may be inflated by contamination, but framed constructively."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Live leaderboard at livebench.ai, open-source code on GitHub, and HuggingFace dataset — anyone can run it on their own models."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Author list includes Yann LeCun (NYU/Meta) and Tom Goldstein (UMD), published at ICLR 2025, and evaluates all major model families."
     }
   }
 }