ai-research-survey

scan.json (34503B)

---

 {
   "paper": {
     "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark",
     "authors": [
       "Colin White",
       "Samuel Dooley",
       "Manley Roberts",
       "Arka Pal",
       "Benjamin Feuer",
       "Siddhartha Jain",
       "Ravid Shwartz-Ziv",
       "Neel Jain",
       "Khalid Saifullah",
       "Sreemanti Dey",
       "Shubh-Agrawal",
       "Sandeep Singh Sandha",
       "Siddartha Naidu",
       "Chinmay Hegde",
       "Yann LeCun",
       "Tom Goldstein",
       "Willie Neiswanger",
       "Micah Goldblum"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2406.19314"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LiveBench is the first LLM benchmark combining frequently-updated questions from recent sources, objective ground-truth scoring without LLM judges, and diverse tasks across six categories. Top models achieve below 70% accuracy, with o1-preview leading at 64.7%. The benchmark correlates well with ChatBot Arena (r=0.91) and Arena-Hard (r=0.88), while revealing that LLM judges have high error rates (21-46%) on challenging math and reasoning questions. Monthly question updates maintain stable model rankings (rank correlation >0.997) while increasing difficulty over time.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our codebase is available at https://github.com/livebench/livebench' (Section 1) and 'we open-source the leaderboard, all questions, all code to run API and open-source models, all model outputs for 40 models, and all code to score the models' (Section 6)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All questions and model answers are released: 'We release all questions, code, and model answers' (Section 1). The benchmark is also available on HuggingFace at https://huggingface.co/livebench (Appendix B.2)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies key environment details: 'All models run with their respective templates from our updated version of FastChat' and 'We run all open-source models with bfloat16' (Section 3). All code is released on GitHub with a README containing setup instructions (Section 6)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'The readme in the above link gives instructions to download all parts of the project and to score new models.' The paper also describes experimental setup in detail (Section 3)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Figure 5 show '95% bootstrap confidence intervals' for all model scores. Standard errors are reported for correlation analyses in Tables 5 and 6."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models outperform others (e.g., 'o1-preview-2024-09-12 substantially outperforms all other models') based on comparing raw scores without any formal significance tests. Pearson correlations are reported with standard errors but no hypothesis tests are conducted."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute scores and percentage differences with baseline context throughout. For example, 'o1-preview-2024-09-12 performs the best overall, 6% better than all other models' (Section 3.1). Tables 1-2 provide full score breakdowns enabling comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the benchmark size of 1000 questions, the number of questions per task (40-100), or the number of models evaluated (40). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap confidence intervals are shown in Figures 1 and 5. Standard errors are reported for correlation coefficients in Tables 5 and 6. Table 15 reports mean and standard deviation for token counts."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares LiveBench against two prominent existing benchmarks: ChatBot Arena and Arena-Hard (Section 3.3, Figures 4 and 6). It also compares many models against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons are made against contemporary benchmarks (ChatBot Arena, Arena-Hard) and the paper evaluates state-of-the-art models including o1-preview, claude-3.5-sonnet, and gemini-1.5-pro (Table 1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "LiveBench has 18 tasks across 6 categories but no ablation study examines how removing tasks or categories affects the benchmark's discriminative power. Per-category correlation analysis (Figure 2) is provided but is not a proper ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: per-task scores, per-category averages, overall LiveBench score, Pearson correlation coefficients, rank correlations across updates, and pass@1 for coding tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly avoids human evaluation as a design choice: LiveBench 'scores answers automatically according to objective ground-truth values' (Abstract). No human evaluation of the benchmark quality or model outputs is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark questions serve as the test set, and models are not tuned on them. Additionally, 'Each month, we do not release the new questions until one month later, so that the public leaderboard always has 1/6 questions that are private' (Section 2.7)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide per-category breakdowns (Coding, Data Analysis, Instruction Following, Language, Math, Reasoning) for all 40 models. Figure 2 shows per-task correlations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 shows the relative worst task for each model. Section 3.2 identifies outlier models. The LLM judging ablation (Tables 8-9) shows where automated judging fails. Section 3.3 discusses models that are 'disproportionately stronger' or weaker in specific categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLM judges have high error rates (21-46%) on hard math and reasoning tasks (Tables 8-9, Section 3.3). They acknowledge 'the results are not definitive' for their judging ablation. They also note residual contamination in some tasks (Appendix A.7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (frequently-updated questions, objective ground-truth scoring, wide variety of tasks, top models below 70%) are all supported by the experimental results. o1-preview achieves 64.7%, the highest score, confirming below 70%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Most claims are descriptive or correlational. Causal-sounding claims are appropriately hedged: 'likely due to the known bias from using gpt-4 itself as the LLM judge' (Section 3.3). The LLM judging experiment (Appendix A.2) provides evidence for the claim that LLM judges struggle with hard questions they cannot solve."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 5) explicitly bounds the scope: 'there are still additions from which it would benefit... non-English language tasks... ground truth scoring... cannot be used for certain use cases... biases due to certain LLM families favoring certain prompt types.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses alternative explanations for benchmark differences: 'We hypothesize that the strong performance of some models such as the gemini-1.5 models on ChatBot Arena compared to LiveBench may be due to having an output style that is preferred by humans.' Appendix A.7 discusses different forms of contamination."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on specific tasks and calls it 'LiveBench score' without overclaiming. It does not frame task accuracy as measuring 'intelligence' or 'general capability' — claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions with snapshots are used throughout: 'gpt-4-0125-preview', 'claude-3-5-sonnet-20240620', 'o1-preview-2024-09-12', etc. Table 4 lists all 80+ model versions with citations."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts are provided verbatim for multiple tasks throughout Sections 2.1-2.6 and Appendix A.3 (math competitions, zebra puzzles, typos, Web of Lies v2, spatial reasoning, connections). All code including prompts is released on GitHub."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'we perform single-turn evaluation with temperature 0, unless otherwise noted in the model card' and 'We run all open-source models with bfloat16.' Appendix A.5 describes how model hyperparameters are set to match model documentation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LiveBench performs direct single-turn model evaluation without tools, agents, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Detailed descriptions of question generation and processing are provided for each task category (Sections 2.1-2.6 and Appendix A.3), including how math questions are modified, how typos are injected, how tables are sampled, and how coding questions are truncated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Conclusions, Limitations, and Future Work' and contains a substantive 'Limitations and Future Work' subsection discussing specific shortcomings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: lack of non-English tasks, inability to evaluate open-ended questions with ground-truth scoring, prompt biases favoring certain LLM families (Section 5). Appendix A.7 discusses residual contamination in coding questions from November 2023."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states specific boundaries: 'we hope to add non-English language tasks', 'ground truth scoring... cannot be used for certain use cases, such as write a travel guide to Hawaii', and 'there are still biases due to certain LLM families favoring certain prompt types.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'we open-source the leaderboard, all questions, all code to run API and open-source models, all model outputs for 40 models, and all code to score the models.' Raw model outputs and questions are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed data collection procedures are described for each task: math competition sources (AMC12, AIME, SMC, USAMO, IMO), coding sources (LiveCodeBench, LeetCode), Kaggle/Socrata datasets, Guardian articles, ArXiv abstracts, and IMDb/Wikipedia synopses (Sections 2.1-2.6)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are publicly available benchmarks, competitions, news articles, and datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from data sources to final questions is documented for each task, including question generation, modification, scoring methodology (Sections 2.1-2.6, Appendix A.3), and the monthly update process (Section 2.7, Appendix A.6)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The footnote on page 1 states 'Sponsored by Abacus.AI' and lists author affiliations including Abacus.AI, NYU, Nvidia, UMD, USC, and Columbia."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on page 1. Multiple first authors are from Abacus.AI, and the paper evaluates Dracarys models (an Abacus.AI product) alongside other models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Abacus.AI sponsored the research and multiple first authors are Abacus.AI employees. The benchmark evaluates Abacus.AI's own Dracarys models, which rank 14th overall (50.1%). The funder has a direct interest in the benchmark's outcomes and in how their models are ranked."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Given that Abacus.AI employees designed the benchmark and evaluate their own products, a formal disclosure would be expected."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the 40 models evaluated. It discusses contamination conceptually and uses temporal prevention (recent data), but individual model cutoffs are not listed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is motivated by train/test overlap concerns. Appendix A.7 explicitly discusses: 'we acknowledge that LiveBench does not fully satisfy [test set contamination]: while nearly all questions are from June 2024 or more recent, there are some coding questions from November 2023.' The benchmark design uses temporal prevention."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is the paper's central concern. They address it through: (1) using recently-released data sources, (2) monthly question updates, (3) keeping 1/6 questions private, (4) modifying existing benchmark tasks to reduce memorization, and (5) discussing different types of contamination in Appendix A.7."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 16 provides approximate costs in USD for running LiveBench on each API model (e.g., o1-preview at ~$47.87, claude-3-haiku at ~$0.90). Table 15 provides mean and standard deviation of input/output tokens per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-model API costs are listed in Table 16 but total computational budget is not stated. GPU hours for running open-source models are not reported. The paper only mentions that evaluation is 'easily within the computational budgets of the authors' institutions' (Section 2.7)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single runs with temperature 0. While temperature 0 reduces stochasticity, no sensitivity analysis across runs or seeds is reported. Some API implementations are not perfectly deterministic even at temperature 0."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'we perform single-turn evaluation with temperature 0,' implying a single run per model per question. Bootstrap confidence intervals are computed over questions, not multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states prompts are 'tailored for each category and task' (Section 2) but does not report how many prompt variants were tried or the budget spent on prompt design. Model hyperparameters are set from documentation, but prompt engineering effort is unreported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated with standardized settings (temperature 0, documented chat templates) and all 40 models' full results are reported in Table 2. No selective reporting of configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 40 models across 18 tasks and reports many correlation coefficients (Tables 5-6) without any correction for multiple comparisons. Claims like 'X substantially outperforms Y' are made from raw score comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Abacus.AI employees designed the benchmark and evaluate their own Dracarys models (which rank 14th/40 overall). The potential bias of benchmark designers evaluating their own models is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates models ranging from 0.5B to 405B parameters but does not report performance as a function of compute budget. phi-3.5-moe with 6.6B active parameters is noted to outperform larger models, but no systematic compute-performance analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses what LiveBench measures versus alternatives. Section 3.3 compares with ChatBot Arena and Arena-Hard, analyzing where rankings diverge and hypothesizing why. The paper discusses biases of LLM judging and human judging approaches. Figure 2 analyzes inter-task correlations to assess category independence."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. LiveBench evaluates models directly via single-turn prompts without agents, tools, or multi-step workflows."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the paper's central concern. All benchmark questions use data from November 2023 or later, and the benchmark is updated monthly with new questions. Section 1 cites evidence of temporal contamination in other benchmarks (e.g., Codeforces performance dropping after training cutoff)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup (prompt format, answer structure, or context) could leak answer information to the models. Appendix A.4 discusses parsing fairness but not feature leakage in the Kapoor & Narayanan sense."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential non-independence between questions (e.g., multiple questions from the same math competition, questions from the same Kaggle dataset, or structural similarities between questions within a task)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses concrete prevention methods: temporal splits (using data after training cutoffs), keeping 1/6 questions private each month, modifying questions from existing benchmarks (rearranging answer choices, changing prose), and not releasing question generation code. These are concrete anti-leakage measures, though detection methods (canary strings, membership inference) are not used."
       }
     }
   },
   "claims": [
     {
       "claim": "LiveBench is the first benchmark that combines frequently-updated questions from recent sources, objective ground-truth scoring without LLM judges, and diverse task coverage across six categories.",
       "evidence": "Section 1 describes the three desiderata and claims novelty. The benchmark includes 18 tasks across math, coding, reasoning, language, instruction following, and data analysis, with questions from recent sources and automated scoring.",
       "supported": "strong"
     },
     {
       "claim": "o1-preview-2024-09-12 performs the best overall, 6% better than all other models, with no current model achieving higher than 70% accuracy.",
       "evidence": "Table 1 shows o1-preview at 64.7% overall, with the next-best model (claude-3-5-sonnet-20241022) at 58.5%. All models are below 70%.",
       "supported": "strong"
     },
     {
       "claim": "LLM judges cannot accurately evaluate challenging math and reasoning questions, with error rates of 21-46%.",
       "evidence": "Tables 8-9 (Appendix A.2) show GPT-4-Turbo judge error rates of 21.4-42.0% on AMC12, AIME, SMC, and Zebra puzzles. The authors acknowledge this is a 'preliminary study' and 'the results are not definitive.'",
       "supported": "moderate"
     },
     {
       "claim": "LiveBench correlates well with ChatBot Arena (r=0.91) and Arena-Hard (r=0.88) but reveals meaningful differences between benchmarks.",
       "evidence": "Section 3.3 reports the correlation coefficients. Figures 4 and 6 show the comparisons and identify outlier models (e.g., GPT-4 models performing better on Arena-Hard, likely due to LLM judge bias).",
       "supported": "strong"
     },
     {
       "claim": "Model rankings remain stable across monthly benchmark updates (rank correlation >0.997) while the benchmark becomes harder over time.",
       "evidence": "Section 3.4 reports rank correlations >0.997 between updates and a 1.2% drop in median/mean scores across two updates.",
       "supported": "moderate"
     },
     {
       "claim": "Math competition performance (math_comp) is the greatest indicator of overall model performance.",
       "evidence": "Table 6 shows math_comp has the second-highest task correlation with overall LiveBench score (r=0.9035, after web_of_lies_v2 at r=0.9136). Section 3.2 states math_comp 'correlates the highest with average LiveBench performance.'",
       "supported": "moderate"
     },
     {
       "claim": "The best-performing open-source models (Llama-3.1-405b and Qwen2.5-72b) outperform GPT-4-turbo.",
       "evidence": "Table 2 shows meta-llama-3.1-405b-instruct-turbo at 51.1% and qwen2.5-coder-32b-instruct at 45.0%, while gpt-4-turbo-2024-04-09 scores 49.6%. However, qwen2.5-72b-instruct is not shown in the results table (qwen2.5-coder-32b-instruct is shown instead).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Multiple first authors are from Abacus.AI, the paper's sponsor. Abacus.AI's Dracarys models are evaluated and rank 14th/40 (50.1%), but no competing interests statement is provided. The benchmark designers evaluating their own product creates potential for favorable task/prompt selection."
     },
     {
       "flag": "No significance tests for model comparisons",
       "detail": "Claims like 'o1-preview substantially outperforms all other models' and 'the best-performing open-source models outperform gpt-4-turbo' are based on comparing raw point estimates. While bootstrap CIs are shown in figures, no formal significance tests support the comparative claims in the text."
     },
     {
       "flag": "Residual contamination acknowledged",
       "detail": "Appendix A.7 acknowledges: 'there are some coding questions from November 2023, and the AMC questions have only undergone a low level of modification from their November 2023 version. Therefore, a limited fraction of LiveBench is likely contaminated on all recent LLMs.' This undermines the central contamination-resistance claim for some tasks."
     },
     {
       "flag": "Preliminary LLM judging study presented as evidence",
       "detail": "The LLM judging ablation (Appendix A.2) is described as 'preliminary' with 'not definitive' results, using only one judge model (GPT-4-Turbo) and two evaluated models. Despite this, it is used to support the paper's central argument against LLM judging."
     }
   ],
   "cited_papers": [
     {
       "title": "To the cutoff... and beyond? A longitudinal perspective on LLM data contamination",
       "authors": ["Manley Roberts", "Himanshu Thakur", "Christine Herlihy", "Colin White", "Samuel Dooley"],
       "year": 2024,
       "relevance": "Provides evidence of test set contamination in LLM benchmarks, including performance drops after training cutoff dates on Codeforces."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark that LiveBench draws coding tasks from; shares the temporal-freshness approach."
     },
     {
       "title": "Chatbot Arena: An open platform for evaluating LLMs by human preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Major human-preference LLM evaluation platform that LiveBench compares against (r=0.91 correlation)."
     },
     {
       "title": "From live data to high-quality benchmarks: The Arena-Hard pipeline",
       "authors": ["Tianle Li", "Wei-Lin Chiang", "Evan Frick"],
       "year": 2024,
       "relevance": "LLM-judge based benchmark that LiveBench compares against (r=0.88 correlation); demonstrates LLM judge biases."
     },
     {
       "title": "Challenging big-bench tasks and whether chain-of-thought can solve them",
       "authors": ["Mirac Suzgun", "Nathan Scales", "Nathanael Schärli"],
       "year": 2023,
       "arxiv_id": "2210.09261",
       "relevance": "Big-Bench Hard benchmark whose Web of Lies task LiveBench extends with a harder v2 version."
     },
     {
       "title": "Instruction-following evaluation for large language models",
       "authors": ["Jeffrey Zhou", "Tianjian Lu", "Swaroop Mishra"],
       "year": 2023,
       "arxiv_id": "2311.07911",
       "relevance": "IFEval benchmark for verifiable instruction following; LiveBench adapts its instruction set with live prompts from Guardian articles."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Analyzes data contamination and its effects on trustworthy LLM evaluation."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da", "Dean Lee"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Demonstrates evidence that several models have overfit to the GSM8K benchmark, motivating contamination-resistant evaluation."
     },
     {
       "title": "Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?",
       "authors": ["Aaditya K Singh", "Muhammed Yusuf Kocyigit"],
       "year": 2024,
       "arxiv_id": "2411.03923",
       "relevance": "Discusses measurement approaches for evaluation data contamination in LLMs."
     },
     {
       "title": "Length-controlled AlpacaEval: A simple way to debias automatic evaluators",
       "authors": ["Yann Dubois", "Balázs Galambosi", "Percy Liang", "Tatsunori B Hashimoto"],
       "year": 2024,
       "arxiv_id": "2404.04475",
       "relevance": "Addresses LLM judge biases including verbosity preference in automatic evaluation, relevant to LiveBench's argument against LLM judging."
     },
     {
       "title": "Functional benchmarks for robust evaluation of reasoning performance, and the reasoning gap",
       "authors": ["Saurabh Srivastava", "Anto PV", "Shashank Menon"],
       "year": 2024,
       "arxiv_id": "2402.19450",
       "relevance": "Modifies MATH dataset by changing numbers and finds performance declines, showing contamination effects on math benchmarks."
     },
     {
       "title": "MMLU-Pro: A more robust and challenging multi-task language understanding benchmark",
       "authors": ["Yubo Wang", "Xueguang Ma", "Ge Zhang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Contemporary static LLM benchmark used on the HuggingFace Open LLM Leaderboard; context for LiveBench's dynamic approach."
     },
     {
       "title": "Data contamination quiz: A tool to detect and estimate contamination in large language models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "arxiv_id": "2311.06233",
       "relevance": "Tool for detecting data contamination in LLMs, relevant to benchmark contamination assessment."
     },
     {
       "title": "Proving test set contamination in black box language models",
       "authors": ["Yonatan Oren", "Nicole Meister", "Niladri Chatterji"],
       "year": 2023,
       "arxiv_id": "2310.17623",
       "relevance": "Provides methods for proving test set contamination in black-box LLMs."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "LiveBench is immediately usable as a fully open-source benchmark with pip-installable code, leaderboard, and monthly updates — practitioners can evaluate any model today."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The contamination problem is well-known; the paper confirms rather than challenges beliefs, though the LLM judging failure rates on hard questions add mild surprise."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or security concerns are raised; this is a benchmark methodology paper."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implicitly argues that popular benchmarks (ChatBot Arena, Arena-Hard, MMLU) are compromised by contamination or LLM judge bias, and shows GPT-4 judges are biased toward their own outputs."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Fully open-source on GitHub with all code, questions, and model outputs; a live leaderboard at livebench.ai; and instructions to evaluate new models."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Published at ICLR 2025, evaluates GPT-4/o1, Claude, Gemini; co-authored by Yann LeCun (NYU/Meta). Abacus.AI is less prominent but the models evaluated are household names."
     }
   }
 }