ai-research-survey

scan.json (26641B)

---

 {
   "paper": {
     "title": "Humanity's Last Exam",
     "authors": ["Long Phan", "Alice Gatti", "Ziwen Han", "Nathaniel Li", "Josephina Hu", "Hugh Zhang", "Chen Bo Calvin Zhang", "Mohamed Shaaban", "John Ling", "Sean Shi", "Michael Choi", "Anish Agrawal", "Arnav Chopra", "Adam Khoja", "Ryan Kim", "Richard Ren", "Jason Hausenloy", "Oliver Zhang", "Mantas Mazeika", "Summer Yue", "Alexandr Wang", "Dan Hendrycks"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.14249"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided in the paper. The paper releases the dataset at https://lastexam.ai but does not mention releasing evaluation code or scripts."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 2,500 questions is publicly released at https://lastexam.ai. Section 1 states: 'we publicly release HLE at https://lastexam.ai.' A private held-out test set is also maintained."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency lists are provided. The paper describes using various model APIs but does not provide setup details for reproducing the evaluation pipeline."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While Section C.1.1 provides evaluation prompts and Section C.5 lists model versions, there are no instructions for running the full evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-3 report only point estimates of accuracy and calibration error (e.g., '8.0%' accuracy for O1) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares model performance across models and categories but uses no statistical significance tests. Differences between models are presented as raw numbers without any test of whether differences are meaningful."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Model performance is presented as raw accuracy percentages and calibration errors without contextualizing the magnitude of differences between models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 2,500 questions but no justification is given for why this number was chosen. No power analysis or statistical reasoning for the dataset size is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation across experimental runs is reported. Results appear to be from single evaluation runs per model. Section 4.2 acknowledges 'inherent noise in model inference' but does not quantify it with repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 8 frontier models against each other (Table 1) and compares HLE difficulty against prior benchmarks like MMLU, GPQA, ARC Prize, and others in Figure 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models are contemporary frontier models including GPT-4o, Claude 3.5 Sonnet, Gemini 2.0 Flash Thinking, o1, DeepSeek-R1, and o3-mini (high), all from late 2024 to early 2025."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper, not a system paper. There is no system with components to ablate. The paper's contribution is the dataset itself."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy (%) and RMS calibration error (%) for each model in Table 1. Token count analysis is also provided in Figure 5 and Section C.4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Expert human reviewers evaluated question quality through two rounds of review (Section 3.2). An audit process with university students verified answers (Section B.2). Expert disagreement rates are reported in Section B.3."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'we publicly release these questions, while maintaining a private test set of held out questions to assess model overfitting.' Figure 4 also confirms a private held-out set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 (Section C.3) provides category-wise performance breakdowns across Math, Biology/Medicine, Physics, CS/AI, Humanities, Chemistry, Engineering, and Other, for both text-only and full dataset evaluations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide qualitative analysis of specific failure cases or error patterns. It notes low accuracy and high calibration error but does not analyze what types of errors models make or why they fail on specific questions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: all models achieve very low accuracy, all models show poor calibration (high RMS calibration error above 70%), and models confidently provide incorrect answers. Section B.3 reports a 15.4% expert disagreement rate, acknowledging dataset limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'State-of-the-art LLMs demonstrate low accuracy and calibration on HLE,' which is supported by Table 1 showing accuracy ranging from 2.7% to 13.4% and calibration errors of 73-89%. The claim of 'broad subject coverage' is supported by Figure 3 and Section B.4."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It presents a benchmark and reports descriptive performance results without claiming causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly bounds claims: 'High accuracy on HLE would demonstrate expert-level performance on closed-ended, verifiable questions...but it would not alone suggest autonomous research capabilities or \"artificial general intelligence.\" HLE tests structured academic problems rather than open-ended research or creative problem-solving abilities.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses that low accuracy is 'partially by design' due to the filtering process and acknowledges 'inherent noise in model inference.' Section B.3 discusses expert disagreement as a factor affecting measured accuracy. The paper also acknowledges that about 15.4% of questions may have expert disagreements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section C.5 (Table 4) provides exact model versions: gpt-4o-2024-11-20, claude-3-5-sonnet-20241022, gemini-1.5-pro-002, o1-2024-12-17, o3-mini-2025-01-31, and specifies January 20, 2025 release for DeepSeek-R1. Grok 2 uses 'grok-2-latest' which is less precise."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full evaluation prompts are provided in Section C.1.1, including both the system prompt for model evaluation and the judge prompt used for o3-mini answer verification. An example structured judge response is also included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 states: 'All models use temperature 0.0 when configurable and not otherwise stated. o3-mini and o1 models only support temperature 1.0.' Gemini 2.0 Flash Thinking is noted as 'sampled at temperature 0.7.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated in a straightforward zero-shot prompting setup."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data collection and filtering pipeline is documented in detail. Section 3.1 describes submission criteria, Section 3.2 describes the multi-stage review (LLM difficulty check, two rounds of expert review), and Figure 4 provides a visual pipeline. Section B.2 describes post-release refinement including community feedback and audit processes."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) serves as a limitations section, discussing what HLE does and does not measure. Section B.3 discusses expert disagreement rates as a limitation. However, there is no dedicated 'Limitations' subsection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 discusses specific threats: expert disagreement rate of 15.4%, higher disagreement in health/medicine domains (~18%), the challenge of verifying research-experience-based questions, and the limitation that reviewers could not always verify solution rationales within 5 minutes. Section 4.2 notes that 'small inflections close to zero accuracy are not strongly indicative of progress.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'HLE tests structured academic problems rather than open-ended research or creative problem-solving abilities, making it a focused measure of technical knowledge and reasoning. HLE may be the last academic exam we need to give to models, but it is far from the last benchmark for AI.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The 2,500 questions comprising the public set are released at https://lastexam.ai. The raw benchmark data is available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the collection process in detail: questions from ~1,000 expert contributors across 500+ institutions and 50 countries, submission format requirements, $500,000 prize pool incentive structure, and question style requirements."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes contributor recruitment: 'questions from nearly 1000 subject expert contributors affiliated with over 500 institutions across 50 countries – comprised mostly of professors, researchers, and graduate degree holders.' The prize pool is described as an incentive mechanism. Co-authorship was offered as an additional incentive."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 documents the full pipeline from submission through LLM difficulty check, expert review rounds 1 and 2, organizer approval, and post-release refinement. Section B.1 notes over 70,000 attempts resulted in approximately 13,000 questions forwarded to human review, culminating in 2,500 accepted questions."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or grant acknowledgment section is found in the paper. The $500,000 prize pool is mentioned but its source is not disclosed. The lead affiliations are Center for AI Safety and Scale AI, but corporate funding details are not provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: organizing team members are from Center for AI Safety and Scale AI. Section A provides detailed institutional affiliations for all contributors. Anthropic and OpenAI employees are among contributors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI (co-organizer) is an AI data company that has a financial interest in benchmark creation and AI evaluation. Center for AI Safety has a mission-aligned interest in demonstrating capability gaps. The funder of the $500,000 prize pool is not disclosed, making independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Scale AI is a commercial company, and some contributors are from companies whose products are evaluated (Anthropic, Google DeepMind, OpenAI), but no conflict-of-interest statement addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates frontier LLMs on the HLE benchmark but does not state training data cutoff dates for any of the models tested. This is relevant because questions may have been created and shared before model training concluded."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 addresses contamination: searchable questions were audited using 'GPT-4o mini/GPT-4o search and Perplexity Sonar models' to identify and remove questions that could be found via web search. The design process itself (Section 3.1) requires questions to be 'resistant to simple internet lookup or database retrieval.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is designed to resist contamination: questions must be original, non-searchable, and pass LLM difficulty checks. Section B.2 describes systematic removal of searchable questions. The private held-out set addresses gaming concerns. However, the paper does not discuss whether pre-release question submission data could have leaked into training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper, not a human subjects study. The expert reviewers and contributors are part of the dataset creation process, not research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Expert contributors voluntarily submit questions and reviewers evaluate them as part of a collaborative benchmark creation effort."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Contributors are characterized as 'professors, researchers, and graduate degree holders' from 500+ institutions across 50 countries, but this is dataset provenance, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The paper describes question inclusion/exclusion criteria (Sections 3.1, 3.2) but these are for dataset items, not research participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment is conducted that would require randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment is conducted that would require blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The pipeline from 70,000 LLM attempts to 13,000 candidate questions to 2,500 accepted questions is documented, but this is data filtering, not participant attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "While token counts are reported in Figure 5 and Section C.4, no monetary costs or wall-clock evaluation times are reported. The cost of running evaluations across all models is not disclosed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not disclose how much API spend was required for the 70,000+ LLM attempts during question submission, or for the final evaluations."
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art LLMs achieve very low accuracy on HLE, with the best model (o3-mini high) achieving only 13.4% accuracy.",
       "evidence": "Table 1 shows accuracy ranging from 2.7% (GPT-4o) to 13.4% (o3-mini high) across 8 frontier models on the full 2,500-question benchmark.",
       "supported": "strong"
     },
     {
       "claim": "All models exhibit poor calibration on HLE, with RMS calibration errors above 70%.",
       "evidence": "Table 1 reports RMS calibration errors ranging from 73% (DeepSeek-R1) to 89% (GPT-4o), indicating models are confidently incorrect.",
       "supported": "strong"
     },
     {
       "claim": "HLE is the first benchmark to combine expert-level difficulty with broad subject coverage across dozens of academic disciplines.",
       "evidence": "Section 3 and Figure 3 show coverage across 100+ subjects. The design process (Section 3.1) explicitly filters for expert difficulty. However, the claim of being 'the final closed-ended benchmark' is aspirational rather than empirically justified.",
       "supported": "moderate"
     },
     {
       "claim": "HLE questions are resistant to internet retrieval and memorization.",
       "evidence": "Section 3.1 requires questions to be 'resistant to simple internet lookup or database retrieval.' Section B.2 describes systematic auditing of searchable questions using search-enabled models. However, the effectiveness of this filtering is only implicitly validated ('current frontier model performance on HLE after applying this procedure is similar to their performance on HLE before applying this procedure').",
       "supported": "moderate"
     },
     {
       "claim": "The expert disagreement rate on HLE is approximately 15.4% for the public set.",
       "evidence": "Section B.3 reports: 'This iterative process yielded a final estimated expert disagreement rate of 15.4% for the public set.' A targeted review on biology, chemistry, and health found approximately 18% disagreement.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Humanity's Last Exam (HLE) is a 2,500-question multi-modal benchmark spanning 100+ academic subjects, created by ~1,000 expert contributors from 500+ institutions. All frontier LLMs tested achieve very low accuracy (2.7% to 13.4%) and exhibit poor calibration (RMS errors 73-89%), showing a significant gap between current AI capabilities and expert-level academic knowledge. Reasoning models use substantially more tokens but only achieve moderate improvements. The benchmark includes both a public set and a private held-out test set to detect overfitting.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All accuracy and calibration results are reported as single-run point estimates with no confidence intervals, error bars, or variance across runs. Given the acknowledged 'inherent noise in model inference,' repeated runs could yield meaningfully different results, especially at low accuracy levels."
     },
     {
       "flag": "Conflict of interest",
       "detail": "Scale AI is a co-organizing institution that commercially benefits from AI evaluation and benchmarking. Several model providers (Anthropic, OpenAI, Google DeepMind) have employees among the contributors. No conflict-of-interest statement is provided."
     },
     {
       "flag": "15.4% expert disagreement rate",
       "detail": "A 15.4% expert disagreement rate on the public set (18% for biology/chemistry/health) means a non-trivial fraction of questions may have debatable answers, which complicates interpretation of model accuracy at these low levels. At ~10% accuracy, the signal-to-noise ratio between real model capability and label noise becomes concerning."
     },
     {
       "flag": "Selection bias in benchmark construction",
       "detail": "Questions are only included if LLMs fail to answer them correctly (Section 3.2). This adversarial filtering guarantees low initial accuracy by design, making it difficult to interpret whether low scores reflect genuine capability gaps or benchmark construction methodology. The paper acknowledges this partially in Section 4.2."
     },
     {
       "flag": "Prize pool source undisclosed",
       "detail": "A $500,000 USD prize pool was offered but the paper does not disclose who funded it, making it impossible to assess potential conflicts of interest in question selection and benchmark design."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt"],
       "year": 2021,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU is the canonical multi-task benchmark that HLE aims to succeed, demonstrating benchmark saturation in LLM evaluation."
     },
     {
       "title": "GPQA: A graduate-level google-proof Q&A benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland"],
       "year": 2023,
       "arxiv_id": "2311.12022",
       "relevance": "Prior expert-level benchmark using multi-stage review and expert-written questions, directly comparable to HLE's methodology."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major code generation benchmark for evaluating LLM software engineering capabilities."
     },
     {
       "title": "FrontierMath: A benchmark for evaluating advanced mathematical reasoning in AI",
       "authors": ["Elliot Glazer", "Ege Erdil", "Tamay Besiroglu"],
       "year": 2024,
       "arxiv_id": "2411.04872",
       "relevance": "Contemporary expert-level math benchmark using multi-stage review, directly comparable in methodology to HLE."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational for LLM code generation evaluation and relevant to benchmark contamination discussions."
     },
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for evaluating LLM agents on ML engineering tasks, relevant to agentic AI evaluation methodology."
     },
     {
       "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
       "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "Benchmark comparing AI agent R&D capabilities against human experts, complementary to HLE's assessment of closed-ended academic capabilities."
     },
     {
       "title": "CyBench: A framework for evaluating cybersecurity capabilities and risks of language models",
       "authors": ["Andy K. Zhang", "Neil Perry", "Riya Dulepet"],
       "year": 2024,
       "arxiv_id": "2408.08926",
       "relevance": "Benchmark evaluating LLM cybersecurity capabilities, relevant to AI safety and capability assessment."
     },
     {
       "title": "Measuring short-form factuality in large language models",
       "authors": ["Jason Wei", "Narina Karina", "Hyung Won Chung"],
       "year": 2024,
       "arxiv_id": "2411.04368",
       "relevance": "Provides the calibration measurement methodology used in HLE's evaluation setup."
     },
     {
       "title": "Evaluating frontier models for dangerous capabilities",
       "authors": ["Mary Phuong", "Matthew Aitchison", "Elliot Catt"],
       "year": 2024,
       "arxiv_id": "2403.13793",
       "relevance": "Evaluation of frontier model dangerous capabilities, relevant to AI safety benchmark methodology."
     },
     {
       "title": "The WMDP benchmark: Measuring and reducing malicious use with unlearning",
       "authors": ["Nathaniel Li", "Alexander Pan", "Anjali Gopal"],
       "year": 2024,
       "arxiv_id": "2403.03218",
       "relevance": "Benchmark measuring potential for malicious use of LLMs, relevant to AI safety evaluation methodology."
     }
   ]
 }