ai-research-survey

scan-v5.json (20087B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI",
     "authors": [
       "Elliott S. Glazer",
       "Ege Erdil",
       "T. Besiroglu",
       "Diego Chicharro",
       "Evan Chen"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2411.04872",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are substantiated: hundreds of problems from 60+ mathematicians (Section 2), 70% of MSC2020 subjects covered (Section 3), <2% solve rate confirmed in Section 4.2, and contamination-prevention measures documented in Section 2.4.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make causal claims requiring controlled study design; it describes a benchmark and presents observational evaluation results.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims about model performance are bounded to the six specific models tested, and the paper explicitly cautions that 'the precise ordering of model performance should be interpreted with significant caution' due to low absolute success rates.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically consider alternative explanations for the <2% solve rate—e.g., that the code-submission evaluation format, 10,000-token limits (exceeded in 45%+ of Claude/GPT-4o attempts), or prompting choices may have suppressed performance independent of mathematical ability.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes the measured proxy (correct numerical/computable answers) from the claimed capability (mathematical reasoning), acknowledging in Section 7 that 'the practical focus on automatically verifiable and numerical answers excludes proof-writing and open-ended exploration.'",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 contains multiple dedicated paragraphs on limitations including exclusion of proof-writing, the gap from real research timescales, low current solve rates limiting discrimination, and an approximately 10% problem error rate.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the ~10% estimated answer error rate (derived from 2/35 second-reviewed problems having errors), subjective and inconsistent difficulty ratings between reviewers, and the benchmark's inability to rank models meaningfully when solve rates are near zero.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the benchmark 'excludes proof-writing and open-ended exploration' and that problems 'fall short of typical mathematical research, which often spans weeks, months or even years of sustained investigation.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments mention 'OpenAI for their support in creating the benchmark' but there is no formal funding disclosure or funding statement in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed in the header (Epoch AI, MIT, UC Berkeley, Harvard, Cornell, etc.).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI acknowledged as supporting the benchmark's creation, while three OpenAI models (o1-preview, o1-mini, GPT-4o) are directly evaluated and ranked on it—a clear funder-outcome conflict.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "The central claimed capability—'advanced mathematical reasoning'—is never formally defined; the paper characterizes it only through examples and expert opinions rather than a precise definition of what cognitive or computational skills constitute 'mathematical reasoning.'",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is unambiguously stated: a benchmark of hundreds of original, expert-crafted, automatically verifiable mathematics problems targeting research-level difficulty to evaluate frontier AI models.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 systematically reviews and distinguishes FrontierMath from MATH, GSM8K, ARB, GHOSTS, OlympiadBench, OmniMATH, Putnam-AXIOM, AIMO, and MiniF2F, explaining how FrontierMath addresses their limitations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that problems requiring 'deep theoretical understanding, creative insight, and specialized expertise' from research mathematicians measure genuine mathematical reasoning, and this is qualitatively validated by three Fields Medalists confirming problem difficulty.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "A three-dimensional difficulty system is described (Background 1–5, Creativity in hours, Execution in hours), and Section A presents five sample problems chosen from each difficulty quintile with ratings.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly shows a <2% solve rate (no ceiling effect), and Figure 2 compares FrontierMath to near-saturated benchmarks (MATH, GSM8K, MMLU) to motivate why a harder benchmark is needed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "The paper collects only qualitative expert opinions from four mathematicians; no quantitative human solve rate or completion time distribution is measured or reported.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "The binary correct/incorrect scoring via automated verification is justified as eliminating human bias and enabling scalable evaluation; edge cases (non-unique solutions, symbolic outputs) are addressed with SymPy and custom verification scripts.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is a primary design goal: all problems are new and unpublished, submitted via encrypted channels, reviewed for originality against known problems, and checked with plagiarism detection tools.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 quotes Tao estimating the benchmark 'will resist AIs for several years at least,' and Section 8 commits to ongoing addition of rigorously vetted new problems to maintain relevance.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper discusses failure modes including guessing susceptibility (addressed via large nonobvious answers), the ~10% problem error rate (Section 2.3), shortcut strategies that undermine difficulty estimates, and the format's exclusion of proof-based problems.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": false,
           "justification": "The full benchmark is not publicly accessible (requires contacting Epoch AI), so the reported evaluation numbers cannot be independently reproduced; only 5 sample problems with solutions are publicly released.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "While the collection methodology is described in detail, there is no formal data card, and the dataset itself is not publicly available, making complete documentation inaccessible to the research community.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "Access is described only as 'reach out to math_evals@epochai.org' with no stated terms, license, or restrictions on use—others cannot determine under what conditions they can use the benchmark.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 explicitly states what the benchmark does and does not measure (no proof-writing, no long-horizon research skills), and the Discussion clarifies that <2% solve rates limit model discrimination, guiding appropriate interpretation.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Current state-of-the-art AI models solve under 2% of FrontierMath problems",
       "evidence": "Evaluation of six frontier models (o1-preview, o1-mini, GPT-4o, Claude 3.5 Sonnet, Grok 2 Beta, Gemini 1.5 Pro 002) with 8 runs each; best model achieved ~2% mean accuracy (Figure 6)",
       "supported": "strong"
     },
     {
       "claim": "FrontierMath problems require multiple hours of effort from expert research mathematicians",
       "evidence": "Difficulty ratings assign Creativity and Execution in hours; Fields Medalists (Tao, Gowers, Borcherds) unanimously characterized problems as 'exceptionally challenging' requiring 'deep domain expertise'",
       "supported": "moderate"
     },
     {
       "claim": "Existing mathematics benchmarks (MATH, GSM8K, MMLU) are near saturation with current AI models",
       "evidence": "Figure 2 shows >94% solve rates for top models on MATH, >96% on GSM8K, >98% on MMLU College Math subset",
       "supported": "strong"
     },
     {
       "claim": "FrontierMath effectively prevents data contamination through novel, unpublished problems",
       "evidence": "Problems created specifically for the benchmark, handled via encrypted channels, checked with Quetext and Copyscape with no significant matches found",
       "supported": "moderate"
     },
     {
       "claim": "The benchmark has an estimated ~10% critical error rate in problem correctness",
       "evidence": "Second reviews of 35 problems found 2 with incorrect answers; Jeffreys prior yields posterior error rate of ~6.9%, rounded up to ~10% accounting for undetected errors",
       "supported": "moderate"
     },
     {
       "claim": "o1-preview demonstrates strongest performance on problems solved at least once",
       "evidence": "Repeated trial results (Table 2) show o1-preview solved one problem 5/5 times and outperformed other models on 3 of 4 repeated-trial problems",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "FrontierMath introduces hundreds of original, research-level mathematics problems where no current frontier AI model achieves more than 2% accuracy, revealing a substantial gap between AI capabilities and expert mathematical ability. The benchmark uses automated verification via code execution to enable scalable, reproducible evaluation while covering 70% of MSC2020 top-level subjects. Three Fields Medalists (Tao, Gowers, Borcherds) qualitatively confirmed the problems as exceptionally difficult, requiring deep domain expertise. The authors acknowledge an estimated ~10% problem error rate, lack of quantitative human baseline, and that the benchmark's access-controlled nature limits independent reproducibility.",
   "red_flags": [
     {
       "flag": "Funder evaluates own models",
       "detail": "OpenAI is acknowledged as supporting benchmark creation, yet three OpenAI models are evaluated and ranked on the benchmark, creating a direct conflict of interest that is not disclosed or addressed."
     },
     {
       "flag": "No quantitative human baseline",
       "detail": "Human performance is characterized only qualitatively through four expert interviews; no systematic human solve rate, completion time distribution, or formal human evaluation is reported."
     },
     {
       "flag": "~10% estimated error rate",
       "detail": "Only 35 of hundreds of problems received second review; 2/35 had incorrect answers, yielding an estimated ~10% critical error rate—higher than the authors' own comparisons to ImageNet (6%) and MMLU (9%)."
     },
     {
       "flag": "Benchmark not publicly available",
       "detail": "The full problem set requires contacting Epoch AI; no license terms are stated. Reported evaluation numbers cannot be independently reproduced or verified."
     },
     {
       "flag": "Inconsistent difficulty ratings",
       "detail": "Section 2.5 explicitly acknowledges that first and second reviewers 'rarely matched and often showed substantial differences' in difficulty ratings, undermining the reliability of the difficulty characterization."
     },
     {
       "flag": "Evan Chen is both co-author and interviewee",
       "detail": "Section 6 presents interviews with mathematicians validating difficulty, but Evan Chen—listed as a core contributor and problem author—is one of four interviewees, creating a circularity in the validation."
     },
     {
       "flag": "Alternative evaluation explanations unexplored",
       "detail": "The <2% solve rate is attributed to mathematical difficulty, but token limits were exceeded in 45%+ of attempts by three models, suggesting evaluation framework constraints may partially explain poor performance."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring Mathematical Problem Solving with the MATH Dataset",
       "relevance": "Foundational math benchmark that FrontierMath directly supersedes; demonstrates saturation at near-perfect accuracy by frontier models"
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems (GSM8K)",
       "relevance": "Elementary-level math benchmark used as comparison point for benchmark saturation analysis"
     },
     {
       "title": "ARB: Advanced Reasoning Benchmark for Large Language Models",
       "relevance": "Prior university/contest-level benchmark that FrontierMath aims to exceed in difficulty"
     },
     {
       "title": "Omni-MATH: A Universal Olympiad Level Mathematic Benchmark",
       "relevance": "Direct competitor benchmark at olympiad level; used in difficulty comparison figures"
     },
     {
       "title": "Putnam-AXIOM: A Functional and Static Benchmark for Measuring Higher Level Mathematical Reasoning",
       "relevance": "Related approach using programmatic variant generation for contamination resistance; demonstrates 50%→34% drop from contamination effects"
     },
     {
       "title": "MiniF2F: A Cross-System Benchmark for Formal Olympiad-Level Mathematics",
       "relevance": "Formal theorem-proving benchmark; contrasted with FrontierMath's informal numerical approach"
     },
     {
       "title": "Mathematical Capabilities of ChatGPT (GHOSTS dataset)",
       "relevance": "Graduate-level mathematical capabilities evaluation; related prior work on hard math benchmarks"
     },
     {
       "title": "Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks",
       "relevance": "Used to contextualize FrontierMath's ~10% error rate estimate against ImageNet's >6% label error rate"
     },
     {
       "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
       "relevance": "Motivates FrontierMath's contamination-prevention design by documenting prevalence of contamination in existing benchmarks"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly useful to AI labs and researchers needing a non-saturated math evaluation, but access is gated through contact with Epoch AI."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The <2% solve rate for frontier models including o1-preview is striking given recent AI progress on competition math, and challenges optimistic narratives about AI mathematical capability."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Frames advanced mathematical reasoning as a key capability milestone for AI, with implications for AI's potential contributions to scientific research, but does not raise direct safety concerns."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Features quotes from three Fields Medalists (Tao, Gowers, Borcherds) and the OpenAI funding/evaluation conflict adds tension; Tao's benchmark contributions while also being an expert evaluator adds color."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Five sample problems with solutions are publicly released and transcripts are downloadable, but the full benchmark requires institutional access."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Epoch AI is a known AI forecasting organization; OpenAI's support is acknowledged; Fields Medalists Terence Tao, Timothy Gowers, and Richard Borcherds are quoted; all major frontier model families are evaluated."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45873709",
         "title": "The Drain of Scientific Publishing",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45873709",
         "created_at": "2025-11-10T08:21:43Z"
       },
       {
         "hn_id": "39032813",
         "title": "Adapting Standard Retrieval Benchmarks to Evaluate Generated Answers",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39032813",
         "created_at": "2024-01-17T20:13:37Z"
       },
       {
         "hn_id": "25074836",
         "title": "Principles of Quantum Communication Theory: A Modern Approach",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=25074836",
         "created_at": "2020-11-12T20:53:00Z"
       }
     ],
     "top_points": 1,
     "total_points": 3,
     "total_comments": 0
   }
 }