ai-research-survey

scan.json (32026B)

---

 {
   "paper": {
     "title": "FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI",
     "authors": [
       "Elliot Glazer",
       "Ege Erdil",
       "Tamay Besiroglu",
       "Diego Chicharro",
       "Evan Chen",
       "Alex Gunning",
       "Caroline Falkman Olsson",
       "Jean-Stanislas Denain",
       "Anson Ho",
       "Emily de Oliveira Santos",
       "Olli Järviniemi",
       "Matthew Barnett",
       "Robert Sandler",
       "Matej Vrzala",
       "Jaime Sevilla"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2411.04872"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark and evaluation framework are deliberately kept private to prevent data contamination. GitHub links are provided only for helper code for two sample problems (artin_code, finite_field_implementation), not for the benchmark or evaluation system itself."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark problems are deliberately withheld to prevent contamination. Only 5 sample problems are publicly released (Section A). The paper states teams should 'reach out to math_evals@epochai.org' for evaluation access."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses Python with pickle and SymPy for verification (Section 2.2, Figure 3), but no requirements.txt, Dockerfile, or detailed dependency listing is provided. The exact Python version and library versions are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The benchmark is private, so external researchers cannot reproduce the evaluation without contacting the authors."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6 shows 'mean accuracy across 8 runs' but no confidence intervals, error bars, or ± values are reported. Section 4.2.1 notes that 'the precise ordering of model performance should be interpreted with significant caution' but provides no quantified uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Model comparisons are made by comparing raw performance numbers without any hypothesis testing. The paper acknowledges ranking instability but does not test whether observed differences are statistically significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports absolute performance numbers (<2% solve rate) and compares across benchmarks (Figure 2) but does not report formal effect sizes (Cohen's d, odds ratios, etc.). The comparisons are descriptive rather than quantified with effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 'hundreds' of problems but no justification is given for this number. The repeated trials use 5 runs per model per problem on 4 selected problems, and 8 runs for the full benchmark, with no power analysis or formal justification for these sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While 8 runs were conducted (Figure 6) and 5 repeated trials on 4 problems (Table 2), no standard deviations, IQR, or other spread measures are reported. Table 2 shows success rates as percentages but without variance across the 5 trials. The paper mentions 'high variability across runs' qualitatively without quantifying it."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares FrontierMath performance against multiple existing benchmarks (GSM8K, MATH, AIME, Omni-MATH, MathVista, MMLU College Math) in Figure 2, and evaluates 6 different models against each other in Figure 6 and Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include o1-preview, o1-mini, GPT-4o (2024-08-06), Claude 3.5 Sonnet (2024-10-22), Grok 2 Beta, and Gemini 1.5 Pro 002 — all state-of-the-art at the time of evaluation (Section 4.2.1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The evaluation framework has key design choices (10,000 token limit, code execution capability, prompt format) that are not systematically varied. Section 8 mentions plans to 'test the effects of increasing the token limit' as future work."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1 accuracy (Figure 6), pass@8 accuracy (Figure 9, Appendix B.4), number of responses per problem, token usage per model, and per-problem success rates across repeated trials (Table 2, Section 4.2.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Model evaluation is entirely automated via verification scripts (Section 2.2). The mathematician interviews (Section 6) evaluate the benchmark's difficulty, not the models' outputs. The paper notes 'even when a model obtained the correct answer, this does not mean that its reasoning was correct' but does not systematically evaluate reasoning quality via human review."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire FrontierMath benchmark consists of novel, unpublished problems (Section 2.4) that were not used for any development or tuning purpose. The benchmark is the held-out test set by design."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Despite rich MSC2020 classification of problems (Table 1, Figure 4) and difficulty ratings across three dimensions (Section 2.5), no per-category or per-difficulty-level performance breakdown is provided. The <2% solve rate is reported only as an aggregate."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 discusses that models often hit token limits, that o1-preview and Gemini 'typically submit a final answer before seeing any experimental results' despite being told to experiment, and that a correct answer was obtained through guessing rather than mathematical understanding. Section 4.2.2 analyzes token usage and response patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is a negative result — models solve <2% of problems. The paper reports that models fail to use the code execution framework effectively (Section 4.2.2), and that correct answers sometimes come from guessing rather than understanding (Section 4.2.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'current state-of-the-art AI models solve under 2% of problems' — supported by Figure 6 and Section 4.2.1. 'Exceptionally challenging mathematics problems crafted and vetted by expert mathematicians' — supported by Section 2, Section 6 (interviews). 'Cover most major branches of modern mathematics' — supported by Table 1 and Figure 4."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive claims about benchmark difficulty and model performance levels, not causal claims. It does not claim that any intervention causes improved performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Evaluating Advanced Mathematical Reasoning in AI' and abstract phrase 'revealing a vast gap between AI capabilities and the prowess of the mathematical community' generalize from 6 specific models to 'AI' broadly. While the tested models were state-of-the-art at the time, the claim extends beyond the tested set without explicit bounding."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses several alternative factors: the numerical answer format excludes proof problems, the hours-scale difficulty doesn't capture months-long research, token limits constrain model performance. Section 4.2.1 notes that correct answers don't imply correct reasoning and that guessing strategies sometimes work."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between what it measures (ability to produce correct numerical answers) and broader mathematical reasoning, acknowledging in Section 7 that 'the practical focus on automatically verifiable and numerical answers excludes proof-writing and open-ended exploration, which are significant parts of modern math research.' Borcherds notes the problems 'aren't quite the same as coming up with original proofs' (Section 6)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 specifies: 'GPT-4o (2024-08-06 version)', 'Claude 3.5 Sonnet (2024-10-22 version)', 'Gemini 1.5 Pro 002', 'o1-preview', 'o1-mini', and 'Grok 2 Beta' with citations to the specific model releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 provides the complete text of the initial prompt, continuation prompt, and final prompt used in evaluation. These are full verbatim prompts, not summaries."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The token limit is stated as 10,000 (Section 4.1), but temperature, top-p, and other sampling parameters for the evaluated models are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Figure 5 describe the evaluation framework in detail: models can write Python code, receive execution results, iterate multiple times, and submit final answers via pickle format. The interaction loop, feedback mechanism, and answer submission protocol are clearly documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data collection pipeline is documented in Sections 2.1-2.5: problem creation guidelines (originality, verifiability, guessproofness, computational tractability), peer review process, originality checks (plagiarism detection), difficulty rating system, and validation procedures."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) contains substantive discussion of limitations: numerical answer format excludes proofs, difficulty limited to hours not months, models too weak for fine-grained comparison, and the practical constraints of automated verification."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 quantifies a specific error rate (~10% estimated) with detailed analysis of 35 second-reviewed problems. Section 7 discusses specific threats: exclusion of proof problems, time-scale limitations, and that the <2% success rate 'temporarily limits FrontierMath's usefulness in evaluating relative performance of models.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'we cannot include problems that require mathematical proofs or formal reasoning steps,' 'they still fall short of typical mathematical research, which often spans weeks, months or even years,' and that current model weakness limits relative performance evaluation. These are specific statements about what FrontierMath does NOT test."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark problems are deliberately kept private to prevent data contamination. Only 5 sample problems are released publicly (Section A). Teams must contact the authors for evaluation access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the collection process in detail: collaboration with 60+ mathematicians, guidelines for problem creation, four requirements (originality, verifiability, guessproofness, computational tractability), submission metadata, and peer review workflow."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes who the mathematicians are (60+ from universities in 12+ countries, 14 IMO gold medals, graduate students to faculty) but does not describe how they were recruited — through what channels, invitation process, or whether the recruitment method could bias the benchmark toward certain mathematical subfields."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: problem creation with guidelines → metadata submission → blind peer review → revision of issues → second review for subset (35 problems) → originality/plagiarism checks. Section 2.3 details types of issues found and error rate estimation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges 'OpenAI for their support in creating the benchmark' and notes one position 'supported by SwissMAP' (ETH Zurich). Various individuals are thanked in the acknowledgments."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed (Epoch AI, various universities). The paper notes that Evan Chen is both a co-author and interviewee (Section 6), and that Terence Tao contributed problems and is interviewed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI supported the benchmark creation and OpenAI's models (o1-preview, o1-mini, GPT-4o) are among those evaluated. OpenAI has a commercial interest in AI evaluation and the perception of their models' capabilities. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Epoch AI is a research organization that may have interests in AI evaluation as a field, but no formal declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper extensively discusses contamination prevention by using novel problems, it does not state the training data cutoff dates for any of the 6 evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is the core innovation. Section 2.4 describes extensive measures: problems are exclusively new and unpublished, submitted through encrypted channels, checked with plagiarism detection tools (Quetext, Copyscape), and reviewed by expert mathematicians for originality."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination prevention is a primary contribution. Section 2.4 details: novel unpublished problems, encrypted communication channels, plagiarism detection tools, expert review for originality. Section 1 frames contamination as a key motivation for the benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The mathematicians are problem contributors and interviewees, not experimental participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Mathematician collaborators and interviewees are not experimental participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Mathematician contributors are described by affiliation and expertise (Section 2.1) but this is collaborator description, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2.2 reports token usage (6,000-17,000 tokens per question) and percentage of questions hitting token limits, but does not report API costs in dollars, wall-clock evaluation time, or per-problem latency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, GPU hours, or overall evaluation time for the benchmark runs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Eight runs were conducted on the full benchmark (Figure 6) and 5 repeated trials on 4 problems (Table 2), showing variability exists. However, no standard deviations or quantified variance across runs is reported — only mean accuracy and per-problem success rates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 explicitly states '8 runs' for the full benchmark evaluation and 'five runs per model per problem' for the detailed repeated trials on 4 problems."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper sets a 10,000 token limit and uses specific prompts but does not report whether these were tuned or what alternatives were considered. Section 8 mentions future plans to 'test the effects of increasing the token limit,' implying the current choice was not validated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 10,000 token limit and prompt design choices are not justified with systematic comparison. No explanation is given for why this specific configuration was chosen over alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluate third-party models on it. The potential bias of benchmark creators evaluating on their own benchmark (e.g., problem selection favoring difficulty patterns that disadvantage certain model types) is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2.2 notes substantial differences in token usage between models (6,000 to 17,000 tokens per question) but does not analyze whether these compute differences affect performance or present performance-compute tradeoff curves."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 features interviews with three Fields Medalists and an IMO coach who assess the benchmark's validity. Section 7 acknowledges the gap between numerical answers and mathematical research: 'we cannot include problems that require mathematical proofs.' Borcherds notes the problems 'aren't quite the same as coming up with original proofs.'"
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All models use the same Python code execution framework, but Section 4.2.2 reveals very different interaction patterns (o1-preview: 1.29 responses, Grok: 3.81 responses; token limit hit rates vary from 16.8% to >45%). This suggests the scaffold may advantage certain models, but the confound is not explicitly addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark design addresses temporal leakage: all problems are 'exclusively new, previously unpublished' (Section 2.4), created specifically for this benchmark after model training. This is the primary anti-contamination strategy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak answer information. The prompt structure (Appendix B.1), MSC classification tags, or problem format conventions are not analyzed for potential information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether structural similarities between benchmark problems (e.g., shared mathematical techniques, similar problem formats) could allow models to transfer knowledge between problems within the benchmark."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes concrete methods: Quetext and Copyscape plagiarism detection tools were run on all problem statements, expert review verified originality against 'popular mathematics websites, online repositories, and academic publications,' and problems were handled through encrypted channels."
       }
     }
   },
   "claims": [
     {
       "claim": "Current state-of-the-art AI models solve under 2% of FrontierMath problems",
       "evidence": "Figure 6 and Section 4.2.1 show all 6 evaluated models (o1-preview, o1-mini, GPT-4o, Claude 3.5 Sonnet, Grok 2 Beta, Gemini 1.5 Pro 002) achieve less than 2% mean accuracy across 8 runs on the full benchmark.",
       "supported": "strong"
     },
     {
       "claim": "FrontierMath problems are exceptionally challenging, requiring hours from expert mathematicians",
       "evidence": "Section 6 reports interviews with Fields Medalists Tao, Gowers, and Borcherds, who 'unanimously characterized the problems as exceptionally challenging, requiring deep domain expertise and significant time investment.' Section 2.5 describes creativity and execution ratings in hours.",
       "supported": "strong"
     },
     {
       "claim": "The benchmark covers most major branches of modern mathematics",
       "evidence": "Table 1 shows distribution across 24+ MSC2020 classification codes. Figure 4 visualizes interconnections. The paper claims coverage of '70% of the top-level subjects in the MSC2020 classification.'",
       "supported": "strong"
     },
     {
       "claim": "FrontierMath is resistant to data contamination",
       "evidence": "Section 2.4 describes originality measures (encrypted channels, plagiarism detection, expert review). However, the paper acknowledges 'in some cases, standard email clients were used when communicating about a subset of the problems' (footnote 6).",
       "supported": "moderate"
     },
     {
       "claim": "The estimated benchmark error rate is approximately 10%",
       "evidence": "Section 2.3: 2 of 35 second-reviewed problems had incorrect answers, yielding a Bayesian posterior of ~6.9%. The paper estimates ~10% accounting for possible undetected errors.",
       "supported": "moderate"
     },
     {
       "claim": "FrontierMath maintains >98% unsolved rate compared to near-saturation of other benchmarks",
       "evidence": "Figure 2 compares FrontierMath (>98% unsolved) against GSM8K (~4% unsolved), MATH (~5% unsolved), AIME (~26% unsolved), and Omni-MATH (~40% unsolved).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "FrontierMath is a benchmark of hundreds of original, expert-crafted mathematics problems spanning most branches of modern mathematics, designed to resist data contamination through novel unpublished problems and automated verification. Current state-of-the-art AI models solve under 2% of problems, revealing a vast gap between AI capabilities and research-level mathematics. The benchmark features a ~10% estimated error rate in problems based on 35-problem second-review analysis. Fields Medalists interviewed characterized the problems as exceptionally challenging, requiring deep domain expertise beyond current AI capabilities.",
   "red_flags": [
     {
       "flag": "Funder conflict of interest",
       "detail": "OpenAI supported the benchmark creation ('We gratefully acknowledge OpenAI for their support') and three OpenAI models (o1-preview, o1-mini, GPT-4o) are among the six models evaluated. This conflict is not explicitly acknowledged."
     },
     {
       "flag": "Limited error rate validation",
       "detail": "Only 35 of hundreds of accepted problems received second-review scrutiny (Section 2.3). The ~10% estimated error rate is based on this small sample, and the paper acknowledges 'we must account for potential undetected errors that even the second review might have missed.'"
     },
     {
       "flag": "No statistical rigor in model comparisons",
       "detail": "With <2% solve rate on hundreds of problems, individual model successes have outsized impact on rankings. The paper acknowledges this but reports model comparisons without confidence intervals, significance tests, or variance measures despite running 8 trials."
     },
     {
       "flag": "No per-category performance breakdown",
       "detail": "Despite rich MSC2020 classification (Table 1, Figure 4) and three-dimensional difficulty ratings (Section 2.5), no performance breakdown by mathematical domain or difficulty level is provided. This hides whether the <2% is uniform or concentrated."
     }
   ],
   "cited_papers": [
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "GSM8K benchmark — a foundational LLM math evaluation dataset now approaching saturation."
     },
     {
       "title": "Measuring mathematical problem solving with the math dataset",
       "authors": ["Dan Hendrycks", "Collin Burns", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2103.03874",
       "relevance": "MATH dataset benchmark for mathematical reasoning, now near-saturated by frontier models."
     },
     {
       "title": "Investigating data contamination in modern benchmarks for large language models",
       "authors": ["Chunyuan Deng"],
       "year": 2023,
       "arxiv_id": "2311.09783",
       "relevance": "Directly addresses the benchmark contamination problem that FrontierMath aims to solve."
     },
     {
       "title": "Mathematical capabilities of chatgpt",
       "authors": ["Simon Frieder"],
       "year": 2024,
       "relevance": "Evaluates LLM mathematical capabilities through 709 graduate-level problems with human-expert evaluation."
     },
     {
       "title": "Omni-MATH: A Universal Olympiad Level Mathematic Benchmark For Large Language Models",
       "authors": ["Bofei Gao"],
       "year": 2024,
       "arxiv_id": "2410.07985",
       "relevance": "Olympiad-level math benchmark with 4,428 competition-sourced problems, representing the difficulty frontier before FrontierMath."
     },
     {
       "title": "Benchmark Data Contamination of Large Language Models: A Survey",
       "authors": ["Cheng Xu"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Survey of benchmark contamination in LLM evaluation, directly relevant to FrontierMath's anti-contamination design."
     },
     {
       "title": "Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks",
       "authors": ["Curtis G. Northcutt", "Anish Athalye", "Jonas Mueller"],
       "year": 2021,
       "arxiv_id": "2103.14749",
       "relevance": "Documents >6% label error rates in ML benchmarks including ImageNet, contextualizing FrontierMath's ~10% error estimate."
     },
     {
       "title": "Are We Done with MMLU?",
       "authors": ["Aryo Pradipta Gema"],
       "year": 2024,
       "arxiv_id": "2406.04127",
       "relevance": "Documents >9% error rate in MMLU benchmark, relevant to benchmark quality and evaluation reliability."
     },
     {
       "title": "Solving olympiad geometry without human demonstrations",
       "authors": ["Trieu H. Trinh"],
       "year": 2024,
       "relevance": "AlphaGeometry achieving olympiad-level geometry performance, representing the frontier of AI mathematical reasoning."
     },
     {
       "title": "Mathematical discoveries from program search with large language models",
       "authors": ["Bernardino Romera-Paredes"],
       "year": 2024,
       "relevance": "FunSearch demonstrating LLMs can make novel mathematical discoveries, relevant to AI's mathematical capability frontier."
     },
     {
       "title": "Olympiadbench: A challenging benchmark for promoting agi with olympiad-level bilingual multimodal scientific problems",
       "authors": ["Chaoqun He"],
       "year": 2024,
       "arxiv_id": "2402.14008",
       "relevance": "8,476 olympiad problems for AI evaluation, representing the prior difficulty standard FrontierMath aims to surpass."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark including college mathematics, now near-saturated — motivating harder benchmarks like FrontierMath."
     }
   ]
 }