ai-research-survey

scan.json (33058B)

---

 {
   "paper": {
     "title": "RealMath: A Continuous Benchmark for Evaluating Language Models on Research-Level Mathematics",
     "authors": [
       "Jie Zhang",
       "Cezara Petrui",
       "Kristina Nikolić",
       "Florian Tramèr"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2505.12575",
     "doi": "10.48550/arXiv.2505.12575"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RealMath is a continuously refreshable benchmark of 1,200+ QA pairs extracted from arXiv papers and Math Stack Exchange, testing LLMs on research-level mathematics with verifiable fixed answers. Frontier models (o3 at 49.1% on Math.arXiv, 70.7% on StackExchange) perform surprisingly well on research math compared to competition benchmarks like FrontierMath (~2%), suggesting LLMs may already serve as useful mathematical assistants. However, 62% of Math.arXiv questions are classified as hard, where o3 drops to 27.9%. Fine-tuning GPT-4o-mini on 500 samples yielded no improvement, suggesting the difficulty stems from missing mathematical knowledge rather than distribution mismatch.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The code and dataset for REALMATH are publicly available' with footnote links to GitHub (code) and Huggingface (dataset)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset is released on Huggingface. The NeurIPS checklist confirms 'We provide access to all three datasets and the code used for their retrieval.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, or dependency list) is provided in the paper. The NeurIPS checklist answer on compute resources says 'Our experiments all involve querying APIs' but no environment details are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for theorem filtering (Appendix A.1), QA generation (Appendix A.1), and evaluation (Appendix A.2) are provided. The pipeline steps are detailed in Section 3.2 with code released on GitHub."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 3 report only point estimates (e.g., '49.1', '43.4') with no confidence intervals or error bars. The NeurIPS checklist question #7 answers 'No' and states results are 'mainly to illustrate the difficulty composition of our dataset.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used anywhere. Claims like 'o3 achieved the highest accuracy' are based solely on comparing raw numbers without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports absolute accuracy for 9 models across 3 datasets, providing full comparative context (e.g., o3 at 49.1% vs GPT-4o-mini at 12.5% on Math.arXiv). Context ablation reports 21.6% vs 42.3%. Effect magnitudes are clear from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes (633 Math.arXiv, 111 CS.arXiv, 542 Math.StackExchange) are reported but not justified. No power analysis or justification for why these sample sizes are adequate, particularly for the small CS.arXiv set of 111 examples."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results are single-run numbers with no variance. Only the fine-tuning experiment (Appendix B.1) reports mean and standard deviation over 5 runs. All other results in Table 3 and figures lack any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine models are compared including older ones as baselines: GPT-4o-mini, Llama-3.1-405B, Claude 3.5 Sonnet alongside frontier models o3, o4-mini, Gemini 2.5 Pro, etc. (Table 3, Section 4.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary frontier models released in 2024-2025: o3, o4-mini, Gemini 2.5 Pro, Grok-3, DeepSeek-R1, Claude 3.7 Sonnet (Figure 1 shows release dates)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'How important is a theorem's context?' ablates the context component, showing o4-mini drops from 42.3% to 21.6% on CS.arXiv without context. The fine-tuning analysis (Section 4.3, Appendix B.1) tests whether fine-tuning improves performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (exact match) is used as the evaluation metric throughout. While breakdowns by difficulty and category provide different views, the underlying metric is always accuracy percentage."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via exact-match checking with an LLM judge (GPT-4o, Appendix A.2). Human review was used only for dataset curation (~6% filtered), not for evaluating model outputs. Given the paper's broader 'mathematical assistant' framing, human evaluation of model reasoning quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For main evaluation, models are evaluated zero-shot on the benchmark (no training on this data). For the fine-tuning experiment, the paper splits into 500 training and 133 test samples (Appendix B.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns by math category (Figure 5, Table 4, Appendix B.3-B.4) and by difficulty level (Figures 4, 12, 13). Category-level accuracy varies from 23.8% (cs.LG) to 64.1% (math.NT) for o3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'Analyzing LLM's mistakes' provides an error breakdown (Figure 6) categorizing failures into reasoning errors, conceptual misunderstanding, missed insights, calculation errors, hallucinations, etc. Figure 3 also shows filtered-out examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The fine-tuning experiment (Section 4.3, Appendix B.1) explicitly reports 'Surprisingly, fine-tuning did not lead to an improvement in accuracy' with GPT-4o-mini going from 12.3% to 10.8% (Figure 10)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 'surprising capabilities in handling research mathematics' are supported by Table 3 (o3 at 49.1% vs ~2% on FrontierMath). Claims about 'continually refreshable dataset' are supported by the pipeline description (Section 3.2). The hedge 'may already serve as valuable assistants' is appropriately tentative."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are modest and appropriately supported. The fine-tuning claim ('fine-tuning did not lead to improvement') uses a controlled experiment. The context ablation (with/without context) is a single-variable manipulation. Language is generally hedged ('may', 'suggests')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Research-Level Mathematics' broadly, but the benchmark tests only constructive theorems with fixed numerical/symbolic answers — excluding proofs, inequalities, and non-constructive results. Results are dominated by combinatorics (28.9%) and number theory (10.2%) per Table 4. The conclusion claims 'current frontier models demonstrate surprisingly strong capabilities in research mathematics' without adequately bounding this to the specific subset tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses alternative explanations for the temporal performance pattern (easier recent math vs. training data recency). The paper also considers whether high performance reflects benchmark selection bias. The label noise discussion considers both source quality and LLM judge limitations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes: 'Although our approach focuses on statement verification rather than proof generation or verification, it nonetheless provides a valuable signal about LLMs' potential as mathematical assistants.' They acknowledge the gap between their measurement (fixed-answer accuracy) and the broader framing (mathematical assistant utility)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names only: 'o3', 'o4-mini', 'Gemini 2.5-pro', 'Claude 3.7 Sonnet', 'Grok-3', 'DeepSeek-R1', etc. No API versions, snapshot dates, or specific model identifiers (e.g., 'gpt-4o-mini-2024-07-18') are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts for theorem filtering (SYSTEM_PROMPT_THEOREM) and QA generation (SYSTEM_PROMPT_GENERATE_QA) are provided in Appendix A.1. The evaluation prompt is provided in Appendix A.2. For the evaluated models, the input format is described (context + question)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the 9 evaluated models or the judge model (o3-mini). The NeurIPS checklist answer on compute resources does not mention hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive direct prompts with context and questions; the evaluation is straightforward prompt-response without tools, retry logic, or multi-step reasoning scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Table 1 detail the full pipeline with counts at each stage: 4,000 papers → 3,922 LaTeX → 14,747 theorems → 407 constructive → 401 QA pairs → 280 after filtering. Filtering criteria are explained at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations section. The 'Discussion on label noise' subsection (Section 3.2) addresses data quality limitations only. Limitations are scattered across the paper (conclusion, NeurIPS checklist) but there is no substantive, consolidated limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed include: LLM judge reliability (~6% low-quality samples passed filtering), label noise from Stack Exchange, the assumption that arXiv theorems are correct (not peer-reviewed), and pipeline selection bias toward combinatorics and number theory (Figure 9, Table 4)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes it covers 'statement verification rather than proof generation' and excludes inequalities/non-constructive results, it does not explicitly state what the results do NOT show. The paper argues through these boundaries rather than treating them as genuine scope limitations: 'it nonetheless provides a valuable signal about LLMs' potential as mathematical assistants.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset is released on Huggingface with code on GitHub, enabling independent verification of the QA pairs and pipeline."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the collection in detail: querying arXiv API for math papers in specified time windows, downloading LaTeX sources, extracting theorems via LLM, filtering, generating QA pairs. Table 2 gives time spans and counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are public repositories (arXiv papers, Math Stack Exchange posts)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the full pipeline with counts at each stage (4,000 → 3,922 → 14,747 → 407 → 401 → 280). Each filtering criterion is explained. Appendix A.5 documents the Stack Exchange pipeline variant. Appendix A.3 documents the manual review stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6: 'J.Z. is funded by the Swiss National Science Foundation (SNSF) project grant 214838. K.N. is supported by an ETH AI Center Doctoral Fellowship.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with ETH Zurich. They evaluate third-party models (OpenAI, Google, Meta, Anthropic, xAI, DeepSeek) with no direct conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "SNSF and ETH AI Center are academic funders with no financial stake in the benchmark outcomes or the performance of evaluated LLMs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 references 'cutoff dates of GPT-4o-mini, Llama-3.1-405B, and Claude-3.5-Sonnet' and Figure 5 shows before/after cutoff splits, but the actual training cutoff dates for these models (and the other 6 evaluated models) are not explicitly stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'Measuring the impact of data contamination' explicitly analyzes potential overlap by splitting Math.arXiv into papers published before vs. after model training cutoffs (2022 vs. 2025) and comparing accuracy."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination resistance is a core design feature. Section 3.1 criterion 3 addresses 'Continuous acquisition' to prevent contamination. Section 4.3 empirically tests contamination via temporal splits. The pipeline is designed to generate fresh data from new papers after each model release."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on a benchmark constructed from public mathematical sources."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. NeurIPS checklist #14-15 confirm: 'This empirical study does not include any human subjects.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "NeurIPS checklist #8 acknowledges 'querying APIs comes with a cost' but no actual API costs, tokens consumed, or wall-clock times are reported for any of the 9 models evaluated across 3 datasets."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget or API spend is stated. NeurIPS checklist #8 answers 'NA' claiming experiments 'do not require computational resources' but this ignores the substantial API costs of evaluating 9 models on 1,200+ questions."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main evaluation results are single-run with no seed sensitivity analysis. Only the fine-tuning experiment (Appendix B.1) reports results across 5 runs. For API-based evaluation, temperature/sampling stochasticity could affect results but is not analyzed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for main experiments is not stated, implying single-run results. Only the fine-tuning experiment explicitly states 'we repeat the evaluation five times' (Appendix B.1)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported for the main evaluation or the fine-tuning experiment. For the fine-tuning, the paper says only that they used 'OpenAI's fine-tuning service' without reporting any hyperparameter settings or search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "No configuration cherry-picking is evident. All 9 models are evaluated on the full dataset with results reported for each. The fine-tuning experiment uses 500 random samples. Results are reported transparently including the negative fine-tuning finding."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in the paper, so multiple comparison correction is not applicable. The absence of statistical testing is captured by the significance_tests item."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors construct the benchmark, select the evaluation pipeline, and report results. No discussion of author-evaluation bias or potential systematic biases in their pipeline design (e.g., using o3-mini as judge may systematically favor or disfavor certain model families)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different compute costs are compared directly (e.g., GPT-4o-mini vs o3) without any discussion of compute budgets, inference costs, or performance-per-dollar comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 and 3.1 explicitly discuss what the benchmark measures vs. what it claims: 'our approach focuses on statement verification rather than proof generation.' The paper compares with competition benchmarks (FrontierMath, AIME) and discusses what different performance patterns mean for the construct being measured."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models receive direct prompts with context and questions; evaluation is straightforward prompt-response."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly analyzes temporal leakage by comparing model accuracy on Math.arXiv papers published before vs. after model training cutoffs (Figure 5 right panel). This is a core design feature of the continually refreshable pipeline."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline's filtering step #5 explicitly removes cases where 'the answer is obvious from the context fed to the LLM (it is common, for instance, for the paper's introduction to state a theorem's result informally).' Section 4.3 also tests the effect of context vs. no context."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether questions drawn from the same paper, same research group, or closely related subfields create non-independence. Multiple theorems can come from the same paper, potentially inflating effective sample size for some categories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splitting is used as a concrete leakage detection method: papers from before and after model training cutoffs are compared (Section 4.3, Figure 5). The refreshable pipeline design itself serves as a contamination prevention mechanism."
       }
     }
   },
   "claims": [
     {
       "claim": "Frontier LLMs achieve surprisingly strong performance on research-level mathematics, with o3 at 49.1% on Math.arXiv, 44.1% on CS.arXiv, and 70.7% on Math.StackExchange.",
       "evidence": "Table 3 reports accuracy for 9 models across 3 datasets (Section 4.2). Performance is compared contextually with near-zero results on FrontierMath.",
       "supported": "strong"
     },
     {
       "claim": "Hard questions remain very challenging for current LLMs, with o3 achieving only 27.9% on the hard subset of Math.arXiv.",
       "evidence": "Figure 4 shows accuracy breakdown by difficulty level. 62.2% of Math.arXiv questions are classified as hard based on weak-model performance (Section 4.2, Appendix B.2).",
       "supported": "strong"
     },
     {
       "claim": "LLM capabilities vary significantly across mathematical domains, with model-specific strengths (o3 excels at theoretical topics, Gemini 2.5-pro at applied ones).",
       "evidence": "Figure 5 shows per-category accuracy. o3 ranges from 23.8% (cs.LG) to 64.1% (math.NT). Gemini 2.5-pro's best categories (cs.LG at 61.9%, math.OC at 56.0%) are o3's worst (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning GPT-4o-mini on 500 RealMath samples does not improve accuracy, suggesting difficulty stems from missing mathematical knowledge rather than distribution mismatch.",
       "evidence": "Appendix B.1, Figure 10: fine-tuned GPT-4o-mini achieves 10.8% vs. 12.3% original (5 runs, mean reported). Training loss decreases but test loss fluctuates (Figure 11).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform reasonably well even without theorem context, suggesting many theorems use concepts that can be understood in isolation.",
       "evidence": "Section 4.3: o4-mini achieves 21.6% without context vs. 42.3% with context on CS.arXiv. Examples of context-free solvability are shown in Figure 7.",
       "supported": "strong"
     },
     {
       "claim": "The benchmark pipeline can be continuously refreshed with 70+ new samples per month, mitigating data contamination risks.",
       "evidence": "Section 3.2 describes the automated pipeline. Table 2 shows data spans across multiple time windows. Section 4.3 provides empirical contamination analysis.",
       "supported": "moderate"
     },
     {
       "claim": "Models perform better on newer (post-cutoff) samples than older ones, possibly because LLMs are more familiar with recent mathematical topics rather than due to research becoming easier.",
       "evidence": "Figure 5 right panel shows GPT-4o-mini at 7.9% before cutoff vs. 16.1% after, Llama-3.1-405B at 13.9% vs. 18.4%, Claude 3.5 Sonnet at 15.7% vs. 20.4% (Section 4.3).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars on main results",
       "detail": "All main results (Table 3, Figures 4-5) are single-run accuracy numbers without confidence intervals, error bars, or variance. The NeurIPS checklist explicitly answers 'No' to the error bars question. Given API stochasticity and small dataset sizes (especially CS.arXiv with 111 samples), single-run results may not be stable."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "The pipeline uses o3-mini to filter theorems, generate QA pairs, and classify difficulty, then uses GPT-4o to judge answer correctness. Using LLMs to both construct and evaluate a benchmark for LLMs creates potential systematic biases — questions that LLMs can process well during construction may also be ones they perform well on during evaluation."
     },
     {
       "flag": "Missing model versions and hyperparameters",
       "detail": "No API versions, snapshot dates, or sampling parameters (temperature, top-p) are reported for any of the 9 evaluated models. Results may not be reproducible given that model behavior changes across API versions."
     },
     {
       "flag": "Sampling bias toward constructive theorems",
       "detail": "The pipeline filters for constructive theorems with fixed numerical/symbolic answers — only 407 of 14,747 theorems (2.8%) pass filtering. This heavily biased subset (dominated by combinatorics at 28.9%) is used to make claims about 'research-level mathematics' broadly. The paper's claim that LLMs 'may already serve as valuable assistants for working mathematicians' extrapolates well beyond the tested subset."
     }
   ],
   "cited_papers": [
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Key benchmark for evaluating LLMs on real-world software engineering tasks, cited as parallel example of 'in the wild' evaluation."
     },
     {
       "title": "BaxBench: Can LLMs generate correct and secure backends?",
       "authors": ["Mark Vero", "Niels Mündler", "Victor Chibotaru", "Veselin Raychev", "Maximilian Baader", "Nikola Jovanović", "Jingxuan He", "Martin Vechev"],
       "year": 2025,
       "relevance": "Benchmark evaluating LLMs on real-world backend code generation, cited as example of practical capability evaluation."
     },
     {
       "title": "Swe-lancer: Can frontier LLMs earn 1 million from real-world freelance software engineering?",
       "authors": ["Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke"],
       "year": 2025,
       "relevance": "Evaluates LLM capabilities on real freelance software engineering tasks, cited as parallel real-world evaluation benchmark."
     },
     {
       "title": "FrontierMath: A benchmark for evaluating advanced mathematical reasoning in AI",
       "authors": ["Elliot Glazer", "Ege Erdil", "Tamay Besiroglu"],
       "year": 2024,
       "relevance": "Expert-crafted benchmark of extremely challenging math problems where models score ~2%; primary comparison point for RealMath's research-level approach."
     },
     {
       "title": "Humanity's last exam",
       "authors": ["Long Phan", "Alice Gatti", "Ziwen Han"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Extremely challenging expert-curated evaluation benchmark, cited as comparison for research-level difficulty evaluation."
     },
     {
       "title": "MathConstruct: Challenging LLM reasoning with constructive proofs",
       "authors": ["Mislav Balunović", "Jasper Dekoninck", "Nikola Jovanović", "Ivo Petrov", "Martin Vechev"],
       "year": 2025,
       "arxiv_id": "2502.10197",
       "relevance": "Related benchmark using constructive mathematical proofs to evaluate LLM mathematical reasoning."
     },
     {
       "title": "Training verifiers to solve math word problems (GSM8K)",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Foundational math reasoning benchmark for LLMs, cited as example of basic-level evaluation approaching saturation."
     },
     {
       "title": "Measuring mathematical problem solving with the MATH dataset",
       "authors": ["Dan Hendrycks", "Collin Burns", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Standard competition-level math benchmark for LLMs, cited as example of evaluation approaching saturation."
     },
     {
       "title": "LeanDojo: Theorem proving with retrieval-augmented language models",
       "authors": ["Kaiyu Yang", "Aidan M Swope", "Alex Gu"],
       "year": 2023,
       "relevance": "Benchmark for LLM-based formal theorem proving, representing the formal verification approach to evaluating mathematical capabilities."
     },
     {
       "title": "Proof or bluff? Evaluating LLMs on 2025 USA math olympiad",
       "authors": ["Ivo Petrov", "Jasper Dekoninck", "Lyuben Baltadzhiev"],
       "year": 2025,
       "arxiv_id": "2503.21934",
       "relevance": "Evaluates LLMs on contemporary math olympiad problems, addressing competition-level mathematical reasoning."
     },
     {
       "title": "LLM agents can autonomously exploit one-day vulnerabilities",
       "authors": ["Richard Fang", "Rohan Bindu", "Akul Gupta", "Daniel Kang"],
       "year": 2024,
       "relevance": "Example of evaluating LLM capabilities on real-world exploits rather than curated competitions, parallel to RealMath's 'in the wild' philosophy."
     },
     {
       "title": "AutoAdvExBench: Benchmarking autonomous exploitation of adversarial example defenses",
       "authors": ["Nicholas Carlini", "Javier Rando", "Edoardo Debenedetti", "Milad Nasr", "Florian Tramèr"],
       "year": 2025,
       "relevance": "Benchmark for evaluating real LLM exploit capabilities, cited as example of 'in the wild' capability evaluation over curated proxies."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Primarily a benchmark/evaluation paper useful for researchers studying LLM math capabilities, not a directly usable tool for practitioners."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the narrative that LLMs are poor at research-level math by showing 49% accuracy on real theorems, contrasting with ~2% on FrontierMath."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; purely evaluates mathematical reasoning capabilities."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild implicit critique of competition-focused benchmarks and FrontierMath's private test set, but framed diplomatically."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Dataset on Huggingface and code on GitHub allow anyone to run evaluations, though requires API access to frontier models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "ETH Zurich is well-known in academia; evaluates prominent models (o3, Gemini, Claude) but the paper itself is not from a major AI lab."
     }
   }
 }