ai-research-survey

scan.json (26536B)

---

 {
   "paper": {
     "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
     "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Chan Jun Shern", "Leon Maksin", "Rachel Dias", "Evan Mays", "Benjamin Kinsella", "Wyatt Thompson", "Johannes Heidecke", "Amelia Glaese", "Tejal Patwardhan"],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2504.01848",
     "doi": "10.48550/arXiv.2504.01848"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PaperBench evaluates AI agents' ability to replicate 20 ICML 2024 papers from scratch, with 8,316 individually gradable rubric tasks co-developed with original authors. Claude 3.5 Sonnet achieved the best score of 21.0% with BasicAgent, while o1 with IterativeAgent reached 24.4%. ML PhD human baselines achieved 41.4% (best@3) after 48 hours, outperforming agents at longer horizons. An LLM-based judge (o3-mini) achieves F1=0.83 on JudgeEval, enabling scalable automated grading.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We open-source our code to facilitate future research' and the conclusion reiterates open-sourcing PaperBench."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (20 papers with rubrics, addenda, JudgeEval gold labels) is released as part of the open-source package."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies Ubuntu 24.04 and A10 GPU for agent environments but does not provide a requirements.txt, Dockerfile, or detailed dependency list for reproducing the evaluation pipeline itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code is open-sourced but specific commands to run evaluations are not detailed in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4, 5, 6 report standard error of the mean (±) for all model results. Figure 3 includes error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Claude 3.5 Sonnet outperforms other models and that IterativeAgent boosts o1/o3-mini but provides no statistical significance tests — only point estimates with SEM."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute percentages with baselines (e.g., 21.0% vs 13.2% for Claude vs o1), providing sufficient context for effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "3 runs per paper per model is used without justification for why 3 runs is adequate given the high variance observed (Tables 10-18 show large per-paper variance)."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard error is reported across 3 runs per paper. Per-paper results in Appendix I show individual run scores and standard errors."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple frontier models compared (GPT-4o, o1, o3-mini, DeepSeek-R1, Claude 3.5 Sonnet, Gemini 2.0 Flash), plus a human PhD baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are frontier models current as of late 2024/early 2025. Human baseline uses current ML PhDs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "BasicAgent vs IterativeAgent comparison (Section 5.3) ablates scaffold design. PaperBench Code-Dev ablates the execution/result-match requirement. Results stratified by requirement type (Table 9)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Replication Score (overall), scores stratified by Code Development/Execution/Result Match requirement types (Table 9), and per-paper breakdowns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "JudgeEval (Section 4.2) validates the LLM judge against human expert judgments. Rubrics were co-developed with original paper authors."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper mentions maintaining 'a held-out set for internal use' (Section 3) and releases two NeurIPS papers as a development set, separate from the 20-paper test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-paper results in Appendix I (Tables 10-18), per-requirement-type breakdown in Table 9, and per-paper node counts in Tables 2 and 7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses specific failure modes: models finishing early, failing to strategize, o3-mini struggling with tool usage, models claiming partial reproduction was the goal."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "IterativeAgent hurts Claude 3.5 Sonnet performance (21.0% → 16.1%, Table 5). Many models score very poorly (<10%). Agents plateau after first hour (Figure 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (21.0% for Claude, human baseline outperforms models, F1=0.83 for judge) are all supported by Tables 4, 5, and 3 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.3 attributes score differences between BasicAgent and IterativeAgent to scaffold design ('prompt tuning used for IterativeAgent is differentially suited for OpenAI o-series models') without controlled ablation isolating the specific mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to 'replicating ML research papers' and acknowledges the 20-paper dataset limitation in Section 7. Claims about agent capabilities are carefully hedged."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses scaffold weakness vs model capability as alternative explanations. Section 7 discusses contamination. The paper notes results 'do not represent the upper limit of these models' capabilities.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (Replication Score = weighted proportion of rubric leaf nodes satisfied) and distinguishes this from broader ML R&D capability. Section 7 acknowledges the benchmark 'does not capture every aspect of real-world research.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions specified in Section 5.2 footnotes: gpt-4o-2024-08-06, o1-2024-12-17, o3-mini-2025-01-31, claude-3-5-sonnet-20241022, gemini-2.0-flash."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts for BasicAgent (Figure 10), IterativeAgent (Figures 11-12), task instructions (Figures 13-14), and judge prompts (Figures 7-9) are provided in the appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reasoning effort settings stated (o1, o3-mini use 'high'). Time limits (12h, 36h) specified. Reproduction runtime cap (12h) stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "BasicAgent and IterativeAgent scaffolds described in detail (Section 5.1, Appendix F), including tool-use loop, context management, file reader, web browser tools, and the 'end task' tool modification."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Paper selection pipeline documented in Appendix B with explicit filtering criteria (commercial, empirical, hardware, model dependency, data, reproducibility, framework, accessibility filters)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' provides substantive discussion of dataset size, contamination, challenging dataset creation, judge performance, and cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: contamination from pre-training on authors' codebases, non-deterministic judge, rubric creation difficulty, and specific cost figures ($400 per rollout, $66 per grading)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers only 20 ICML 2024 papers, acknowledges results 'do not represent the upper limit of these models' capabilities,' and that 'PaperBench does not capture every aspect of real-world research.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark, rubrics, and JudgeEval gold labels are open-sourced. Per-paper per-run results are provided in Appendix I."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Paper selection from ICML 2024 described in Appendix B with systematic filtering. Rubric creation process described in Appendix C. Human baseline recruitment described in Section 5.4."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and footnote 17 describe human baseline recruitment: ML PhDs from named universities (Berkeley, Cambridge, CMU, etc.), CV screen followed by ML and git technical test. 8 participants, assigned to papers they were most confident about."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: paper selection (Appendix B) → rubric creation (Appendix C) → agent execution (Section 5.1) → reproduction (Section 2.2) → grading (Section 2.3). JudgeEval construction described in Section 4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of funding sources. All authors are from OpenAI but no explicit funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as affiliated with OpenAI on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a financial interest in demonstrating model capabilities and in the benchmark ecosystem. The paper evaluates OpenAI models (o1, o3-mini, GPT-4o) alongside competitors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement. OpenAI employees evaluating OpenAI models constitutes an undisclosed conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the evaluated models. Section 7 acknowledges contamination risk but does not state cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses that original codebases exist online and models pre-trained on large corpora 'may have internalized solutions.' However, no empirical overlap analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 acknowledges contamination risk from pre-training on authors' codebases. Section 2.5 implements blacklists preventing agents from using original code repositories. Monitor checks for blacklist violations (Appendix E)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human baseline study (Section 5.4) was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics approval for the human baseline study involving 8 ML PhD participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 17 lists participant universities (Berkeley, Cambridge, CMU, Columbia, Cornell, Purdue, TU Wien, UMass Amherst). Qualification: ML PhDs with CV screen and technical test."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and footnote 17: participants must be enrolled in or have completed a PhD in ML, pass a CV screen, and pass an ML and git technical test."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "Participants were assigned to papers 'they were most confident about replicating' — self-selection rather than randomization, and this non-random assignment is not discussed as a limitation."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding described. Participants knew they were being compared to AI agents. Evaluators (the LLM judge) graded both human and agent submissions."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 describes the process: 3 independent attempts per paper, evaluated after one week, only the best performer extended for remaining weeks. One human attempt for test-time-model-adaptation ended at 24 hours (noted in Figure 3 caption)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7: ~$400 per o1 IterativeAgent 12-hour rollout, $66 per paper for o3-mini grading, $10 per paper for Code-Dev grading. Figure 5 plots judge cost. Section 4.1 estimates judge token usage."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Agent runtime limits (12h, 36h) stated. Single A10 GPU per run. Total cost estimate: $8000 per eval run for 20 papers. 646 total runs mentioned in Section 2.5."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "3 runs per paper per model reported in Appendix I (Tables 10-18), showing substantial variance across runs. The paper explicitly notes 'we see high variance in results on the same paper' and recommends 'several seeds when evaluating.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2: '3 runs per paper.' Footnotes note exceptions (2 runs for some configurations). Human baseline: 3 independent attempts per paper."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported for scaffold prompt tuning. Section F.1 mentions 'preliminary experiments' to adjust prompts but doesn't quantify the search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports results for BasicAgent and IterativeAgent but does not explain how many scaffold variants were tried before settling on these two, or what validation process was used."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple models compared across 20 papers with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI authors evaluate OpenAI models using OpenAI-developed scaffolding and judges, without acknowledging potential self-comparison bias. The scaffold was tuned on preliminary experiments which may favor OpenAI models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows performance over time (compute). Table 5 compares 12h vs 36h runs. The paper notes agents plateau after 1 hour."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A discusses future directions for rubric-based evaluation. Section 7 discusses limitations of the benchmark. The paper explicitly states it 'does not capture every aspect of real-world research' and discusses specification gaming risks (Appendix A.3)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2-5.3 evaluates two different scaffolds (BasicAgent, IterativeAgent) and finds large differences (Claude drops from 21% to 16.1%, o1 rises from 13.2% to 24.4%), explicitly noting 'models' sensitivities to prompting.'"
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses that ICML 2024 papers' codebases exist online and models may have internalized solutions. Papers selected from recent conference to mitigate this."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1: agents are not shown the rubric during attempts. Section 2.5: blacklists prevent access to original codebases. Monitor checks for violations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 20 papers share structural similarities that could affect results, or whether model pre-training on ML paper corpora creates systematic advantages."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5 and Appendix E describe a post-hoc monitor that checks agent logs for blacklisted URLs. 10 violations found and disqualified across 646 runs."
       }
     }
   },
   "claims": [
     {
       "claim": "Claude 3.5 Sonnet (New) with BasicAgent achieves the best average Replication Score of 21.0% on PaperBench.",
       "evidence": "Table 4 reports 21.0 ± 0.8% for Claude 3.5 Sonnet, highest among 6 models tested with BasicAgent.",
       "supported": "strong"
     },
     {
       "claim": "o1 with IterativeAgent achieves 24.4%, significantly outperforming its BasicAgent score of 13.2%.",
       "evidence": "Tables 4 and 5 show o1 BasicAgent at 13.2 ± 0.3% vs IterativeAgent at 24.4 ± 0.7%.",
       "supported": "strong"
     },
     {
       "claim": "Human ML PhD baseline (best@3) achieves 41.4% on a 3-paper subset after 48 hours, outperforming o1 at 26.6%.",
       "evidence": "Section 5.4 and Figure 3 show human-agent comparison over time. Humans surpass agents after 24 hours.",
       "supported": "moderate"
     },
     {
       "claim": "The LLM-based judge (o3-mini with SimpleJudge) achieves F1=0.83 on JudgeEval, making it a reasonable stand-in for human judges.",
       "evidence": "Table 3 reports F1=0.83 for o3-mini at $66/paper cost. Expert human performance is treated as ideal.",
       "supported": "moderate"
     },
     {
       "claim": "Models are proficient at writing code quickly but fail to effectively work beyond the first hour to improve submissions.",
       "evidence": "Figure 3 shows o1 scores plateauing after 1 hour. Table 9 shows Code Development scores far exceed Execution and Result Match scores.",
       "supported": "strong"
     },
     {
       "claim": "PaperBench Code-Dev is weakly correlated with full PaperBench (Pearson r=0.48).",
       "evidence": "Footnote 5 in Section 2.6 reports r=0.48 with PB = 0.45*PBCD + 0.05.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own products",
       "detail": "All authors are OpenAI employees. OpenAI models (o1, o3-mini, GPT-4o) are evaluated alongside competitors. The judge uses OpenAI models (o3-mini). The scaffold was tuned using preliminary experiments that may favor OpenAI models. No competing interests statement."
     },
     {
       "flag": "Small sample size for human baseline",
       "detail": "Only 8 human participants on 4 papers (3 attempts each). Best@3 metric on 3 papers is a very small sample for human-AI comparison claims. Participants self-selected papers they were 'most confident about,' potentially inflating human scores."
     },
     {
       "flag": "Non-random assignment in human study",
       "detail": "Human participants chose papers they were confident about replicating, introducing selection bias. Some received A100 GPUs instead of A10s. These are not discussed as threats to validity."
     },
     {
       "flag": "Small and potentially unrepresentative benchmark",
       "detail": "20 papers from ICML 2024 Spotlight/Oral only. Selection required author cooperation (42 reached, 20 agreed), introducing potential bias toward papers whose authors were willing to collaborate."
     },
     {
       "flag": "Judge evaluated by judge's own developer",
       "detail": "JudgeEval uses only 4-5 papers and is created by the same team that built SimpleJudge. The evaluation of the judge is limited in scope and potentially biased toward the judge's strengths."
     }
   ],
   "cited_papers": [
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark evaluating AI agents on Kaggle ML engineering tasks, directly related predecessor."
     },
     {
       "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark",
       "authors": ["Zachary S. Siegel", "Sayash Kapoor"],
       "year": 2024,
       "arxiv_id": "2409.11363",
       "relevance": "Agent benchmark for computational reproducibility of research papers given existing repositories."
     },
     {
       "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
       "authors": ["Hjalmar Wijk", "Tao Lin"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "Benchmark for AI R&D capabilities with human expert comparisons, directly comparable methodology."
     },
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Evaluates language agents on ML experimentation tasks including Kaggle competitions."
     },
     {
       "title": "Can LLMs Generate Novel Research Ideas? A Large-Scale Human Study with 100+ NLP Researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2024,
       "arxiv_id": "2409.04109",
       "relevance": "Large-scale study comparing LLM vs human research ideation capabilities."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundation agent architecture (ReAct) used as basis for PaperBench's BasicAgent scaffold."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "relevance": "LLM-as-judge methodology foundational to PaperBench's automated grading approach."
     },
     {
       "title": "AI Sandbagging: Language Models Can Strategically Underperform on Evaluations",
       "authors": ["Teun van der Weij"],
       "year": 2024,
       "arxiv_id": "2406.07358",
       "relevance": "Discusses strategic underperformance on evaluations, relevant to PaperBench's specification gaming concerns."
     },
     {
       "title": "Agent-as-a-Judge: Evaluate Agents with Agents",
       "authors": ["Mingchen Zhuge"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Agent-based judging framework for evaluating AI agent outputs on complex tasks."
     },
     {
       "title": "DSBench: How Far Are Data Science Agents to Becoming Data Science Experts?",
       "authors": ["Liqiang Jing"],
       "year": 2024,
       "arxiv_id": "2409.07703",
       "relevance": "Benchmark for data science agent capabilities, related evaluation methodology."
     }
   ]
 }