ai-research-survey

scan.json (31887B)

---

 {
   "paper": {
     "title": "REXBENCH: Can coding agents autonomously implement AI research extensions?",
     "authors": [
       "Nicholas Edwards",
       "Yukyung Lee",
       "Yujun (Audrey) Mao",
       "Yulu Qin",
       "Sebastian Schuster",
       "Najoung Kim"
     ],
     "year": 2025,
     "venue": "arXiv preprint (under review)",
     "arxiv_id": "2506.22598",
     "doi": "10.48550/arXiv.2506.22598"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Agent implementations released at https://github.com/tinlaboratory/RexBench and benchmark at https://rexbench.com/. Gold solutions deliberately held private in Bitbucket for evaluation integrity."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark tasks, extension instructions, and original codebases are publicly available at rexbench.com. Gold solutions are held out by design for fair evaluation, which is standard for benchmarks with leaderboards."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Each task has an environment.yml file with package versions (Section 3.2). Evaluation uses Apptainer containers with specified hardware (Table 2). 'We added version information for all of the packages (via an environment.yml file in the repository).'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 details the full evaluation pipeline: submission format (git patches), infrastructure setup (OpenStack VMs, Apptainer containers), and execution procedure. Appendix C describes agent configurations. The website hosts submission instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2: 'Error bars show standard error of the mean of all runs per model computed using the closed form formula (2σ, no normality assumption).' Figure 5 shows regression coefficients with 95% confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 uses a mixed-effects linear regression with significance tests (p-values reported: β = −0.036, p < 0.01 for line changes; p < .05 for repository popularity). However, agent-level performance comparisons (e.g., 'best performing agents achieving 25%') lack formal significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Regression coefficients with CIs reported in Figure 5 (e.g., β = −0.036 for line changes). Success rates are absolute values with context (e.g., '25% final success rate' out of 12 tasks × 3 runs)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 12 tasks or 3 runs per task were chosen. No power analysis. The paper acknowledges the benchmark is small but does not justify the specific sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "'We run each task three times with the same agent model to account for agent random variation.' Standard error bars are shown in Figure 2. Per-task results across 3 runs shown in Tables 8-16."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine agent configurations compared across 3 frameworks (aider, Claude Code, OpenHands) and 4 LLM backbones. Different configurations serve as baselines for each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models are recent: Claude 3.7 Sonnet, o1, o4-mini, DeepSeek R1. All frameworks are current: aider, Claude Code, OpenHands. The paper evaluates the state of the art in coding agents."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The hint experiment (Section 4.2) systematically ablates information provided: no hints → localization hints → step-by-step hints. This tests which capabilities (information finding vs. planning vs. implementation) are bottlenecks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics defined in Section 3.4: final success rate, execution success rate, and file recall. Additionally, cost and time efficiency are analyzed (Figure 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is fully automated via numerical match with gold solutions. Section 5.1 includes manual error analysis of top-2 agents' outputs, but this is post-hoc qualitative analysis, not a systematic human evaluation metric. The paper notes this limitation: 'the cause of failure is difficult to identify' without process-level metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Gold solutions are held in private Bitbucket repositories. Evaluation infrastructure is privately hosted. 'We store all the gold extensions in private Bitbucket repositories.' Agents have no access to evaluation scripts or reference solutions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-task breakdowns provided in Tables 8-16 for every agent-LLM-hint combination. Results also broken down by framework, LLM backbone, and hint level. Error distribution in Table 7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 extensively discusses explicit errors (empty diffs, Python errors, timeouts) and implicit errors (logic errors, value errors). Error distribution in Figure 7/Table 7. Debugging difficulty estimated per error."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Most results are negative — most agents achieve near-zero success. DeepSeek R1 achieves 0% across most configurations. The 'overthinking' failure mode is reported. Detailed hints sometimes hurt performance (e.g., Othello task drops from 100% to 0% with detailed hints)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: 'all agents evaluated fail to autonomously implement the majority of the extensions' (best is 25%), 'best performance under this setting remains below 40%' (39% with hints for OpenHands + Claude). All claims in the abstract are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The hint experiment (Section 4.2) is a controlled manipulation supporting causal claims about hint effects. The regression in Section 5.2 uses appropriate language ('significant negative effect') with a mixed-effects model including random effects for model identity. Ablation-style claims (hint levels) have adequate design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: 'with an initial focus on Natural Language Processing (NLP) and Machine Learning (ML)' (Section 1). Limitations section notes the setting is 'idealized' and more constrained than real research. Acknowledges 12 tasks is a starting point for a 'community-driven effort.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 discusses multiple explanations for failures: overthinking behavior, models varying in ability to implement different solution strategies, interaction between hints and model capabilities. Section 5.2 considers four possible sources of difficulty. Limitations discuss how controlled setting differs from real scenarios."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the proxy gap: 'Realistic research extensions tend to be quite open-ended, which makes automatic assessment challenging' (Section 3.1). They acknowledge that numerical match with gold solutions is a proxy for genuine research extension capability and discuss how the idealized setting differs from real research."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models identified by marketing names only: 'Claude 3.7 Sonnet', 'o1', 'o4-mini', 'DeepSeek R1'. No API versions, snapshot dates, or model IDs provided. The Claude system card is cited but no specific API version is used. These marketing names can map to different model checkpoints over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Agent prompts provided: Claude Code prompt is 'Read the instructions in instructions.md and carry out the specified task' plus 'Please do not execute any code' (Appendix C). OpenHands uses same prompt. Aider's two-stage approach described. Full task instruction example in Appendix A (WinoDict)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1: 'we set the temperature to 0.7 for Claude 3.7 Sonnet and DeepSeek-R1, and specified reasoning effort as medium for all OpenAI models.' Max retries (5) for aider, max steps (250) for OpenHands also stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows capabilities per framework (repo navigation, tool use, bash execution, Python execution). Appendix C provides detailed configuration for each framework. Appendix D analyzes OpenHands tool usage distribution (Figure 6). Agent interaction patterns described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the benchmark construction pipeline: code replication verification → gold implementation → instruction writing → multi-round revision for clarity and self-containment. Papers converted from PDF to markdown using PyMuPDF4LLM. Environment.yml files added."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Explicit 'Limitations and broader impacts' subsection at the end of Section 6 with substantive discussion of automatic evaluation constraints, idealized setting, and missing process-level metrics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: tasks are 'much more informative and clearer than an actual task a human researcher may face'; automatic evaluation constrains task open-endedness; process-level metrics like landmark evaluation are missing; risk of 'reward hacking or gamification of the benchmark.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Multiple explicit boundaries: 'primarily in the AI domain with a focus on topics aligning with the expertise of our team'; 12 tasks as 'a motivating start'; setting is more constrained than real research; agents were not given code execution capability; extensions are 'specifically-scoped questions' not open-ended exploration."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Agent log files and patch files from individual runs are not publicly released. The paper mentions requesting agent logs for submissions but does not state these are publicly available. Only aggregate results (tables, figures) are provided in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes benchmark construction in detail: domain experts verified codebase replication, implemented gold solutions, validated by co-authors, wrote instructions through multiple revision rounds. Section 4.1 describes experimental procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The benchmark consists of research extension tasks created by the author team. Task selection criteria are described in Section 3.2."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: Figure 1 shows end-to-end workflow. Section 3.5 details evaluation infrastructure (patch application → container execution → result collection → evaluation). Section 4.1 describes experimental procedure for running agents."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'This work was supported by funding from Good Ventures Foundation and Open Philanthropy awarded to NK and SS, from Google awarded to NK, and from WWTF through the project \"Understanding Language in Context\" (WWTF Vienna Research Group VRG23-007) awarded to SS.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations clearly listed: University College London, Boston University, University of Vienna. No authors are affiliated with the companies whose models are evaluated (Anthropic, OpenAI, DeepSeek)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Good Ventures/Open Philanthropy (foundation), WWTF (Austrian research fund), and Google. None have direct stake in whether specific agents succeed. Google's models are not evaluated. The research is conducted by academic researchers."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement in the paper. While there may be no conflicts, the absence of an explicit declaration is noted."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the four models evaluated (Claude 3.7 Sonnet, o1, o4-mini, DeepSeek R1). The paper addresses contamination through benchmark design (novel extensions) rather than by checking model training cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensively discussed in Section 3.1: 'If solutions to any of the tasks are openly available on the web, LLMs that serve as the backbone for the agents may have been trained on the solutions.' They use novel extensions stored in private Bitbucket repos to prevent overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Core design principle: 'We circumvent this problem by including only novel research extensions.' Gold solutions in private Bitbucket repos. 'We use Bitbucket instead of GitHub since GitHub data has been used in the past to train LLMs.' Source papers are public, but the specific extensions and solutions are novel."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates AI coding agents on benchmark tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports average cost per task and total cost for each agent-model combination (e.g., aider + o4-mini: $0.03 avg, $1.02 total). Figure 4 plots cost vs. final success on the Pareto frontier."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 specifies hardware per task (A100 40GB, K80 12GB, or CPU). Table 6 reports total duration per agent. Gold solution runtimes in Table 2 (20min to 6h). 'Including preliminary and failed runs...approximately 4–5x the reported amount.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "'We run each task three times with the same agent model to account for agent random variation.' Results in Tables 8-16 show per-run variation. Gold solutions control random seeds: 'we (1) fix all random seeds in the codebase wherever possible.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'We run each task three times with the same agent model to account for agent random variation.' Three runs explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search reported. Temperature set to 0.7 and reasoning effort to 'medium' without justification for these choices. No search budget or alternatives explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of temperature=0.7, reasoning effort='medium', and other agent configurations are not justified. No validation set used for selection. These appear to be default or arbitrary choices."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 tests four predictors in a regression model with individual p-values but no multiple comparison correction applied. Nine agent configurations are compared without correction for the number of comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluate agents on it. While they evaluate third-party tools (not their own system), they do not discuss potential bias in benchmark construction (e.g., task selection, instruction design) that could systematically favor certain agent architectures."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 explicitly plots cost vs. final success and time vs. final success, identifying Pareto-optimal agents. 'aider + o4-mini and OpenHands + Claude 3.7 Sonnet lie on the Pareto frontier for both cost and time.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses desiderata and validity: tension between realism and automatic evaluation, calibration of instruction granularity, self-containment requirements. Limitations section discusses how the idealized setting differs from real research extension. Multiple revision rounds ensured task quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The experimental design crosses frameworks with models: 3 frameworks × 4 LLMs. Table 3 documents capability differences across frameworks. Results clearly show scaffold effects (e.g., same Claude 3.7 Sonnet backbone yields different results across aider/Claude Code/OpenHands). This design separates scaffold from model effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Core design feature: extensions are novel implementations that do not exist publicly. 'To the best of our knowledge, none of our extensions exist on top of the existing codebases publicly.' Bitbucket chosen over GitHub to avoid training data inclusion. Source papers are public but solutions are new."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether task instructions or the self-contained input design might leak implementation hints. The paper ensures self-containment (all needed info is provided) but does not discuss whether this constitutes feature leakage relative to what agents would have in realistic settings."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 12 tasks share structural similarities that could inflate apparent generalizability. Tasks span different extension types (model, algorithm, data, evaluation) but potential dependencies between tasks are not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Concrete prevention methods: gold solutions stored in private Bitbucket repositories (not GitHub); evaluation infrastructure privately hosted; novel extensions with no public implementations. 'We use Bitbucket instead of GitHub since GitHub data has been used in the past to train LLMs.'"
       }
     }
   },
   "claims": [
     {
       "claim": "Current coding agents fail to autonomously implement the majority of research extensions, with the best agents achieving only 25% success rate.",
       "evidence": "Figure 2 and Section 4.3: OpenHands + Claude 3.7 Sonnet and Claude Code both achieve 25% final success rate across 12 tasks × 3 runs. Most other configurations achieve near-zero success.",
       "supported": "strong"
     },
     {
       "claim": "Even with step-by-step implementation hints, the best agent achieves only 39% success rate.",
       "evidence": "Figure 3 and Table 4: OpenHands + Claude 3.7 Sonnet with first-level hints reaches 39% final success. Second-level hints did not yield additional benefits.",
       "supported": "strong"
     },
     {
       "claim": "Claude 3.7 Sonnet is the strongest backbone LLM for agent-based research extension tasks.",
       "evidence": "Figure 2: Claude 3.7 Sonnet yields the best performance across frameworks — 14% (aider), 25% (Claude Code), 25% (OpenHands). o4-mini is second with 3-8% depending on framework.",
       "supported": "moderate"
     },
     {
       "claim": "The amount of code changes required (lines of gold solution) is the strongest predictor of task difficulty for agents.",
       "evidence": "Section 5.2, Figure 5: Mixed-effects regression shows β = −0.036, p < 0.01 for line changes. Other factors (file count, citations, repository popularity) had negligible or non-significant effects.",
       "supported": "strong"
     },
     {
       "claim": "Agents generally locate the correct files to edit but fail at implementation.",
       "evidence": "Figure 2: File recall ranges from 0.18-0.87 across agents, generally high (0.64-0.87 for top agents), while final success rates are much lower (0-25%). The gap between file recall and final success indicates implementation, not localization, is the bottleneck.",
       "supported": "moderate"
     },
     {
       "claim": "Overthinking is a prominent failure mode for reasoning models (DeepSeek R1, o1, o4-mini).",
       "evidence": "Section 5.1.1: 'aider + DeepSeek-R1 was especially prone to this behavior, overthinking being one of the most prominent failure modes (close to one third of total failures).' Described qualitatively without systematic quantification.",
       "supported": "moderate"
     },
     {
       "claim": "REXBENCH is robust to data contamination by using novel extensions stored in private repositories.",
       "evidence": "Section 3.1: Extensions are novel implementations not publicly available. Gold solutions in private Bitbucket repos. 'To the best of our knowledge, none of our extensions exist on top of the existing codebases publicly.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "REXBENCH evaluates 9 LLM agent configurations on 12 research extension tasks and finds all agents struggle, with the best (OpenHands + Claude 3.7 Sonnet and Claude Code) achieving only 25% success rate. Hints improve performance to 39% but inconsistently, and detailed step-by-step hints do not help beyond information localization hints. Task difficulty is primarily driven by the amount of code changes required (β = −0.036, p < 0.01). Agents show high file recall but low execution and final success rates, indicating implementation logic rather than code navigation is the primary bottleneck.",
   "red_flags": [
     {
       "flag": "Very small benchmark",
       "detail": "Only 12 tasks with 3 runs each — total of 36 data points per agent. This severely limits statistical power and generalizability. A single task (Othello, which swings from 0% to 100% depending on agent) can substantially shift overall success rates."
     },
     {
       "flag": "No formal agent comparison tests",
       "detail": "Claims about which agents/models are 'best' (e.g., Claude 3.7 Sonnet as strongest backbone) are based on comparing raw percentages without significance tests. With only 12 tasks × 3 runs and high variance, many of these differences may not be statistically significant."
     },
     {
       "flag": "Missing model version specificity",
       "detail": "All models identified by marketing names (Claude 3.7 Sonnet, o1, o4-mini, DeepSeek R1) without API versions or snapshot dates. Model behavior changes across versions, making exact reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Foundational benchmark for LLM coding agent evaluation using real GitHub issues, directly relevant as a comparison point for agent benchmarking methodology."
     },
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Attempts to automate the full research pipeline, directly relevant to the survey's coverage of AI research automation capabilities."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Jun Shern Chan"],
       "year": 2025,
       "arxiv_id": "2504.01848",
       "relevance": "Most directly comparable benchmark to REXBENCH; evaluates paper replication rather than extension, relevant to research automation evaluation methodology."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "Open-source agent framework evaluated in this paper, relevant as a major agentic coding platform."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Chan Jun Shern", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for ML engineering agents using Kaggle-style tasks, relevant to understanding agent capabilities in ML contexts."
     },
     {
       "title": "SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories",
       "authors": ["Ben Bogin", "Kejuan Yang", "Shashank Gupta", "Kyle Richardson"],
       "year": 2024,
       "relevance": "Evaluates agent ability to set up research environments, directly relevant as a benchmark for coding agents in research settings."
     },
     {
       "title": "MLAgentBench: evaluating language agents on machine learning experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Benchmark for ML experimentation agents covering classical ML tasks and Kaggle challenges, relevant to agent capability evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Open-weight reasoning model evaluated as agent backbone; relevant to understanding reasoning model capabilities for agentic tasks."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Prompting methodology for deliberate problem solving with LLMs, one of the benchmark tasks tests extension of this work."
     },
     {
       "title": "Curie: Toward Rigorous and Automated Scientific Experimentation with AI Agents",
       "authors": ["Patrick Tser Jern Kon", "Jiachen Liu"],
       "year": 2025,
       "arxiv_id": "2502.16069",
       "relevance": "Evaluates AI agents on scientific experiment planning and execution, closely related benchmark for research automation."
     },
     {
       "title": "Can LLMs generate novel research ideas? a large-scale human study with 100+ NLP researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2024,
       "arxiv_id": "2409.04109",
       "relevance": "Large-scale study evaluating LLM research ideation capability, relevant to understanding AI research automation potential."
     },
     {
       "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks",
       "authors": ["Frank F. Xu", "Yufan Song", "Boxuan Li"],
       "year": 2024,
       "arxiv_id": "2412.14161",
       "relevance": "Benchmark for LLM agents on real-world consequential tasks, relevant to evaluating agent capabilities beyond coding."
     }
   ]
 }