ai-research-survey

scan.json (30870B)

---

 {
   "paper": {
     "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
     "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan", "Michael Chen", "Josh Clymer", "Jai Dhyani", "Elena Ericheva", "Katharyn Garcia", "Brian Goodrich", "Nikola Jurkovic", "Holden Karnofsky", "Megan Kinniment", "Aron Lajko", "Seraphina Nix", "Lucas Sato", "William Saunders", "Maksym Taran", "Ben West", "Elizabeth Barnes"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2411.15114",
     "doi": "10.48550/arXiv.2411.15114"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "RE-Bench introduces 7 hand-crafted ML research engineering environments with data from 71 attempts by 61 human experts. AI agents (o1-preview, Claude 3.5 Sonnet) achieve 4x higher scores than humans at 2-hour budgets, but humans surpass agents at 8 hours and achieve 2x agent scores at 32 hours. Agents generate and test solutions 10x faster than humans but struggle with long-horizon agency, variety in approaches, and recovering from failures.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Environments are open-sourced at github.com/METR/ai-rd-tasks, agent trajectories at transcripts.metr.org. The abstract states 'We open-source the evaluation environments, human expert data, analysis code and agent trajectories.' The footnote notes analysis code and anonymized human data are 'coming soon.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Agent trajectories are released at transcripts.metr.org. Anonymized human expert data is stated as 'coming soon' but agent data is available. The environments themselves are open-sourced."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware specs are given per environment (e.g., '4×H100, 1×52 core CPU, 400 GB RAM') but no requirements.txt, Dockerfile, or detailed software dependency list is provided in the paper itself. They reference Vivaria platform but do not provide a full environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While environments are open-sourced and the paper describes the evaluation procedure, there are no step-by-step reproduction instructions in the paper. The open-source release may contain these, but the paper itself lacks a 'Reproducing Results' section with specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows '95% confidence intervals generated via percentile bootstrapping.' Shaded regions on the main comparison plots represent confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests are reported. Comparisons between agents and humans are made by comparing score@k curves and averages without p-values or statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes: 'best AI agents achieve a score 4× higher than human experts when both are given a total time budget of 2 hours' and 'achieving 2× the score of the top AI agent when both are given 32 total hours.' These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 7 environments and 71 human attempts by 61 experts but provides no justification for these sample sizes. No power analysis is discussed. The paper acknowledges noise from the small number of environments (Section 6.3) but does not justify the chosen N."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance is reported in multiple forms: Figure 4 shows percentile ranges of human performance, histograms in Appendix G show score distributions per environment, and confidence intervals via bootstrapping are shown in main results. Table 6 reports standard deviation of starting scores."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human expert performance serves as the primary baseline. Multiple agent configurations (two scaffolds, multiple models) are compared. Starting solutions provide a zero-score baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates o1-preview and Claude 3.5 Sonnet (claude-3-5-sonnet-20241022), which were frontier models at time of writing. AIDE scaffold was chosen as it 'performed the best in recent studies of MLE-bench.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple scaffolds (Modular vs AIDE), multiple models, multiple time allocations (30min vs 2h vs 8h), and best-of-k with varying k. This serves as a form of ablation over key design dimensions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: normalized score per environment, average normalized score across environments, score@k at various time budgets, per-environment breakdowns, and cost comparisons (Figure 11)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human expert performance is the core comparison. Additionally, agent runs were manually inspected: 'we only carefully inspected the 2 best performing runs of each task' and cheating was manually checked (Appendix A.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Most environments use held-out test sets separate from what the agent can observe. For example, Scaffolding for Rust CodeContests uses 'a held-out test set' of 175 problems. The paper acknowledges the overfitting concern for environments that show test scores to agents (Section 6.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 provides per-environment breakdowns. Appendix G-I provides detailed per-environment histograms, score@k curves, and time series for all 7 environments."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 extensively analyzes agent failures: lack of variety in solutions, misunderstandings of instructions, stubborn incorrect assumptions, difficulty recovering from failures. Annotated agent failure transcripts are provided in Appendix D (Figures 13-15)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that most agent runs score close to 0, that agents accumulate 'more issues and false assumptions than useful insights,' and that specific environments like Scaling Law Experiment involve agent guesswork rather than skill (Section 6.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (4x score at 2h, humans exceeding at 8h, 2x at 32h, agents generating solutions 10x faster, Triton kernel result) are all supported by figures and data in the paper (Figure 2, Section 5.1)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about causal language. Claims are comparative ('agents achieve a score 4× higher') based on controlled comparisons under equivalent conditions. The paper explicitly notes limitations (Section 6) and avoids strong causal claims about why agents perform differently."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 extensively bounds generalization: 'We expect the human–AI gap in real-world AI R&D to be much larger than the gap observed on these evaluations.' Table 7 explicitly compares RE-Bench scale to real AI R&D across multiple dimensions. The paper states these results should not imply agents can automate real AI R&D."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1-6.2 discusses multiple alternative explanations: agents may be cheaper per unit, different workflows could change results, environments may overestimate agent capability due to limited scope, or underestimate due to cost advantages. Section 5.1 discusses noise/overfitting as alternative explanation for high scores."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that RE-Bench measures performance on short self-contained tasks, not real AI R&D automation. Section 6.1 and Table 7 lay out exactly why benchmark performance is a proxy and what it does not capture (long time horizons, engineering complexity, coordination)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: 'claude-3-5-sonnet-20241022' and 'o1-preview' and 'claude-3-5-sonnet-20240620' (Section 4). GPT-3.5-turbo-0125 is specified for the scaffolding task."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full task prompts/instructions are provided for all 7 environments in Appendix C. System prompts for the agent scaffolding are described in detail in Appendix A.2.1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature, top-p, and other sampling parameters for the model API calls are not reported. The paper describes the scaffold architecture but does not state LLM sampling hyperparameters used for agent generation."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Both scaffolds are described in detail in Appendix A.2.1: Modular agent (system prompt, tool access, context management, timeout handling) and AIDE (modifications for compatibility). The scaffolding code modifications are enumerated."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Score normalization is documented (Section 3.2.2). Human expert selection criteria are described (Section 3.4). Agent run procedures, clock pausing for API issues, and cheating detection procedures are described (Appendix A)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 'Other Limitations' is a dedicated subsection. Sections 6.1 and 6.2 also discuss over/underestimation of capabilities extensively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: noisy agent results from rare successes, Finetune GPT-2 for QA having high variance, agent solutions being overfitted to test scores, Scaling Law Experiment scores involving luck, and environments having unrepresentative design constraints."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 explicitly compares RE-Bench to real AI R&D across dimensions. Section 6.1 states: 'we find it fairly plausible that the first agents that match top human performance in these environments may still be far from capable of AI R&D automation.' Section 3.3 identifies specific missing skill areas."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Agent trajectories are available at transcripts.metr.org. The paper states 'We open-source the evaluation environments, human expert data, analysis code and agent trajectories.' Anonymized human expert data is noted as 'coming soon.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Human data collection is described in detail: SSH into VMs, 8-hour limit, instructions to be 'greedy,' allowed tools, frequent commits, score logging, research logs, post-run manual inspection (Appendix A.1). Agent data collection is described in Appendix A.2."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Three recruitment sources are described: professional networks of METR staff, ML RS/RE hiring process applicants, and graduate student outreach (Section 3.4, Table 5). Selection criteria for each source are stated. The paper notes meaningful score differences between sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw scores to normalized results is documented: scoring function records timestamped entries, log-linear interpolation is used for time curves, normalization formula is given (Section 3.2.2), bootstrapping for confidence intervals is described. Cheating detection and invalid run filtering procedures are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgment of financial support is present. The work is by METR (Model Evaluation and Threat Research) but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: METR is the primary affiliation, with footnotes for Joel Becker (Qally's), Josh Clymer and Aron Lajko (Redwood Research), Jai Dhyani (Independent), Nikola Jurkovic (Harvard), and Holden Karnofsky (Anthropic/Carnegie Endowment)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. METR is an AI safety organization that has an organizational interest in demonstrating the value of AI R&D evaluations, which is the core contribution of this paper. Holden Karnofsky is affiliated with Anthropic, whose model is evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates o1-preview and Claude 3.5 Sonnet on novel benchmark tasks but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues contamination is mitigated by novelty: environments are 'novel, hand-crafted' (Section 1), Table 1 lists 'Novel, non-contaminated' as a criterion and marks RE-Bench as satisfying it. Appendix B.1 discusses novelty of reference solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination by designing novel environments. Table 1 marks RE-Bench as novel/non-contaminated. The creation process (Section 3.2.3) involved hand-crafting environments that are not directly available online, and Section 2.2 discusses contamination risk as a design concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The study involves 61 human experts as participants but there is no link to any pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite involving 71 human expert attempts."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 breaks down experts by source. Section 3.4 describes experience levels: professional network experts have '5+ years experience' or worked at 'Google DeepMind, Google, Anthropic, OpenAI, FAR Labs, Redwood Research.' Hiring applicants passed CV/CodeSignal screens. Graduate students are from UCB, CMU, Stanford, MIT. Appendix F shows experience measures by score quintile."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Selection criteria are described for each source: professional network experts needed 5+ years relevant experience or recent work at top ML orgs; hiring applicants passed CV screen, CodeSignal, interview, and short task; graduate students were selected for relevant advisors or being in ML PhD programs at specific universities (Section 3.4)."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We attempted to match experts to environments where they had relevant experience, whilst trying to maintain a balanced distribution' but does not describe a formal randomization procedure. Assignment was purposive, not randomized."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants were told context about what METR was trying to accomplish (Appendix A.1). No discussion of whether this knowledge could affect performance."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 71 attempts by 61 experts but does not discuss whether any experts dropped out, were excluded from analysis, or failed to complete their sessions."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 reports: 'agents use ~29M input tokens and ~499K output tokens in each 8-hour run, at a cost of approximately $123' compared to '$1,855' paid to human experts on average. Figure 11 shows results by token/labor cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified per environment (Appendix C): e.g., '4×H100, 1×52 core CPU, 400 GB RAM.' Token usage and costs are reported. Time budgets (8h, 30min, 2h) are explicitly stated. Number of runs per configuration is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results across multiple independent runs (3-5 per environment per agent for 8h runs, up to 128 for 30min runs). Score distributions and variance across runs are shown in Appendix G histograms and Figures 7-8."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of runs is stated: '3 to 5 8-hour attempts per environment' (Section 4.1), and specific k values for best-of-k experiments are shown in Figures 6-8. Table 5 reports 71 total human runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the agent scaffolding. The paper notes 'Preliminary studies found that o1-preview performed poorly in the Modular scaffold' but does not quantify how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper transparently reports results for all configurations tested (both scaffolds, multiple models, multiple time allocations) rather than cherry-picking. Figure 2 aggregates using the best allocation found for each agent, which is clearly stated."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "METR designed the benchmark and evaluated agents on it. The paper does not explicitly discuss the bias of being both benchmark creators and evaluators. However, they do note that better scaffolds could improve results (Section 6.3)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 11 explicitly shows 'Best score@k results by token cost for AI agents and by money paid to human experts.' Figure 2 shows performance vs total time budget. The paper extensively discusses compute/cost tradeoffs between agents and humans."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 maps environments to AI R&D skills identified by Epoch AI's survey (Table 4), and explicitly identifies gaps (missing areas like distributed training, research direction setting). Section 6.1 and Table 7 discuss why RE-Bench may over/underestimate real AI R&D capability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly evaluates two different scaffolds (Modular and AIDE) and reports how results differ between them. Section 6.3 acknowledges 'Different agent scaffolds or prompts might be able to achieve a much better score.' Results are reported separately per scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses temporal leakage by designing novel environments that did not exist online before the evaluation. Table 1 marks RE-Bench as 'Novel, non-contaminated.' Environments were hand-crafted specifically for this evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses that agent solutions may be overfitted because 'all environments except Scaling Law Experiment provide the agents with the test score output.' The paper acknowledges this as a form of information leakage and plans to use validation-only scoring in future iterations."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Non-independence between train and test data is not a primary concern here since the benchmark consists of novel environments, not a dataset split from a larger corpus."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method (canary strings, membership inference, etc.) is used. The mitigation is entirely through novelty of environments rather than detection."
       }
     }
   },
   "claims": [
     {
       "claim": "Best AI agents achieve a score 4× higher than human experts when both are given a total time budget of 2 hours per environment.",
       "evidence": "Figure 2 shows score@k results by total time budget. At 2 hours, agents significantly outperform humans across the aggregate metric.",
       "supported": "strong"
     },
     {
       "claim": "Humans narrowly exceed top AI agent scores given an 8-hour budget, and achieve 2× the score of the top AI agent at 32 total hours.",
       "evidence": "Figure 2 shows human performance crossing agent performance around 8 hours and reaching approximately 2× agent scores at 32 hours.",
       "supported": "strong"
     },
     {
       "claim": "AI agents generate and test solutions over ten times faster than humans.",
       "evidence": "Section 5.1: 'AIDE and modular agents run score 36.8 and 25.3 times per hour respectively, while human experts only do so 3.4 times.'",
       "supported": "strong"
     },
     {
       "claim": "Both o1-preview and Claude 3.5 Sonnet found solutions to the kernel optimization problem that beat all 9 human experts.",
       "evidence": "Section 5.1 and Figure 18 show agent solutions achieving faster kernel execution times than all human experts.",
       "supported": "strong"
     },
     {
       "claim": "Agents in the Modular scaffold do significantly better over many short 30-minute attempts, while AIDE agents do best with fewer 2-hour attempts.",
       "evidence": "Figure 6 directly compares three time allocation strategies across scaffolds.",
       "supported": "strong"
     },
     {
       "claim": "82% of expert attempts achieve a non-zero score and 24% match or exceed the reference solutions.",
       "evidence": "Table 6 shows per-environment data on experts scoring > 0.05, and the text states these aggregate statistics.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Small number of environments",
       "detail": "Only 7 environments are used, acknowledged by the authors as a significant source of noise. Results may not generalize to the broader space of AI R&D tasks."
     },
     {
       "flag": "Non-random expert-task assignment",
       "detail": "Experts were matched to environments based on relevant experience rather than random assignment. This introduces potential selection bias and makes aggregate human performance estimates harder to interpret."
     },
     {
       "flag": "Test score leakage to agents",
       "detail": "6 of 7 environments show test scores to agents during evaluation, potentially enabling overfitting. The Finetune GPT-2 for QA result dropped from 0.88 to 0.69 upon re-running, indicating overfitting to evaluation noise."
     },
     {
       "flag": "Heterogeneous expert quality",
       "detail": "Average scores differ substantially by recruitment source (0.48 for hiring applicants vs 0.98 for professional network), making the 'average human expert' a potentially misleading comparison point."
     },
     {
       "flag": "METR evaluates its own benchmark",
       "detail": "METR designed the benchmark, selected the environments, recruited the experts, and ran the evaluations. No independent replication or evaluation is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "relevance": "Foundational work on code generation evaluation (Codex/HumanEval), introduces pass@k metric referenced in this paper."
     },
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["J.S. Chan"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Directly comparable ML engineering benchmark for AI agents; AIDE scaffold was selected based on its MLE-bench performance."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C.E. Jimenez"],
       "year": 2023,
       "relevance": "Major code generation benchmark; paper discusses its feasibility issues as motivation for RE-Bench design."
     },
     {
       "title": "GPQA: A graduate-level google-proof q&a benchmark",
       "authors": ["D. Rein"],
       "year": 2023,
       "relevance": "Benchmark with human comparisons used in frontier model evaluation."
     },
     {
       "title": "The AI scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["C. Lu"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Directly relevant work on AI agents automating scientific research."
     },
     {
       "title": "MLAgentBench: Evaluating language agents on machine learning experimentation",
       "authors": ["Q. Huang"],
       "year": 2024,
       "arxiv_id": "2310.03302",
       "relevance": "Prior benchmark for ML agent capabilities, compared in Table 1."
     },
     {
       "title": "Sabotage evaluations for frontier models",
       "authors": ["J. Benton"],
       "year": 2024,
       "arxiv_id": "2410.21514",
       "relevance": "AI safety evaluation work on autonomous agent risks, directly motivates RE-Bench."
     },
     {
       "title": "OpenDevin: An open platform for AI software developers as generalist agents",
       "authors": ["X. Wang"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "AI agent platform for software development tasks."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["G. Mialon"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "General AI assistant benchmark with human comparisons, compared in Table 1."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "authors": ["S. Zhou"],
       "year": 2024,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic agent evaluation environment with human comparisons."
     },
     {
       "title": "Interviewing AI researchers on automation of AI R&D",
       "authors": ["D. Owen"],
       "year": 2024,
       "relevance": "Epoch AI survey on AI R&D automation used to assess RE-Bench coverage of key AI R&D skills (Table 4)."
     },
     {
       "title": "DISCOVERYWORLD: A virtual environment for developing and evaluating automated scientific discovery agents",
       "authors": ["P. Jansen"],
       "year": 2024,
       "arxiv_id": "2406.06769",
       "relevance": "Scientific discovery agent evaluation environment with human comparisons."
     }
   ]
 }