ai-research-survey

scan.json (31932B)

---

 {
   "paper": {
     "title": "Measuring AI Ability to Complete Long Software Tasks",
     "authors": ["Thomas Kwa", "Ben West", "Joel Becker", "Amy Deng", "Katharyn Garcia", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Nate Rush", "Sydney Von Arx", "Ryan Bloom", "Thomas Broadley", "Haoxing Du", "Brian Goodrich", "Nikola Jurkovic", "Luke Harold Miles", "Seraphina Nix", "Tao Lin", "Chris Painter", "Neev Parikh", "David Rein", "Lucas Jun Koba Sato", "Hjalmar Wijk", "Daniel M. Ziegler", "Elizabeth Barnes", "Lawrence Chan"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.14499"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section I states 'Code and data to reproduce some of the core figures in this paper will be provided in the supplementary material' — future tense, not yet released. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states data will be provided in supplementary material (Appendix I) but no actual download link or dataset is provided. HCAST tasks are noted as available 'on request' (Appendix B.1.1 footnote 12). SWAA tasks are not publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using Vivaria platform and various APIs but does not specify environment details for reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Appendix I notes code will be provided in supplementary material but offers no concrete instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% CI is reported for the main time horizon trend via hierarchical bootstrap (Figure 1, Section 3.2). The shaded region in figures represents 95% CI calculated by bootstrapping over task families, tasks, and runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports p = 0.006 for o3 lying above the long-run trend (Section 3.2). Bootstrap tests are used for confidence intervals."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: doubling time of 207 days (95% CI 166-240 days), o3 has 110-minute time horizon vs GPT-2's 2-second horizon, 80% time horizons are 4-6x shorter than 50% horizons (Section 3.2.1). R² values reported (0.83, 0.80)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or explicit justification for why 170 tasks, 12 frontier models, or ~800 baselines were chosen. The paper acknowledges limited models ('only seven frontier models in this time span' for 2024-2025) but does not formally justify sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance is reported via hierarchical bootstrap with 10,000 samples over task families, tasks, and runs (Section 3.2). Error bars shown on figures. Run-to-run variance analyzed in Figure 6 sensitivity analysis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human baselines are included throughout: 800+ baselines totaling 2,529 hours (Section 2.2). 12 frontier models compared against each other and against human performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models span 2019-2025, including contemporary frontier models: o3, Claude 3.7 Sonnet, o1, Claude 3.5 Sonnet (New). The most recent models are from early 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation/sensitivity analysis in Section 5 and Appendix H: alternative curve fits, task family weighting, regularization, baseline noise, task suite composition (removing RE-Bench), continuous vs binary scoring."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: 50% time horizon, 80% time horizon (Section 3.2.1), average success rate (Figure 3), per-task-family success rates, messiness-adjusted performance, continuous scoring (Appendix H), cost per successful run (Figure 14)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is included for internal PR tasks (Section C.2): repository maintainers manually scored model and baseliner solutions on a 0-1 scale as if reviewing PRs. Qualitative analysis of 63 failed runs was done manually (Section 3.3, Appendix D)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWAA tasks were developed blind to AI performance (Appendix B.1.3): 'all tasks were written before seeing AI attempts, and elicitation was carried out on a separate development task suite.' Agent scaffolds were developed on held-out dev tasks not in HCAST (Appendix C.3.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by task source (Table 7: HCAST, RE-Bench, SWAA), by task length buckets (Figure 3 right, Figure 4), by messiness score (Figure 12), and per-model (Figure 3 left)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in Section 3.3 and Appendix D: 31 GPT-4 and 32 o1 failed runs categorized into failure types (Table 3.3). Specific examples of failure modes provided (poor planning, incorrect reasoning, premature abandonment, repeating failed actions)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: models struggle on 'messier' tasks (Section F.2), performance much lower on less structured tasks, AI agents fail to proactively seek information (Section D.2.2), internal PR results show models scoring 0 on several issues (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: 50% time horizon of ~110 minutes for o3 (Section 3.2), doubling every ~7 months (207 days, Section 3.2), improvements driven by reliability/reasoning/tool use (Section 3.3), 5-year extrapolation to 1-month tasks (Section 5). Abstract appropriately hedges with 'If these results generalize.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 claims improvements are 'primarily driven by greater reliability, ability to adapt to mistakes, logical reasoning, and capacity for tool use' but this is based on qualitative analysis of 63 transcripts, not controlled experiments isolating these factors. The causal attribution is not rigorously supported."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively bounds generalizations: Section 4 on external validity, Section B.2 lists 5 specific limitations of the task suite, Section 5 explicitly states 'If these results generalize to real-world software tasks', and Section E.3 discusses interpreting time horizon caveats (context effects, task distribution effects)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations discussed: SWE-bench doubling time differences may be due to annotator time bias (Section 4.1), messiness factors could explain benchmark-vs-real-world gaps (Section F.2), baseline selection effects (Section C.1.4), elicitation quality differences (Section E.4), compute scaling vs algorithmic improvement (Section E.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that time horizon on benchmark tasks is a proxy for real-world capability. Section 4 is dedicated to external validity. Section B.2 lists systematic differences between tasks and real work. Section E.3 notes 'time horizon is always measured relative to a domain, task distribution, and baseliners' level of skill and context.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions provided in Table 6 and Appendix C.3: 'GPT-4 0314', 'GPT-4 1106', 'GPT-4 Turbo', 'Claude 3.5 Sonnet (Old)', 'Claude 3.5 Sonnet (New)', 'davinci-002', 'gpt-3.5-turbo-instruct', 'o1', 'o1-preview'. Footnote 6 explains release date mapping for API model changes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes scaffolding approaches (Appendix C.3.1) and mentions 'a simple prompting scaffold' for SWAA tasks but does not include the actual prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or sampling parameters are reported for any model. Token limits are mentioned indirectly ('the system reaches its predefined usage threshold') but no specific values given."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Scaffolding described in Appendix C.3.1: modular-public scaffold provides Python/Bash commands with context management; triframe scaffold generates plans and multiple candidate actions with scoring. Both use ReAct-style reasoning. Table 6 maps models to scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data processing documented: task scoring thresholds explained (Section 3.1), baseline filtering for successful runs (Section C.1.4), geometric mean of baseline times (Section C.1.4), task family weighting by 1/√n (Section C.4), binarization of continuous scores described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion' includes a 'Limitations and future work' subsection. Section E.4 provides detailed limitations. Section B.2 'Limitations of the task suite' is a dedicated subsection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Highly specific threats discussed: task suite lacks automatic scoring realism, no multi-agent interaction, lax resource constraints, static environments (Section B.2). Baseliner skill effects quantified (Section C.2 shows 5-18x speed difference). Elicitation effort varies by model (Section E.4)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Clear scope boundaries: 'we evaluate models only on software and research tasks' (Section 5 footnote 8). Section E.3 states time horizon is relative to domain/distribution/baseliner skill. Section B.2 lists 5 specific ways tasks differ from real work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data not available. Appendix I states code and data 'will be provided' in supplementary material (future tense). 'The full codebase is infeasible to publicize and anonymize.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection thoroughly described: HCAST tasks from Rein et al. [8], RE-Bench from Wijk et al. [2], SWAA created by the team with blinding to model performance (Appendix B.1.3). Human baseline collection procedure detailed in Appendix C.1. Agent runs described in Appendix C.3."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Baseliner recruitment described: 'skilled professionals in software engineering, ML, and cybersecurity, with the majority having attended world top-100 universities' with 'an average of about 5 years of relevant experience' (Section 2.2). Internal employees used for SWAA (Section C.1.3). Payment incentives described (Section C.1.4)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: task creation → human baselining → agent evaluation → score binarization → logistic regression → time horizon computation (Figure 2). Baseline filtering criteria stated (Section C.1.4). Task family weighting explained (Section C.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgments section thanks reviewers but does not mention any grants or funding agencies. METR is the author organization but its funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as affiliated with METR (Model Evaluation & Threat Research). Daniel Ziegler noted as Anthropic (work done at METR). Luke Harold Miles noted as Ohm Chip (work done at METR)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "METR's funding sources are not disclosed, so independence cannot be assessed. METR is an AI safety evaluation organization; its institutional interest in demonstrating rapid AI capability growth could create a non-independent dynamic, though this is not disclosed or discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. No declaration of financial interests, patents, or equity."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates are not stated for any of the 12+ models evaluated. The paper discusses contamination conceptually (footnotes about release dates vs training dates) but does not state specific training data cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Partially addressed: SWAA tasks were 'written before seeing AI attempts' (Appendix B.1.3). HCAST tasks are not publicly shared to 'reduce the likelihood of AI systems accidentally or intentionally being trained on them' (footnote 12). Internal PR tasks described as 'uncontaminated' (Section C.2)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination addressed through design: tasks are kept private (HCAST footnote 12), SWAA developed blind to model performance, internal PRs are uncontaminated by design. For SWE-bench Verified, the paper notes potential contamination but uses it only for external validity, not as the main benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned. The human baselining study involves human participants performing tasks but no pre-registration link (OSF, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite the study involving human participants (baseliners completing tasks with screen and audio recording)."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographics partially reported: 'skilled professionals in software engineering, ML, and cybersecurity, with the majority having attended world top-100 universities' and 'an average of about 5 years of relevant experience' (Section 2.2)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion/exclusion criteria for baseliner selection. The paper states they are 'skilled professionals' but does not specify screening criteria, minimum qualifications, or exclusion rules."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study comparing treatment vs control conditions for human participants. Baseliners are performing tasks to establish timing, not randomized into conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable — baseliners are performing tasks to establish timing benchmarks, not participating in a blinded experimental comparison."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition partially reported: 558 baselines with 286 successful from HCAST/RE-Bench (Section C.1), 249 with 236 successful from SWAA. Section C.1.4 discusses why failed baselines were filtered and the bias this introduces."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 14 shows cost of a successful run as a fraction of human expert salary ($143.61/hour). Section E.4 states 'more than 80% of successful runs cost less than 10% of what it would cost for a human.' Cost information not included for o3 and o4-mini (Figure 18 caption)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section C.3.2: 'about 2,000 H100-hours for RE-Bench environments and 50,000 CPU hours for other environments... plus roughly 50,000 H100-hour equivalents of compute used internally or from API providers for inference.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple runs per task: 'We perform 8 runs per agent/task pair' (Section 2.3). Variance across runs is captured in the hierarchical bootstrap. Run-to-run variance is one component of the sensitivity analysis (Figure 6)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3: 'We perform 8 runs per agent/task pair' with a footnote that 'This number is approximate, because a small number of runs failed due to internal infrastructure issues.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the scaffolding or elicitation. Section E.4 notes varying elicitation effort ('around 2-3 engineer weeks of iterative development' for o1 and Claude 3.5 Sonnet) but no systematic search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Scaffold selection is documented: Table 6 maps each model to its scaffold. Section E.4 acknowledges differential elicitation effort. The sensitivity analysis (Appendix H) tests robustness across multiple configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "Only one formal statistical test is reported (p = 0.006 for o3). No multiple comparison issue arises."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section E.4 explicitly acknowledges: 'We have put a limited amount of effort into eliciting models to get good performance on our tasks, so while our results are a reasonable lower bound, some models may have somewhat greater capabilities than we demonstrate.' This addresses the bias of authors controlling evaluation quality."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 18 plots success rate vs inference cost, showing models plateau below their token limits. Section E.4 notes 'more than 80% of successful runs cost less than 10% of human equivalent' and discusses room for more inference compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Extensive construct validity discussion: Section B.2 lists 5 systematic differences between tasks and real work. Section F.2 analyzes 16 'messiness factors' to assess how representative the benchmark is. Section 4 dedicates multiple experiments to external validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 documents which scaffold was used for each model. Appendix C.3.1 describes the two scaffold types (modular-public and triframe/duet). The paper acknowledges scaffold differences could affect results, and o1/o1-preview use different scaffolds due to their tool-use limitations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Tasks designed to minimize temporal leakage: HCAST tasks kept private (footnote 12), SWAA developed blind to model performance (Appendix B.1.3), internal PRs described as 'uncontaminated' (Section C.2). SWE-bench contamination acknowledged (Section 4.1)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of whether evaluation setup provides hints not available in real usage. The paper does not address whether task instructions or environment features leak information about expected solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Task families used to address non-independence: 'we weight each task by the inverse square root of the number of tasks in the family it belongs to' (Section 2.3). Correlation between task families is reported (Figure 22, average correlation ~0.73)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Prevention methods used: tasks kept private and not published (footnote 12), SWAA tasks blinded during development, internal PRs confirmed uncontaminated. While no post-hoc detection method (canary strings, membership inference) is used, proactive prevention is implemented."
       }
     }
   },
   "claims": [
     {
       "claim": "The 50% task-completion time horizon for frontier AI models has been doubling approximately every 7 months (207 days) since 2019.",
       "evidence": "Logistic regression on 12 frontier models from 2019-2025, with 95% bootstrapped CI of 166-240 days doubling time (Section 3.2, Figure 1).",
       "supported": "strong"
     },
     {
       "claim": "Current frontier models (o3) have a 50% time horizon of approximately 110 minutes.",
       "evidence": "Logistic regression fit on 170 tasks with human baselines (Section 3.2, Figure 4).",
       "supported": "strong"
     },
     {
       "claim": "The 80% time horizon follows a similar doubling rate (204 days) but is 4-6x shorter than the 50% horizon.",
       "evidence": "Figure 17 shows 80% horizon trend with similar slope (Section 3.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "AI improvement is primarily driven by greater reliability, tool use, logical reasoning, and adaptation to mistakes.",
       "evidence": "Qualitative analysis of 63 failed agent transcripts (31 GPT-4, 32 o1) categorized into failure types (Section 3.3, Table 3.3).",
       "supported": "weak"
     },
     {
       "claim": "If extrapolated, AI will reach a 1-month (167 hours) time horizon between mid-2028 and mid-2031.",
       "evidence": "Sensitivity analysis in Figure 6 with 80% CI spanning roughly 2 years. Central estimate mid-2029 (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "Models perform worse on 'messier' tasks, but the rate of improvement over time is similar for high and low messiness subsets.",
       "evidence": "16 messiness factors rated on HCAST/RE-Bench tasks. Success rate decreases ~8.1% per messiness point. Figure 12 shows similar improvement slopes (Section F.2).",
       "supported": "moderate"
     },
     {
       "claim": "Contract baseliners take 5-18x longer than repository maintainers to complete internal PR tasks.",
       "evidence": "Table 5 comparing maintainer vs baseliner times on 5 internal PRs (Section C.2).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "The 50% task-completion time horizon for frontier AI models has been doubling approximately every 7 months from 2019-2025, with current models (o3) reaching ~110 minutes. The trend appears consistent across 50% and 80% success rates and multiple robustness checks, though external validity remains uncertain. Models still struggle significantly on 'messier' tasks lacking clear feedback loops, though improvement rates appear similar across messiness levels. Naive extrapolation predicts AI capable of month-long software tasks by mid-2028 to mid-2031.",
   "red_flags": [
     {
       "flag": "Causal claims from qualitative analysis",
       "detail": "The claim that improvements are 'primarily driven by' specific factors (Section 3.3) is based on qualitative analysis of 63 transcripts, not controlled experiments. This causal attribution is not well-supported by the study design."
     },
     {
       "flag": "Differential elicitation effort",
       "detail": "Section E.4 acknowledges '2-3 engineer weeks of iterative development' for o1 and Claude 3.5 Sonnet while 'All other models use the same scaffolding with at most minor changes.' This differential effort could inflate apparent improvement rates for better-elicited models."
     },
     {
       "flag": "Baseline selection bias",
       "detail": "Section C.1.4 acknowledges that filtering for successful baselines biases toward shorter task length ratings, potentially underestimating model performance. The payment scheme incentivized contractors to give up early on difficult tasks."
     },
     {
       "flag": "No competing interests disclosure",
       "detail": "METR as an AI safety evaluation organization has institutional interest in demonstrating rapid AI capability growth. No funding sources or competing interests are disclosed."
     },
     {
       "flag": "Extrapolation beyond data range",
       "detail": "The extrapolation to 1-month horizon (Section 5) extends well beyond the observed data range (2 seconds to 110 minutes). While the paper acknowledges this limitation, the forecasts receive prominent treatment in the abstract and conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
       "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "Core benchmark used in this study for evaluating AI agent capabilities on ML research engineering tasks."
     },
     {
       "title": "HCAST: Human-Calibrated Autonomy Software Tasks",
       "authors": ["David Rein", "Joel Becker", "Amy Deng"],
       "year": 2025,
       "relevance": "Core benchmark providing 97 diverse software tasks with human baselines used in this study."
     },
     {
       "title": "Evaluating frontier models for dangerous capabilities",
       "authors": ["Mary Phuong", "Matthew Aitchison"],
       "year": 2024,
       "arxiv_id": "2403.13793",
       "relevance": "Related work on contextualizing AI benchmark performance for dangerous capability assessment."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu", "Hao Yu"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Agentic AI capability benchmark evaluating agents across diverse environments."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Industry-standard software engineering benchmark used for external validity analysis in this paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark for code generation, foundational benchmark in AI coding evaluation."
     },
     {
       "title": "SWE-Lancer: Can frontier LLMs earn $1 million from real-world freelance software engineering?",
       "authors": ["Samuel Miserendino", "Michele Wang"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Concurrent work on evaluating AI agents on real-world freelance software engineering tasks."
     },
     {
       "title": "TheAgentCompany: benchmarking LLM agents on consequential real world tasks",
       "authors": ["Frank F. Xu", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2412.14161",
       "relevance": "Benchmark requiring multi-agent interaction in realistic settings, relevant to agentic AI evaluation."
     },
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "ML engineering benchmark for evaluating AI agent capabilities."
     },
     {
       "title": "Forecasting frontier language model agent capabilities",
       "authors": ["Govind Pimpale", "Axel Højmark"],
       "year": 2025,
       "arxiv_id": "2502.15850",
       "relevance": "Related work on forecasting AI agent capabilities using compute and other metrics."
     },
     {
       "title": "Algorithmic progress in language models",
       "authors": ["Anson Ho", "Tamay Besiroglu"],
       "year": 2024,
       "arxiv_id": "2403.05812",
       "relevance": "Documents algorithmic improvements reducing compute requirements, relevant to AI capability trends."
     },
     {
       "title": "AIDE: AI-driven exploration in the space of code",
       "authors": ["Zhengyao Jiang", "Dominik Schmidt"],
       "year": 2025,
       "arxiv_id": "2502.13138",
       "relevance": "Shows that proper elicitation can dramatically improve AI agent performance on software tasks."
     }
   ]
 }