ai-research-survey

scan.json (27243B)

---

 {
   "scan_version": 2,
   "active_modules": [],
   "paper": {
     "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
     "authors": ["Joel Becker", "Nate Rush", "Beth Barnes", "David Rein"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.09089"
   },
   "methodology_tags": ["rct", "qualitative"],
   "key_findings": "In a randomized controlled trial with 16 experienced open-source developers completing 246 tasks, allowing AI tools (primarily Cursor Pro with Claude 3.5/3.7 Sonnet) increased task completion time by 19%, contradicting developer forecasts of 24% speedup and expert forecasts of ~39% speedup. The slowdown was robust across alternative estimators, outcome measures, and subsets. Contributing factors include developer over-optimism about AI usefulness, high developer familiarity with repositories, large/complex codebases, and low AI reliability (<44% generation acceptance rate).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or analysis scripts are provided in the paper. The paper describes detailed data collection and regression analyses but does not release code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link is provided. The paper describes collecting screen recordings, developer forecasts, and implementation times, but does not release the underlying data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency information is provided for reproducing the analyses."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The regression specification is described in Appendix D, but there are no scripts or instructions to replicate the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported throughout, using HC3 standard errors (Section D.2, Figure 15). The paper also reports CIs from alternative uncertainty estimation procedures including clustered standard errors and hierarchical bootstrap."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses regression-based inference with HC3 standard errors and reports p-values (e.g., Table 4 balance checks with Welch t-tests, Table 5 chi-square test). Figure 15 shows alternative uncertainty procedures."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The primary result is reported as a 19% increase in completion time (from the log-linear regression coefficient transformed via exp(β)-1). Context is provided with baseline completion times (Figure 4) and the effect is reported with confidence intervals."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The study has 16 developers and 246 tasks. The paper acknowledges being 'not powered for statistically significant multiple comparisons when subsetting our data' (Section 3.3) but does not justify why 16 developers was the chosen sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Table 4 for forecasted times. Multiple uncertainty estimation procedures are compared in Figure 15. The paper reports variance across developers (Figure 17) and across subsets."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study design inherently includes a baseline: the AI-disallowed condition serves as the control/baseline against which AI-allowed performance is compared. The paper also compares results against prior literature (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is between AI-allowed and AI-disallowed conditions within the same study, using contemporary AI tools (Claude 3.5/3.7 Sonnet, GPT-4o, Gemini 2.5 Pro). Prior work comparisons in Table 3 include contemporary studies."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT measuring a single treatment (AI allowed vs. not allowed), not a multi-component system. There is no system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple outcome measures: self-reported implementation time, screen recording time, pre-review and post-review time, lines of code per hour, and activity time breakdowns (Figure 6, Figure 22). Alternative estimators are also compared (Figure 13)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes extensive human evaluation: 143 hours of screen recordings were manually labeled with fine-grained activity labels (Section 2.4), exit interviews and surveys were conducted (Section G.5), and PR quality was assessed through the repositories' standard review processes."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT, not a prediction task. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: per-developer speedup (Figure 17), by prior task exposure and external resource needs (Figure 7), by AI experience (Figure 10), by scope creep (Figure 9), by month (Figure 23), and by activity type (Figures 6, 19-21)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is essentially a discussion of a failure case (AI slowing developers down). Section C.1.4 discusses low AI reliability and developers' experiences with failed AI generations. Qualitative examples of AI failures are provided throughout."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is a negative result: AI tools slow down experienced developers by 19%, contradicting expectations. The paper also reports which hypothesized contributing factors had evidence against them (Table 1, Section C.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (19% slowdown, developer forecast of 24% speedup, post-hoc estimate of 20% speedup, expert predictions of 38-39% speedup) are all supported by the results in the paper (Figure 1, Section 3.1, Table 6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('AI tooling slowed developers down') and uses an RCT design with randomized treatment assignment, which is the gold standard for causal inference. Balance checks confirm successful randomization (Table 4). Issues are defined before randomization to prevent confounding."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalization in Section 4.1 ('Key Caveats'), Table 2 ('What the evidence does not show'), and throughout the discussion. It states results do not imply AI is unhelpful in other settings, with future models, or with better elicitation strategies."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically investigates 21 alternative explanations for the slowdown in Section 3.3 and Appendix C, categorized into direct productivity loss, experimental artifacts, factors raising developer performance, and factors limiting AI performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the proxy-outcome distinction. Section 1 critiques prior work for using non-fixed outcome measures (lines of code, PRs) that may not correspond to productivity. It uses task completion time as its measure, notes this is 'a fixed outcome measure,' and discusses scope creep (Section C.2.3) as a potential gap between time and productivity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies models used: Claude 3.7 Sonnet (thinking mode), Claude 3.7 Sonnet, Claude 3.5 Sonnet, GPT-4o, Gemini 2.5 Pro, and o1, with usage percentages (Section C.3.6). These are marketing names without snapshot dates, but for an RCT studying tool usage in the wild, this represents what developers actually used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT where developers use AI tools naturally. There are no researcher-designed prompts — developers prompt AI tools as they see fit. The study measures the effect of allowing AI usage, not of specific prompts."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT where developers use commercial AI tools (Cursor Pro) with default settings. The researchers do not control hyperparameters — they are studying the tools as used in practice."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates Cursor Pro as a third-party tool used as-is by developers. The authors cannot describe Cursor's internal scaffolding. Section F.2.2 provides a primer on Cursor's features (chat, agent mode, autocomplete) as understood from the user perspective."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents data preprocessing: imputation of post-review times for unreviewed issues (Section C.3.4), filtering criteria for screen recordings (>10% broken time, >20% discrepancy with self-reported time, cheating removal), resulting in 74 of 128 recordings (Section 2.4). Issue dropout is documented (Section C.3.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 'Key Caveats' serves as a dedicated limitations section, discussing setting-specific factors, AI-specific factors, and agent capabilities. Table 2 explicitly lists what the evidence does not show."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses 21 specific threats grouped into four categories (Table 1, Appendix C). These are highly specific: 'Our sample of 16 developers' is acknowledged, experimentally driven overuse, sampling bias in recruitment, unfamiliar development environment, and more."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 is a gold-standard scope boundary statement, listing six specific things the evidence does NOT show (e.g., 'AI systems do not currently speed up many or most software developers'). Section 4.1 adds specific caveats about setting-specific factors and AI-specific factors."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (screen recordings, implementation times, forecasts, survey responses) is not publicly available. Only aggregated results and regression outputs are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: Section 2.2.2 covers screen recordings, Cursor analytics, implementation notes. Section 2.4 covers developer forecasts, expert forecasts, exit interviews, and fine-grained activity labels. Appendix G provides full developer instructions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section G describes recruitment: outreach via professional networks, Reddit communities, and GitHub profiles; filtering for 500+ star repos, 5+ recent commits; 51 initial respondents filtered to ~20 then 16. Selection criteria are explicit."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 51 interested developers → ~20 meeting criteria → 16 final participants (3 dropped, Section G.6). 246 tasks completed (136 AI-allowed, 110 AI-disallowed). Screen recording filtering: 128 recordings → 74 valid after quality filters (Section 2.4). Imputation methods for missing post-review times are described (Section C.3.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. METR (Model Evaluation & Threat Research) is the authors' organization, but no grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as being from METR (Model Evaluation & Threat Research). METR is not a developer of the AI tools being evaluated (Cursor, Claude, GPT-4o)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding source is not disclosed, so independence cannot be assessed. METR's organizational mission involves AI safety and capability evaluation, which could create incentives in either direction regarding AI productivity results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT measuring developer productivity with AI tools, not an evaluation of a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT measuring developer productivity, not a benchmark evaluation where train/test overlap is relevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT, not a benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, AEA registry, or similar) anywhere in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval. The study involves human participants (developers) completing tasks and being recorded, but no ethics review is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Developer demographics are reported: typically over a decade of software experience, 5 years average on their repository, 1,500 commits on average, 59% of repository lifetime. AI experience levels: 93% used LLMs, 44% used Cursor (Section 2.1). Table 7 provides per-developer statistics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section G.2.1 lists explicit eligibility criteria: 1+ year professional experience, 6+ months as active maintainer, repository must be open source with 500+ stars and 3000+ LOC, must have a list of issues. Section G describes further filtering steps."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Randomization is described: issues randomized via 'simulated fair coin flip' (Section 2.2). 25 early issues used block randomization before switching (Section E.3). Balance checks confirm successful randomization (Table 4, Table 5)."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'this study is not blinded' (Section C.2.1) — developers know which condition they are in. This is acknowledged as a limitation and investigated as a potential confound (experimentally driven overuse, Section C.2.1)."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section G.6 reports 3 developers were dropped (2 due to repository pausing development, 1 due to cheating). Section C.3.3 reports issue dropout: 7 AI-disallowed and 6 AI-allowed issues not completed, plus 24 intentionally dropped issues with reasons. Robustness checks on dropout subsets are provided."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No AI inference costs or token consumption is reported. The paper mentions providing Cursor Pro subscriptions and paying developers $150/hour but does not report API costs or token usage."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total study cost is partially disclosed ($150/hour for developers, with ~40 hour cap per developer) but total computational budget, API costs, and infrastructure costs are not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Allowing AI tools increases task completion time by 19% for experienced open-source developers.",
       "evidence": "Log-linear regression on 246 tasks (136 AI-allowed, 110 AI-disallowed) with HC3 standard errors, controlling for developer forecasts as difficulty proxy. Robust across alternative estimators (Figure 13), outcome measures (Figure 22), and developer subsets (Section C.3).",
       "supported": "strong"
     },
     {
       "claim": "Developers forecast 24% speedup from AI before tasks and estimate 20% speedup after completing the study, despite being slowed down by 19%.",
       "evidence": "Pre-task forecasts collected before randomization (Section 2.4). Post-hoc estimates collected via exit survey (Section G.5.1). Observed speedup from regression (Section D.1). Figure 1 displays all three together.",
       "supported": "strong"
     },
     {
       "claim": "Economics and ML experts predict 39% and 38% speedup respectively, overestimating even more than developers.",
       "evidence": "34 economics experts and 54 ML experts provided incentivized forecasts with detailed study information (Section 2.4, Table 6, Section G.9).",
       "supported": "strong"
     },
     {
       "claim": "Five factors contribute to slowdown: over-optimism about AI, high developer familiarity, large/complex repositories, low AI reliability, and implicit repository context.",
       "evidence": "Systematic investigation of 21 factors using subset analyses, surveys, interviews, and screen recordings (Table 1, Appendix C). Evidence includes: developer forecasts vs. actuals, subset analysis by familiarity (Figure 7), <44% generation acceptance rate, and qualitative developer reports.",
       "supported": "moderate"
     },
     {
       "claim": "Developers accept less than 44% of AI generations.",
       "evidence": "Cursor analytics data from 13 developers with METR-provided subscriptions (Section C.1.4).",
       "supported": "moderate"
     },
     {
       "claim": "75% of developers experience slowdown from AI.",
       "evidence": "Per-developer speedup estimates shown in Figure 17, using heterogeneous treatment effect estimation (Section D.4).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample size (16 developers)",
       "detail": "The study has only 16 developers, which limits statistical power for subgroup analyses. The paper acknowledges being 'not powered for statistically significant multiple comparisons when subsetting' but still presents many subset analyses (Figures 7-12) that readers might over-interpret."
     },
     {
       "flag": "No pre-registration",
       "detail": "For an RCT, the absence of pre-registration is notable. Without pre-registration, it is harder to verify that the analysis plan was not adjusted after seeing results. The 21-factor analysis in particular could reflect post-hoc hypothesis generation."
     },
     {
       "flag": "No IRB/ethics approval mentioned",
       "detail": "The study involves human participants completing recorded work tasks for pay, but no ethics board review is mentioned. This is unusual for a study with human subjects."
     },
     {
       "flag": "Non-blinded design with potential demand effects",
       "detail": "Developers knew their condition assignment. Section C.2.1 acknowledges this could lead to experimentally driven overuse of AI. Some developers reported overusing AI due to the experiment, though Figure 8 suggests similar slowdown regardless of reported AI usage patterns."
     },
     {
       "flag": "Funding source not disclosed",
       "detail": "METR's funding sources are not disclosed. While METR is not a developer of the tools being tested, their organizational focus on AI safety and capability evaluation means their funders may have interests in demonstrating (or not) AI productivity effects."
     }
   ],
   "cited_papers": [
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "RCT finding 56% speedup with Copilot on synthetic JavaScript tasks — key prior work this study contrasts with."
     },
     {
       "title": "How much does ai impact development speed? an enterprise-based randomized controlled trial",
       "authors": ["Elise Paradis", "Kate Grey", "Quinn Madison"],
       "year": 2024,
       "arxiv_id": "2410.12944",
       "relevance": "Enterprise RCT finding 21% speedup on coding tasks — another RCT this study positions against."
     },
     {
       "title": "The effects of generative ai on high-skilled work: Evidence from three field experiments with software developers",
       "authors": ["Zheyuan Cui", "Mert Demirer", "Sonia Jaffe", "Leon Musolff", "Sida Peng", "Tobias Salz"],
       "year": 2025,
       "relevance": "Three field experiments finding 26% output increase — uses non-fixed outcome measures that this study critiques."
     },
     {
       "title": "The impact of large language models on open-source innovation: Evidence from github copilot",
       "authors": ["Doron Yeverechyahu", "Raveesh Mayya", "Gal Oestreicher-Singer"],
       "year": 2025,
       "relevance": "Natural experiment finding 37% output increase using lines of code metric — critiqued by this study for non-fixed outcome measures."
     },
     {
       "title": "Significant productivity gains through programming with large language models",
       "authors": ["Thomas Weber", "Maximilian Brandmaier", "Albrecht Schmidt", "Sven Mayer"],
       "year": 2024,
       "doi": "10.1145/3661145",
       "relevance": "Found 65% increase in task requirements satisfied with AI tools on synthetic tasks."
     },
     {
       "title": "Generative AI and labour productivity: a field experiment on coding",
       "authors": ["Leonardo Gambacorta", "Han Qiu", "Shuo Shan", "Daniel M Rees"],
       "year": 2024,
       "relevance": "Field experiment at BIS finding 55% output increase — uses non-fixed outcome measure."
     },
     {
       "title": "Generative ai at work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "doi": "10.1093/qje/qjae044",
       "relevance": "Landmark study finding AI benefits less experienced workers more, compressing performance distributions."
     },
     {
       "title": "Experimental evidence on the productivity effects of generative artificial intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "doi": "10.1126/science.adh2586",
       "relevance": "Experimental evidence on generative AI productivity effects with heterogeneous effects by experience."
     },
     {
       "title": "Measuring ai ability to complete long tasks",
       "authors": ["Thomas Kwa", "Ben West", "Joel Becker"],
       "year": 2025,
       "arxiv_id": "2503.14499",
       "relevance": "METR's benchmark for measuring AI capability on long software tasks — contextualizes the gap between benchmarks and real-world impact."
     },
     {
       "title": "Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts",
       "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker"],
       "year": 2025,
       "arxiv_id": "2411.15114",
       "relevance": "Benchmark evaluating AI R&D capabilities against human experts on complex tasks."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Key software engineering benchmark referenced in context of environment complexity effects on AI performance."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Test-time compute scaling — relevant to whether more token spend could improve AI reliability in this setting."
     },
     {
       "title": "The simple macroeconomics of ai",
       "authors": ["Daron Acemoglu"],
       "year": 2024,
       "doi": "10.1093/epolic/eiae042",
       "relevance": "Skeptical economic perspective on AI's productivity impact — contextualizes the broader economic debate."
     }
   ]
 }