ai-research-survey

scan.json (28517B)

---

 {
   "paper": {
     "title": "AI-Assisted Programming Decreases the Productivity of Experienced Developers by Increasing the Technical Debt and Maintenance Burden",
     "authors": [
       "Feiyang (Amber) Xu",
       "Poonacha K. Medappa",
       "Murat M. Tunc",
       "Martijn Vroegindeweij",
       "Jan C. Fransoo"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.10165",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials link is provided anywhere in the paper. The data was collected via GitHub API but no analysis code is shared."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or data archive is provided. The authors describe collecting data via GitHub's API for Microsoft-owned repositories but do not release the dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, requirements files, or software dependencies are mentioned. The econometric software used for the DiD analysis is not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While the methodology is described in detail (Section 3), there are no instructions for replicating the data collection or analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported in the lead-lag analysis (Table 10) and subgroup analyses (Tables 11-14). Figure 5 shows 95% confidence intervals on the subgroup coefficient plots. Standard errors are reported in parentheses in all regression tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance is tested throughout using t-statistics and p-values. Regression tables (Tables 5, 6, 8) report significance levels (*p<0.1; **p<0.05; ***p<0.01). The lead-lag analysis (Table 10) reports full t-statistics and p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage changes derived from log-transformed coefficients. For example, '17.7% increase in lines of code added (exp(0.163)-1)', '2.4% more code rework (exp(0.024)-1)', '19% decrease in commits' for core contributors, and '43.5% increase in commit activity' for peripheral contributors (Sections 4.1, 4.2, 6.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or explicit justification for the sample size is provided. The sample includes 2,755 repositories and 1,699 contributors, which are described but not justified in terms of statistical adequacy. No discussion of whether these sample sizes are sufficient for the subgroup analyses."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in descriptive statistics tables (Tables 3, 4). Standard errors are reported for all regression coefficients. The study uses clustered robust standard errors at the project/individual level to account for heteroskedasticity."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The DiD design inherently includes a baseline comparison: the control group consists of repositories/contributors using non-Copilot-endorsed languages (R, C, C#, C++, Java, PHP, Scala). Pre-treatment periods serve as the temporal baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The control group uses contemporary non-Copilot-endorsed languages from the same time period and platform (Microsoft-owned GitHub repositories), ensuring contemporaneous comparison. The study also references and compares with concurrent work by Yeverechyahu et al. (2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational econometric study using a natural experiment design, not a system with components that could be ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple outcome metrics are used: lines of code added, commits, PRs (development activity); PR rework (technical debt); PR reviews and PR reviewed repositories (maintenance effort). These are measured at both project and individual levels."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant here. This is an econometric analysis of developer behavior data, not a system whose outputs require human judgment."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a predictive modeling study. The DiD framework analyzes the full dataset without train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided. Figure 5 and Tables 11-14 show results by contributor subgroup (0-25%, 25-50%, 50-75%, 75-100% based on pretreatment commits). The core vs. peripheral contributor analysis (Table 6) provides further breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where effects are not significant: peripheral contributors show no significant change in PR reviews or PR reviewed repos, and the 50-75% subgroup shows no significant change in commits or PRs. The paper acknowledges that Copilot usage cannot be identified at the individual level, which is a limitation of the analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that core contributors experienced a 19% decrease in commit activity post-Copilot. Several subgroup coefficients are not statistically significant (e.g., 50-75% subgroup for commits, p=0.532). The core result itself is arguably a negative finding about AI-assisted programming."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported by results: productivity increases (Table 5), primarily driven by peripheral developers (Tables 11-14, Figure 5), code requires more rework (Table 5, PR rework coefficient), core developers review 6.5% more code (Table 6, PR Review coefficient exp(0.06)-1), and show a 19% drop in productivity (Table 6, Core Contributor x Copilot coefficient for commits). All claims are traceable to specific regression results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a Difference-in-Differences (DiD) design exploiting the natural experiment of Copilot's staggered language endorsement (Section 3.1). The parallel trends assumption is tested with an event study (Section 3.4.2, Figure 4, Table 10). Robustness checks include CEM matching (Section 5.1) and Oster sensitivity analysis for omitted variable bias (Section 5.2, delta=7.82). This is an appropriate causal identification strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about 'AI-Assisted Programming' decreasing productivity of 'Experienced Developers', but the study only examines Microsoft-owned OSS repositories during the Copilot technical preview period (June 2021 - July 2022). The paper does not adequately bound its claims to this specific setting. The discussion extends to 'knowledge-intensive industries' generally (Section 7.2) without sufficient hedging."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses alternative explanations through multiple robustness checks: CEM matching to address selection bias (Section 5.1), Oster sensitivity analysis for omitted variable bias (Section 5.2, delta=7.82), and parallel trends testing (Section 3.4.2). The DiD design controls for time-invariant repository characteristics and common temporal shocks. However, the paper acknowledges it cannot observe individual Copilot usage."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GitHub Copilot was 'developed on OpenAI's GPT-3 model' but does not specify the exact model version, API version, or snapshot date. No specific Copilot version is identified beyond 'technical preview.'"
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study analyzing the effects of Copilot adoption on developer behavior. The authors did not use prompting as part of their methodology; they analyzed naturally occurring development activity."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an econometric observational study, not an ML system. While the paper could report econometric model specifications in more detail, the 'hyperparameters' concept applies to ML/LLM settings. The regression specifications and fixed effects are described in Section 3.4."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is an econometric analysis of developer behavior, not an AI system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing pipeline is documented: data collected via GitHub API (Section 3.2), treatment/control groups defined by programming language endorsement, 2,755 repositories total (1,660 treatment, 1,095 control), individual-level filtering from 37,334 contributors to 5,308 to 1,699 (footnote 14 explains the filtering criteria), data aggregated to monthly panels (Section 3.1), and log transformations applied to dependent variables (Table 2 note)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The discussion section (Section 7) focuses on contributions and future research but does not include a separate subsection addressing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed in a structured manner. While the paper addresses some methodological concerns implicitly through robustness checks (matching, Oster analysis), it does not explicitly enumerate threats such as the inability to observe individual Copilot usage, potential confounds from concurrent events, or limitations of the programming-language-based treatment assignment."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The study is limited to Microsoft-owned OSS repositories during the technical preview period, but the paper does not clearly bound its claims to this setting. The title and discussion generalize to 'AI-Assisted Programming' broadly without stating scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is not made available. No data download link, supplementary data files, or database access is provided. The data was collected via GitHub's API but is not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in Section 3.2: data collected via GitHub's API service, focusing on Microsoft-owned OSS repositories. The observation period is July 2020 to July 2022 (12 months before and after Copilot technical preview). Treatment and control groups are defined by programming language endorsement. Repository-level and individual-level datasets are described with sample sizes."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. This is a mining study of publicly available GitHub data. The data source is a standard platform (GitHub) with selection criteria described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: collection from GitHub API, treatment/control group assignment by language, filtering from 37,334 contributors to 5,308 (active in 3+ repos) to 1,699 (language filter) as explained in footnote 14. Monthly aggregation of measures is described. Log transformation with +1 offset for zeros is documented (Table 2 note). CEM matching sample reduction from 2,755 to 2,510 repositories is documented (Section 5.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section listing grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Tilburg University, the Netherlands, with institutional email addresses provided. Tilburg University is not the company whose product is being evaluated (Microsoft/GitHub Copilot)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so it is impossible to assess whether the funder is independent of the outcome. The absence of a funding disclosure makes this unanswerable in the affirmative."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on any benchmark. It is an observational study examining the effects of Copilot adoption on developer behavior and code quality metrics."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on any benchmark. It analyzes naturally occurring developer behavior data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on any benchmark. It is a mining/observational study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. This is a mining study of publicly available GitHub repository data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines public GitHub data on repository-level and contributor-level activity."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were directly recruited. The study analyzes GitHub contributors' activity data, not demographics of recruited subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses inclusion/exclusion criteria for repositories and contributors (described in Section 3.2 and footnote 14), but these are data selection criteria, not human recruitment criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'randomization' in this study comes from the natural experiment of Copilot's language endorsement, not from assignment of human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Contributor filtering from 37,334 to 1,699 is described as data selection criteria."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational econometric study, not an AI system. There is no inference cost to report for the authors' methodology."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an econometric analysis, not a computationally intensive AI system. The compute requirements for running DiD regressions are trivial."
       }
     }
   },
   "claims": [
     {
       "claim": "Repositories using Copilot-endorsed languages experienced a 17.7% increase in lines of code added, 4.1% increase in commits, and 4.3% increase in PRs after Copilot's introduction.",
       "evidence": "Table 5: DiD coefficients of 0.163*** (code added), 0.04* (commits), and 0.042*** (PRs) with project and month fixed effects. N=66,120-66,168.",
       "supported": "strong"
     },
     {
       "claim": "PRs submitted to Copilot-endorsed repositories required 2.4% more rework, indicating increased technical debt.",
       "evidence": "Table 5: PR Rework coefficient of 0.024** (p<0.05) with PR controls, project FE, and month FE. N=66,168, Adj. R2=0.81.",
       "supported": "moderate"
     },
     {
       "claim": "Core contributors decreased their commit activity by 19% and increased PR reviews by 6.5% after Copilot's introduction.",
       "evidence": "Table 6 and Section 6.2: Core Contributor x Copilot interaction coefficient of -0.357*** for commits (exp(-0.357+0.142)-1 ≈ -19%) and 0.06* for PR Reviews. Figure 5 shows subgroup breakdowns. The PR review effect is significant only at the 10% level.",
       "supported": "moderate"
     },
     {
       "claim": "Peripheral contributors (bottom 25%) increased commit activity by 43.5% and submitted 17.7% more PRs.",
       "evidence": "Table 13: 0-25% subgroup coefficient of 0.3614*** (t=8.08, p<0.001) for commits. Table 14: 0-25% subgroup coefficient of 0.1630*** (t=5.15, p<0.001) for PRs. Both highly significant.",
       "supported": "strong"
     },
     {
       "claim": "The treatment effect on PR rework is robust to omitted variable bias, with an Oster delta of 7.82.",
       "evidence": "Table 9: Oster sensitivity analysis shows delta=7.82, meaning unobservables would need to be 7.82 times as influential as observables to explain away the result. Rmax set to 1.00.",
       "supported": "strong"
     },
     {
       "claim": "The parallel trends assumption holds for the DiD design.",
       "evidence": "Table 10 and Figure 4: All pre-treatment period coefficients (b12 through b2) are statistically insignificant (p-values ranging from 0.170 to 0.987), supporting the parallel trends assumption.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational"
   ],
   "key_findings": "This study uses a Difference-in-Differences design exploiting GitHub Copilot's staggered programming language endorsement to analyze its impact on Microsoft-owned OSS projects. While Copilot-endorsed repositories saw increased development productivity (17.7% more code added, 4.3% more PRs), they also experienced 2.4% more PR rework, indicating technical debt accumulation. The effects are heterogeneous: peripheral contributors (bottom 25%) increased commits by 43.5%, while core contributors (top 25%) decreased commits by 19% and increased PR reviews by 6.5%, suggesting a redistribution of effort from development to maintenance among experienced developers.",
   "red_flags": [
     {
       "flag": "Cannot observe individual Copilot usage",
       "detail": "The study cannot identify which individual contributors actually used Copilot. Treatment assignment is based on programming language of the repository, not verified tool usage. Contributors to Copilot-endorsed language repositories may not have used Copilot, and some control group members may have used other AI tools."
     },
     {
       "flag": "Overgeneralized title and claims",
       "detail": "The title claims 'AI-Assisted Programming Decreases the Productivity of Experienced Developers' but the study examines only Microsoft-owned OSS repositories during the Copilot technical preview period (June 2021 - July 2022) with a specific set of programming languages. The generalization to all AI-assisted programming and all experienced developers is not warranted."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "Despite strong econometric methodology, the paper lacks a dedicated limitations or threats-to-validity section. Key concerns such as the intent-to-treat nature of the analysis, potential confounds from other concurrent changes at Microsoft, and external validity are not systematically addressed."
     },
     {
       "flag": "Small effect size on key outcome",
       "detail": "The core technical debt claim rests on a 2.4% increase in PR rework, which is statistically significant at the 5% level but substantively small. The practical significance of this effect is not discussed. The paper focuses attention on the larger heterogeneous effects (19% decrease for core, 43.5% increase for peripheral) which are more striking but come from subgroup analyses."
     },
     {
       "flag": "No data or code released",
       "detail": "Despite using publicly available GitHub API data, neither the dataset nor the analysis code is released, making independent verification impossible."
     },
     {
       "flag": "Missing funding and conflict of interest disclosures",
       "detail": "The paper contains no funding disclosure, no competing interests statement, and no financial interests declaration. While the authors are at Tilburg University (not a direct conflict), the absence of any such disclosures is a transparency gap."
     }
   ],
   "cited_papers": [
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from Github Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Foundational RCT on Copilot productivity showing 55.8% faster task completion, directly motivates this study's investigation of secondary effects."
     },
     {
       "title": "The Impact of Large Language Models on Open-source Innovation: Evidence from GitHub Copilot",
       "authors": ["Doron Yeverechyahu", "Raveesh Mayya", "Gal Oestreicher-Singer"],
       "year": 2024,
       "arxiv_id": "2409.08379",
       "relevance": "Uses similar DiD design with Copilot language endorsement to study innovation effects; finds shift toward routine and incremental changes in OSS."
     },
     {
       "title": "The Impact of Generative AI on Collaborative Open-Source Software Development: Evidence from GitHub Copilot",
       "authors": ["Frank Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "arxiv_id": "2410.02091",
       "relevance": "Studies Copilot's impact on collaborative OSS development, finding increased contributions but also increased coordination time for code integration."
     },
     {
       "title": "GitHub Copilot AI Pair Programmer: Asset or Liability?",
       "authors": ["Arghavan Moradi Dakhel", "Vahid Majdinasab", "Amin Nikanjam", "Foutse Khomh", "Michel C. Desmarais", "Zhen Ming (Jack) Jiang"],
       "year": 2023,
       "relevance": "Evaluates quality of Copilot-generated code and cautions that novice programmers may place undue trust in AI-generated code."
     },
     {
       "title": "Generative AI and the Nature of Work",
       "authors": ["Manuel Hoffmann", "Sam Boysel", "Frank Nagle", "Sida Peng", "Kevin Xu"],
       "year": 2025,
       "relevance": "Shows GitHub Copilot access reallocates developers' effort toward core coding tasks and away from project management, complementary to this study's findings."
     },
     {
       "title": "Does AI-Assisted Coding Deliver? A Difference-in-Differences Study of Cursor's Impact on Software Projects",
       "authors": ["Han He", "Chris Miller", "Siddharth Agarwal", "Christian Kästner", "Bogdan Vasilescu"],
       "year": 2025,
       "arxiv_id": "2511.04427",
       "relevance": "Uses DiD design to study Cursor's impact on software projects, extending the causal analysis framework to another AI coding assistant."
     },
     {
       "title": "Generative AI at Work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey R. Raymond"],
       "year": 2025,
       "relevance": "Foundational study on GenAI productivity effects in customer support, showing less-experienced workers benefit more -- a pattern this paper finds in OSS."
     },
     {
       "title": "Security Vulnerabilities in AI-Generated Code: A Large-Scale Analysis of Public GitHub Repositories",
       "authors": ["Marcel Schreiber", "Paul Tippe"],
       "year": 2025,
       "relevance": "Documents security vulnerabilities in AI-generated code and technical debt from latent security weaknesses, directly relevant to code quality concerns."
     },
     {
       "title": "Good Vibrations? A Qualitative Study of Co-Creation, Communication, Flow, and Trust in Vibe Coding",
       "authors": ["Vlada Pimenova", "Sarah Fakhoury", "Christian Bird", "Margaret-Anne Storey", "Madeline Endres"],
       "year": 2025,
       "arxiv_id": "2509.12491",
       "relevance": "Qualitative study of vibe coding practices documenting speed-quality tradeoffs and maintenance challenges that this study quantifies."
     },
     {
       "title": "When Combinations of Humans and AI are Useful: A Systematic Review and Meta-Analysis",
       "authors": ["Michael Vaccaro", "Abdullah Almaatouq", "Thomas Malone"],
       "year": 2024,
       "relevance": "Meta-analytic evidence on human-AI collaboration effectiveness, relevant to understanding AI's role in development teams."
     },
     {
       "title": "Technical Debt and Firm Performance",
       "authors": ["Rajiv D. Banker", "Yi Liang", "Narayan Ramasubbu"],
       "year": 2021,
       "doi": "10.1287/mnsc.2020.3tried",
       "relevance": "Foundational study on economic consequences of technical debt that this paper extends to the AI-assisted development context."
     },
     {
       "title": "Vibe Coding in Practice: Motivations, Challenges, and a Future Outlook – a Grey Literature Review",
       "authors": ["Ahmed Fawzy", "Ahmad Tahir", "Kelly Blincoe"],
       "year": 2025,
       "arxiv_id": "2510.00328",
       "relevance": "Documents the speed-quality paradox in vibe coding, finding that rapid AI-assisted development creates maintenance demands."
     }
   ]
 }