ai-research-survey

scan.json (24449B)

---

 {
   "paper": {
     "title": "Speed at the Cost of Quality: How Cursor AI Increases Short-Term Velocity and Long-Term Complexity in Open-Source Projects",
     "authors": ["Hao He", "Courtney Miller", "Shyam Agarwal", "Christian Kästner", "Bogdan Vasilescu"],
     "year": 2026,
     "venue": "MSR 2026",
     "arxiv_id": "2511.04427",
     "doi": "10.1145/3793302.3793349"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["observational"],
   "key_findings": "Cursor adoption leads to a statistically significant but transient velocity increase (281% lines added in month 1, dissipating by month 3) alongside persistent increases in static analysis warnings (+30%) and code complexity (+41%). Panel GMM models reveal that accumulated technical debt subsequently reduces future velocity, creating a self-reinforcing cycle. Robustness checks across adoption intensity, activity levels, other AI tools, programming languages, and alternative DiD estimators generally support these findings, though the Callaway-Sant'Anna estimator yields non-significant quality results.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at https://doi.org/10.5281/zenodo.18368661 (Section 'Data Availability')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package is referenced for data availability."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements.txt, or dependency details in the paper. SonarQube Community is mentioned as a tool but no version or setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references a replication package but does not include step-by-step reproduction instructions in the paper itself. Whether the Zenodo package contains them cannot be verified from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported in Tables 2, 3, and 6. Figure 3 shows confidence intervals around treatment effect estimates. Table 2 reports '±' ranges."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance is reported throughout with p-value thresholds (*p<0.05, **p<0.01, ***p<0.001) in Tables 2, 3, and 6. Pre-trend tests use heteroscedasticity- and cluster-robust Wald tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage changes with context (e.g., '+28.58% (±13.7%)' for lines added, '+41.64% (±7.62%)' for code complexity in Table 2). Log-transformed estimates enable interpretation as percentage changes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 806 treated repositories is determined by what was found via GitHub code search, not by power analysis. No power analysis or sample size justification is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported for all estimates in Tables 2, 3, and 6. The Borusyak et al. estimator provides robust standard errors."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The matched control group of 1,380 never-adopting repositories serves as the baseline for comparison in the DiD design."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Control repositories are matched from the same time period (2024-2025) and undergo the same temporal trends, ensuring contemporaneity."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study measuring a single treatment (Cursor adoption), not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five outcome metrics are used: commits, lines added, static analysis warnings, duplicate line density, and code complexity."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this observational mining study measuring repository-level metrics."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational causal inference study, not a predictive modeling study requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (JS/TS, Python, Go), adoption intensity (high contributor, configuration changes), activity level, and other AI tool usage (Figure 4, Figure 7, Table 7). Appendix D breaks down SonarQube warnings by 20 categories (Table 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where the Callaway-Sant'Anna estimator yields non-significant results for quality outcomes (Appendix B.3) and discusses the transient nature of velocity gains as a form of 'failure' of sustained benefit."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that duplicate line density shows no significant effect (Table 2), commits show no significant ATT with two of three estimators, and the Callaway-Sant'Anna estimator yields non-significant quality results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'transient increase in velocity', 'substantial and persistent increase in static analysis warnings and code complexity', and 'velocity slowdown' from technical debt are all supported by Tables 2, 3, and Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses difference-in-differences with propensity score matching, the Borusyak et al. imputation estimator for staggered adoption, and panel GMM with instrumental variables — all appropriate causal identification strategies for observational data. The parallel trends assumption is tested (Section 3.3.3, Figure 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5.2 explicitly bounds external validity: 'Our results may not generalize to other LLM agent assistants, proprietary software projects, and programming languages beyond the three dominant ones.' Section 5.1.3 discusses the open-source context as a specific limitation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 discusses many alternative explanations: observable adoption bias, usage intensity uncertainty, model heterogeneity, imperfect matching, contamination from other AI tools. Section 4.3 presents robustness checks against these. Section 5.1.1 discusses excitement-frustration-abandonment as an alternative mechanism for transient velocity gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes that commits and lines added are 'productivity proxies' with 'moderate-to-strong correlation with perceived productivity' (Section 3.2.1, citing prior work). It also acknowledges that quality metrics take 'the technical debt perspective' rather than claiming to measure overall quality (Section 3.2.1). The conclusion notes complexity metrics 'were designed for human-written code' and may not appropriately measure AI-generated code."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any AI models itself. It studies the effect of Cursor adoption on repositories. The paper explicitly acknowledges model/version heterogeneity across studied repositories as a feature (Section 3.5.1)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It is an observational study analyzing repository metrics."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key model parameters are reported: propensity score matching uses 1:3 nearest-neighbor matching, logistic regression specification in Equation 1, GMM uses lags 2-3 as instruments, 10-star threshold, 10,000 candidate sample per cohort. Regression specifications are fully detailed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used by the authors. The paper studies Cursor as a black-box treatment."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is thoroughly documented: GitHub code search with adaptive partitioning (Section 3.1.2), 10-star filtering threshold with cited precision, propensity score matching pipeline (Section 3.1.3), SonarQube analysis (Section 3.2.1), GHArchive collection (Section 3.2.2), multi-collinearity check removing issue comments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 'Limitations and Threats to Validity' provides extensive discussion spanning internal validity (3.5.1) and external validity (3.5.2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5.1 discusses specific threats: observable adoption through configuration files creating selection bias, uncertainty about usage intensity/persistence, model/version heterogeneity, imperfect matching (AUC 0.83-0.91 meaning incomplete variance explanation), contamination from other AI tools. These are all specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5.1 explicitly states estimates represent 'intent-to-treat' effects, not treatment-on-treated. Section 3.5.2 lists specific exclusions: other AI assistants, proprietary software, underrepresented languages. The paper also states estimates capture Cursor vs. 'state-of-the-practice' (not vs. no AI at all)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is available at Zenodo (https://doi.org/10.5281/zenodo.18368661)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection in detail: GitHub code search API with adaptive partitioning algorithm, GHArchive for monthly time series, SonarQube for code quality metrics. Time periods, inclusion criteria (10+ stars), and specific API endpoints are mentioned."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public GitHub repositories identified via code search. The repository selection process is documented in Section 3.1.2."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 23,308 Cursor files found → 3,306 non-fork repos → 806 with 10+ stars → propensity score matching → 1,380 controls. GHArchive and SonarQube pipelines described. Table 7 shows observation counts for each subset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSF awards (2206859, 2317168, 2120323), NSF Graduate Research Fellowship (DGE214073), Google research award, Digital Infrastructure Fund, and Google Cloud research credits."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from Carnegie Mellon University. They do not work for Cursor/Anysphere, so there is no direct product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF and Google fund the research. Google has a competing product (Gemini Code Assist) but is not directly invested in Cursor's outcomes. The funders do not have a direct stake in whether Cursor is found to be beneficial or harmful."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It is an observational mining study of repository-level outcomes."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a repository mining study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository inclusion criteria are documented in Section 3.1."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not a method or tool. No inference is performed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Google Cloud research credits for BigQuery analysis but does not quantify the compute budget for SonarQube analysis of 2,000+ repositories or other computational work."
       }
     }
   },
   "claims": [
     {
       "claim": "Cursor adoption leads to a statistically significant but transient increase in development velocity, with lines added increasing by 281.3% in the first month but gains dissipating after two months.",
       "evidence": "Table 2 shows ATT of +28.58% for lines added overall; Figure 3 shows horizon-specific effects of 281.3% in month 0 and 48.4% in month 1, with non-significant effects afterward. All five outcomes pass pre-trend tests at p<0.05.",
       "supported": "strong"
     },
     {
       "claim": "Cursor adoption leads to a persistent increase of 30.3% in static analysis warnings and 41.6% in code complexity.",
       "evidence": "Table 2 reports ATT estimates with p<0.001. Figure 3 shows effects persisting through month +6. However, the Callaway-Sant'Anna estimator yields non-significant results for these quality outcomes (Table 6, Appendix B.3), weakening the evidence.",
       "supported": "moderate"
     },
     {
       "claim": "Technical debt accumulation (static analysis warnings and code complexity) causally reduces future development velocity.",
       "evidence": "Table 3 panel GMM models show a 100% increase in code complexity causes 64.5% decrease in lines added, and 100% increase in static analysis warnings causes 50.3% decrease. Sargan tests (p>0.05) and AR(2) tests (p>0.05) validate instruments.",
       "supported": "strong"
     },
     {
       "claim": "Cursor adoption leads to inherently more complex code, even after controlling for velocity dynamics.",
       "evidence": "Table 3, column L→C shows Cursor coefficient of 0.086 (p<0.001) for code complexity even with velocity controls and codebase size dynamics accounted for.",
       "supported": "strong"
     },
     {
       "claim": "Cursor adoption has no significant effect on duplicate line density.",
       "evidence": "Table 2 shows ATT of +7.03% which is marked significant but Figure 3 shows inconsistent temporal patterns. Heavy adopters show modest increases (Figure 4).",
       "supported": "weak"
     },
     {
       "claim": "The effects are amplified in repositories with higher sustained Cursor usage.",
       "evidence": "Figure 4, Row 1 shows stronger quality effects in 'High Contributor Adoption' and 'Cursor Configuration Changes' subsets compared to the full sample.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Estimator disagreement on quality outcomes",
       "detail": "The Callaway-Sant'Anna estimator finds non-significant effects for all three quality outcomes (static analysis warnings, code complexity, duplicate line density), while Borusyak et al. and TWFE find significant positive effects. The paper reports the most favorable estimator (Borusyak) in the main text. Appendix B.3 discusses this but the resolution is that Callaway-Sant'Anna has lower power due to small cohorts — this is plausible but the disagreement is non-trivial."
     },
     {
       "flag": "Proxy validity of .cursorrules for adoption",
       "detail": "Using .cursorrules files as a proxy for Cursor adoption introduces multiple biases: only repositories that committed configuration files are included, timing may not reflect actual adoption start, and the paper cannot distinguish between Composer (multi-file editing) and full agentic mode. The paper acknowledges this extensively but the estimates are fundamentally limited by this proxy."
     },
     {
       "flag": "SonarQube warning analysis is descriptive only",
       "detail": "Appendix D's breakdown of warning categories by pre/post adoption (Table 8) is described as a 'convenience sample' with known architectural limitations preventing precise tracking. These descriptive statistics should not be interpreted causally but may be read as such."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Controlled experiment finding that early-2025 AI tools including Cursor do not help experienced developers, contradicting self-reported productivity gains."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "RCT measuring Copilot's productivity impact (56% faster task completion), foundational reference for AI coding productivity studies."
     },
     {
       "title": "On the use of agentic coding: An empirical study of pull requests on GitHub",
       "authors": ["Miku Watanabe", "Hao Li", "Yutaro Kashiwa", "Brittany Reid", "Hajimu Iida", "Ahmed E Hassan"],
       "year": 2025,
       "arxiv_id": "2509.14745",
       "relevance": "Examines 567 Claude Code pull requests finding 83.8% acceptance rate, complementary to this paper's project-level analysis."
     },
     {
       "title": "The Impact of Large Language Models on Open-source Innovation: Evidence from GitHub Copilot",
       "authors": ["Doron Yeverechyahu", "Raveesh Mayya", "Gal Oestreicher-Singer"],
       "year": 2024,
       "relevance": "DiD study estimating 17.82% increase in Python package releases after Copilot availability, closest prior work to this study's methodology."
     },
     {
       "title": "The effects of generative AI on high skilled work: Evidence from three field experiments with software developers",
       "authors": ["Zheyuan Kevin Cui", "Mert Demirer", "Sonia Jaffe", "Leon Musolff", "Sida Peng", "Tobias Salz"],
       "year": 2024,
       "relevance": "Field experiments at Microsoft, Accenture, Cisco reporting 22-36% productivity gains from AI coding tools."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Foundational study on security vulnerabilities in AI-generated code from Copilot."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2023,
       "relevance": "Controlled study on security implications of AI coding assistants with human participants."
     },
     {
       "title": "The Impact of Generative AI on Collaborative Open-Source Software Development: Evidence from GitHub Copilot",
       "authors": ["Fangchen Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "arxiv_id": "2410.02091",
       "relevance": "Estimates only 6.5% project-level productivity increase from Copilot using proprietary backend data."
     },
     {
       "title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
       "authors": ["Elise Paradis", "Kate Grey", "Quinn Madison"],
       "year": 2025,
       "relevance": "Enterprise RCT on AI impact on development speed, one of few studies on agentic tools beyond Copilot."
     },
     {
       "title": "Who is using AI to code? Global diffusion and impact of generative AI",
       "authors": ["Simone Daniotti", "Johannes Wachs", "Xiangnan Feng", "Frank Neffke"],
       "year": 2025,
       "arxiv_id": "2506.08945",
       "relevance": "Large-scale study using neural classifier to identify AI-generated code on GitHub, finding 30% AI use raises quarterly commits by 2.4%."
     },
     {
       "title": "Code with Me or for Me? How Increasing AI Automation Transforms Developer Workflows",
       "authors": ["Valerie Chen", "Ameet Talwalkar", "Robert Brennan", "Graham Neubig"],
       "year": 2025,
       "arxiv_id": "2507.08149",
       "relevance": "Qualitative research documenting developer challenges and workflow changes with AI coding assistance at different automation levels."
     },
     {
       "title": "Intuition to Evidence: Measuring AI's True Impact on Developer Productivity",
       "authors": ["Anand Kumar"],
       "year": 2025,
       "arxiv_id": "2509.19708",
       "relevance": "Enterprise study on AI's impact on developer productivity with field evidence."
     }
   ]
 }