ai-research-survey

scan.json (25335B)

---

 {
   "scan_version": 3,
   "active_modules": [],
   "paper": {
     "title": "Speed at the Cost of Quality: How Cursor AI Increases Short-Term Velocity and Long-Term Complexity in Open-Source Projects",
     "authors": [
       "Hao He",
       "Courtney Miller",
       "Shyam Agarwal",
       "Christian Kästner",
       "Bogdan Vasilescu"
     ],
     "year": 2026,
     "venue": "MSR '26",
     "arxiv_id": "2511.04427",
     "doi": "10.1145/3793302.3793349"
   },
   "methodology_tags": [
     "observational"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package provided at https://doi.org/10.5281/zenodo.18368661 (Data Availability section)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package at Zenodo includes the dataset. The paper states 'We provide a replication package for this paper.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment setup. The paper mentions SonarQube Community server and BigQuery but does not specify software versions or dependencies for reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a replication package link but does not include step-by-step reproduction instructions in the paper itself. Whether the Zenodo archive contains them is not stated."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors reported in Tables 2 and 3. Figure 3 shows confidence intervals around treatment effect estimates. Table 2 includes ± percentage change intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance reported throughout with p-value thresholds (p<0.05, p<0.01, p<0.001). Pre-trend Wald tests, Sargan tests, and AR(2) tests all reported with p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes reported as percentage changes with baseline context (e.g., '28.58% increase in lines added', '41.64% increase in code complexity'). Table 2 provides log-transformed estimates and their percentage-change interpretations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 806 treated repositories is determined by data availability (all qualifying repos found via GitHub search), not by power analysis. No power analysis or sample size justification is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors reported in parentheses for all regression estimates in Tables 2 and 3. Confidence bands shown in event study plots (Figures 3 and 4)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The matched control group of 1,380 never-adopter repositories serves as the baseline comparison, constructed via propensity score matching (Section 3.1.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Control repositories are contemporaneous — matched from the same time period and observed over the same January 2024 to August 2025 window."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple robustness checks function as ablations: high contributor adoption subset, cursor configuration changes subset, activity-level subsets, and other-AI-tools subsets (Section 4.3, Figure 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five outcome metrics: commits, lines added, static analysis warnings, duplicate line density, and code complexity (Section 3.2.1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a large-scale observational mining study of repository metrics. Human evaluation of code quality outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a causal inference study using observational data, not a prediction task. No train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by outcome variable (5 metrics), by time horizon (Figures 3, 4), and by subgroup (high adoption, active repos, other AI tools). Table 1 provides descriptive statistics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where effects are weak or absent (e.g., no significant effect on commits, no significant effect on duplicate line density). Section 5.1 discusses the transient nature of velocity gains as a 'failure' of sustained benefit."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results reported: velocity gains in lines added not sustained (main effect on commits insignificant), duplicate line density effect insignificant (Table 2), velocity increase from lines added does not significantly cause more warnings in GMM (Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'statistically significant, large, but transient increase in velocity' and 'substantial and persistent increase in static analysis warnings and code complexity' are supported by Tables 2-3 and Figures 3-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims justified via difference-in-differences design with staggered adoption (Borusyak et al. estimator), propensity score matching, pre-trend tests, and panel GMM with instrumental variables. Multiple robustness checks address alternative explanations (Section 4.3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5.2 explicitly bounds generalization: 'Our results may not generalize to other LLM agent assistants, proprietary software projects, and programming languages beyond the three dominant ones.' Section 5.1.3 discusses open-source-specific contextual factors."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations discussed: excitement-frustration-abandonment cycle (Section 5.1.1), inactive repositories driving transient gains (Section 4.3 Row 2), contamination from other AI tools (Section 4.3 Row 3), non-compliance (Section 4.3 Row 1). Each is tested with robustness checks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames its measurements as proxies: 'development velocity' is operationalized as commits and lines added, 'software quality' as static-analysis warnings, cognitive complexity, and duplicate density. Figure 1 shows a theory diagram with labeled causal arrows. The limitations section acknowledges these are imperfect proxies, and the title itself ('Speed at the Cost of Quality') uses shorthand that the body unpacks with specific metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The study does not use LLMs as part of its methodology. It studies the effect of Cursor adoption by others. The paper acknowledges it lacks information on which Cursor version or LLM backend each repository used (Section 3.5.1)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in the study methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key modeling parameters reported: 1:3 nearest-neighbor matching ratio, propensity score model specification (Equation 1), 6-month lag structure, 10-star threshold, lags 2-3 as instruments for GMM, AUC values 0.83-0.91 for propensity models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used in the study methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data pipeline documented: GitHub code search API with adaptive partitioning (Section 3.1.2), 10-star filter yielding 806 repos, propensity score matching yielding 1,380 controls (Section 3.1.3), monthly metric collection from GHArchive and SonarQube (Section 3.2), multicollinearity check removing issue comments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 'Limitations and Threats to Validity' provides extensive discussion of internal validity (Section 3.5.1) and external validity (Section 3.5.2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: observable adoption proxy (config files don't capture all users), usage intensity uncertainty, model/version heterogeneity, imperfect matching with unobserved confounders, contamination from other AI tools. Each is study-specific, not generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: estimates are ITT effects, results reflect Cursor adoption vs. state-of-practice (not vs. no AI), may not generalize to enterprise, other tools, or other languages. Section 3.5.1 states estimates 'should be interpreted as the impact of systematic Cursor adoption compared to the current state-of-the-practice.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Replication package at Zenodo (doi.org/10.5281/zenodo.18368661). Data sourced from public GHArchive and GitHub API, which are independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection described in detail: GitHub code search API with adaptive partitioning algorithm for .cursorrules files (Section 3.1.2), GHArchive for time series (Section 3.1.3), SonarQube for code quality metrics (Section 3.2.1). Time period: January 2024 to August 2025."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from mining public GitHub repositories. Standard benchmark NA."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 23,308 Cursor files across 3,306 non-fork repos → 10-star filter → 806 repos (Section 3.1.2). Control group: population of all ≥10 star repos → 10,000 candidates per month → propensity score matching → 1,380 controls (Section 3.1.3). Filtering counts provided at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSF awards (2206859, DGE214073, 2317168, 2120323), research awards from Google and the Digital Infrastructure Fund, and Google Cloud research credits."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with Carnegie Mellon University, clearly stated. No affiliation with Cursor/Anysphere."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funded by NSF and research awards. Google provided credits for BigQuery analysis. Neither NSF nor Google has a direct stake in whether Cursor improves or degrades code quality. The study evaluates a third-party product (Cursor)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The study does not evaluate a pre-trained model's capability on any benchmark. It is a mining study measuring the impact of tool adoption on repository-level metrics."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study. No model is being tested on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a repository mining study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository inclusion criteria are documented in data_preprocessing_documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "The study does not propose a method with inference costs. It is an empirical observational study."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Google Cloud credits mentioned for BigQuery analysis and SonarQube used locally, but no quantification of total compute budget (e.g., how long SonarQube analysis took across all repos, BigQuery costs)."
       }
     }
   },
   "claims": [
     {
       "claim": "Cursor adoption leads to a statistically significant but transient increase in development velocity, with 281.3% increase in lines added in the first month that dissipates after two months.",
       "evidence": "Table 2 shows 28.58% average increase in lines added (p<0.001). Figure 3 event study plots show significant effects only in months 0-1 post-adoption, with non-significant effects from month 2 onward.",
       "supported": "strong"
     },
     {
       "claim": "Cursor adoption leads to persistent increases in static analysis warnings (30.3%) and code complexity (41.6%).",
       "evidence": "Table 2 reports ATT estimates with p<0.001 for both metrics. Figure 3 shows sustained significant effects across months 0-6 post-adoption for both metrics.",
       "supported": "strong"
     },
     {
       "claim": "Code complexity increases even after controlling for velocity dynamics, indicating Cursor generates inherently more complex code.",
       "evidence": "Table 3 GMM model shows Cursor has significant coefficient (0.086, p<0.001) for code complexity after controlling for lines added, lines of code, and temporal dynamics.",
       "supported": "strong"
     },
     {
       "claim": "Accumulated technical debt (static analysis warnings and code complexity) decreases future development velocity, creating a self-reinforcing cycle.",
       "evidence": "Table 3 GMM models show 100% increase in code complexity causes 64.5% decrease in lines added, and 100% increase in warnings causes 50.3% decrease. Sargan and AR(2) tests validate the instrument conditions.",
       "supported": "strong"
     },
     {
       "claim": "Effects are amplified in repositories with higher sustained Cursor usage and attenuated in repositories using multiple AI tools.",
       "evidence": "Figure 4 robustness checks show stronger quality degradation in high-contributor-adoption and cursor-configuration-changes subsets, and slightly weaker effects when other AI tools are present.",
       "supported": "moderate"
     }
   ],
   "key_findings": "Cursor adoption produces a large but transient velocity increase (281% more lines added in month 1, dissipating by month 2) alongside persistent quality degradation (30% more static analysis warnings, 42% more code complexity). Panel GMM analysis shows code complexity increases independently of velocity effects, and accumulated technical debt reduces future velocity, creating a self-reinforcing cycle. Robustness checks across adoption intensity, activity levels, and AI tool contamination confirm the findings.",
   "red_flags": [
     {
       "flag": "Proxy-based treatment identification",
       "detail": "Cursor adoption is identified via .cursorrules files in git history, which is an imperfect proxy. Developers can use Cursor without committing config files, and committing config files doesn't guarantee sustained use. The authors acknowledge this is an ITT estimate."
     },
     {
       "flag": "Quality metrics may not capture AI-specific patterns",
       "detail": "The authors themselves note (Section 6) that complexity metrics designed for human-written code may not appropriately penalize AI-generated patterns. SonarQube cognitive complexity may flag structurally valid but verbose AI code differently than genuinely problematic complexity."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": [
         "Joel Becker",
         "Nate Rush",
         "Elizabeth Barnes",
         "David Rein"
       ],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding that AI tools including Cursor do not help experienced OSS developers solve real tasks faster, contradicting self-reported productivity claims."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": [
         "Sida Peng",
         "Eirini Kalliamvakou",
         "Peter Cihon",
         "Mert Demirer"
       ],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Foundational RCT measuring Copilot's productivity impact (56% task completion improvement), a key comparison for this study's findings."
     },
     {
       "title": "On the use of agentic coding: An empirical study of pull requests on GitHub",
       "authors": [
         "Miku Watanabe",
         "Hao Li",
         "Yutaro Kashiwa",
         "Brittany Reid",
         "Hajimu Iida",
         "Ahmed E Hassan"
       ],
       "year": 2025,
       "arxiv_id": "2509.14745",
       "relevance": "Empirical study of Claude Code PRs on GitHub finding 83.8% acceptance rate, complementary to this study's project-level analysis."
     },
     {
       "title": "The effects of generative AI on high skilled work: Evidence from three field experiments with software developers",
       "authors": [
         "Zheyuan Kevin Cui",
         "Mert Demirer",
         "Sonia Jaffe",
         "Leon Musolff",
         "Sida Peng",
         "Tobias Salz"
       ],
       "year": 2024,
       "relevance": "Field experiments at Microsoft/Accenture/Cisco on AI coding productivity (22-36% improvements), key enterprise comparison."
     },
     {
       "title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
       "authors": [
         "Elise Paradis",
         "Kate Grey",
         "Quinn Madison"
       ],
       "year": 2025,
       "relevance": "Enterprise RCT on AI coding tool impact, providing enterprise context that this OSS study lacks."
     },
     {
       "title": "Intuition to Evidence: Measuring AI's True Impact on Developer Productivity",
       "authors": [
         "Anand Kumar"
       ],
       "year": 2025,
       "arxiv_id": "2509.19708",
       "relevance": "Enterprise study indicating different adoption patterns than open-source, relevant context for generalization."
     },
     {
       "title": "Who is using AI to code? Global diffusion and impact of generative AI",
       "authors": [
         "Simone Daniotti",
         "Johannes Wachs",
         "Xiangnan Feng",
         "Frank Neffke"
       ],
       "year": 2025,
       "arxiv_id": "2506.08945",
       "relevance": "Large-scale study using neural classifier to identify AI code on GitHub, finding 30% AI use raises commits by 2.4%."
     },
     {
       "title": "The Impact of Generative AI on Collaborative Open-Source Software Development: Evidence from GitHub Copilot",
       "authors": [
         "Fangchen Song",
         "Ashish Agarwal",
         "Wen Wen"
       ],
       "year": 2024,
       "arxiv_id": "2410.02091",
       "relevance": "DiD study of Copilot finding only 6.5% project-level productivity increase from proprietary backend data."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": [
         "Hammond Pearce",
         "Baleegh Ahmad",
         "Benjamin Tan",
         "Brendan Dolan-Gavitt",
         "Ramesh Karri"
       ],
       "year": 2022,
       "relevance": "Early study on security vulnerabilities in Copilot-generated code, prior generation to the agentic tools studied here."
     },
     {
       "title": "Code with Me or for Me? How Increasing AI Automation Transforms Developer Workflows",
       "authors": [
         "Valerie Chen",
         "Ameet Talwalkar",
         "Robert Brennan",
         "Graham Neubig"
       ],
       "year": 2025,
       "arxiv_id": "2507.08149",
       "relevance": "Qualitative research on developer challenges with AI coding, supports the excitement-frustration-abandonment interpretation."
     },
     {
       "title": "Ai agents, productivity, and higher-order thinking: Early evidence from software development",
       "authors": [
         "Suproteem K Sarkar"
       ],
       "year": 2025,
       "relevance": "Early evidence on agentic AI productivity in software development, directly relevant to agentic tool impact."
     },
     {
       "title": "Generative AI and the Nature of Work",
       "authors": [
         "Manuel Hoffmann",
         "Sam Boysel",
         "Frank Nagle",
         "Sida Peng",
         "Kevin Xu"
       ],
       "year": 2024,
       "relevance": "Study on how LLM adoption increases work autonomy, a proposed mechanism for productivity increases."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly informs teams adopting AI coding tools about quality tradeoffs and suggests concrete process adaptations like scaling QA with velocity."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "Directly contradicts the widely-held '10x productivity' narrative around Cursor/AI coding tools, showing velocity gains vanish after two months while technical debt persists."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about code quality degradation and security warnings increasing, but safety/risk is secondary to the productivity narrative."
     },
     "drama_conflict": {
       "score": 3,
       "justification": "Directly challenges Cursor's productivity claims and the broader AI coding hype with empirical evidence of a self-reinforcing technical debt cycle — a classic 'emperor has no clothes' paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Replication package available on Zenodo with data and code, reproducible with moderate effort for researchers familiar with econometric methods."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Cursor is one of the most talked-about AI coding products with millions of users, and the paper is from Carnegie Mellon, a top-tier CS institution."
     }
   }
 }