ai-research-survey

scan.json (23790B)

---

 {
   "paper": {
     "title": "Usage, Effects and Requirements for AI Coding Assistants in the Enterprise: An Empirical Study",
     "authors": ["Maja Vukovic", "Rangeet Pan", "Tin Kam Ho", "Rahul Krishna", "Raju Pavuluri", "Michele Merler"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.20112"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or analysis scripts are released. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Survey responses are not released. The 35-paper corpus is described but no structured dataset is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency information provided. The paper mentions using Gemini 2.5 pro and Claude Sonnet 4 for LLM-based summarization but provides no setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The survey questionnaire is included in Appendix A, but there are no instructions for reproducing the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts and percentages (e.g., '88% perceived productivity gains', '44 out of 57 reported at least 25% increase') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims across divisions and experience levels (e.g., differences in motivations across divisions, Figure 6) but uses no statistical tests to determine whether differences are significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Productivity gains are reported as self-reported percentages (25%, 50%, 100%) without any standardized effect sizes or baseline context beyond raw numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=57 for the primary survey with no justification for sample size, no power analysis, and no discussion of whether this sample is sufficient for the subgroup comparisons attempted."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Only raw counts and percentages are given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its own 57-person survey results against 35 prior surveys (Section 3), providing a baseline of existing knowledge about AI coding assistant usage."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 35 reviewed surveys are from 2023-2025, which are contemporary for the topic. Selection criteria explicitly required recency."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey, not an evaluation with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper itself is a survey collecting human opinions; there is no system output to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to survey research."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by division (Figures 6, 7), years of experience (Figure 3b), programming language proficiency (Figure 3c), and feature categories (Figure 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.6 specifically discusses users who perceived no gain (13 of 57, 22%), including detailed reasons: limitations of the assistant, correctness issues, prompting overhead."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 22% of users found no productivity gain, that AI tools made debugging harder for some, and that experienced developers see discrepancies between perceived and measured productivity."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper surveys 57 developers and reviews 35 surveys, and discusses requirements for AI coding assistants. These are all supported by the paper content."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language like 'AI coding assistants are helping drive the productivity' (Section 5) from observational self-report data without addressing confounds such as selection bias (users who adopt tools may already be more productive) or placebo effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'in the Enterprise' broadly, but the sample is 57 developers from a single company (IBM). The paper does not bound its conclusions to this specific organizational context."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the findings. For example, the high perceived productivity could reflect novelty effects, self-selection bias, or demand characteristics from surveying within the employer organization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures self-reported perceived productivity gains and presents them as actual productivity improvements without distinguishing between perceived and measured productivity. The related work acknowledges 'discrepancies between perceived and measured productivity' (Section 2) but does not apply this insight to its own findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'Gemini 2.5 pro and Claude Sonnet 4' for LLM-based summarization of surveys but provides no version IDs or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts used for LLM-based analysis of the 35 surveys are provided in Tables 1 and 2 (Supplementary Material), including the complete extraction instructions for both Gemini and Claude."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, etc.) are reported for the LLM-based analysis pipeline."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLMs are prompted directly for information extraction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the search and selection criteria for the 35 surveys, and Section 3.2 describes the LLM-based extraction and human verification methodology. The pipeline is shown in Figure 1. Filtering criteria (recency, practitioner surveys, SE-specific, publicly available) are stated, with counts (approximately 50 → 35)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper ends with a brief conclusion (Section 5) that mentions future work but does not discuss study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge that findings from 57 IBM employees may not generalize to other enterprises."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the survey responses nor the extracted data from the 35 reviewed papers are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states the survey was launched in May 2025 within a tech company, collected via an online form with 25 questions and 57 responses. The full questionnaire is in Appendix A."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 57 responses were received from within a 'tech company' but does not describe how participants were recruited, whether it was voluntary, how they were invited, or what the response rate was."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how free-text survey responses were coded into categories. Section 4.3.2 mentions responses were 'manually mapped' to themes but the mapping criteria and process are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All six authors are clearly listed as IBM Research, Yorktown Heights. The paper surveys IBM employees about AI coding assistants including IBM's own watsonx Code Assistant."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM Research employees surveyed IBM employees about AI coding assistants including IBM's watsonx Code Assistant. IBM has a financial interest in positive findings about enterprise AI coding assistants."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. IBM authors evaluating IBM's own product (watsonx Code Assistant is prominently featured in Figure 5) without declaring conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. It is a survey study."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for this survey study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite surveying 57 human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed demographics are provided: business unit breakdown (Figure 3a), years of SE experience (Figure 3b), programming language proficiency (Figure 3c), IDE usage (Figure 4)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for participant selection. The paper says 57 responses were received but does not describe who was eligible or how participants were selected."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with randomized conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a cross-sectional survey."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Some questions have different response counts (e.g., '35 respondents to this question' in Section 4.3.3 vs. 57 total) but no systematic reporting of attrition or explanation for missing responses."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; cost reporting is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; compute budget is not applicable."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a web search process (Section 3.1) with filtering criteria but does not follow PRISMA or any named review protocol. No flow diagram with counts at each filtering stage is provided. The search terms are listed but not as reproducible database queries."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The 35 reviewed surveys are not assessed for methodological quality. All are treated equally regardless of sample size (ranging from 10 to 17,420), study design, or rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether the 35 surveyed studies skew toward positive findings about AI coding assistants."
       }
     }
   },
   "claims": [
     {
       "claim": "88% of surveyed developers perceived productivity gains when using AI coding assistants",
       "evidence": "Section 4.3.3: '88% of them perceived productivity gains when using AI Coding assistant. Among the 57 responding users, 44 of them reported at least a 25% increase in productivity.'",
       "supported": "moderate"
     },
     {
       "claim": "73.7% of practitioners reported that AI code assistants transformed their development process",
       "evidence": "Section 4.3.5: '42 out of 57 (73.7%), reported that the use of AI code assistants has transformed their development process.'",
       "supported": "moderate"
     },
     {
       "claim": "There is no 'one size fits all' motivating use case across users of AI coding assistants",
       "evidence": "Section 4.3.2 and Figure 6 show different motivation distributions across divisions (Research focused on productivity, Sales balanced with code quality, Consulting emphasizing code understanding).",
       "supported": "moderate"
     },
     {
       "claim": "Over 65% of respondents replaced Google Search and StackOverflow with AI coding assistants",
       "evidence": "Section 4.3.3: 'Over 65% respondents reported that they previously relied on Google Search, followed by StackOverflow.'",
       "supported": "weak"
     },
     {
       "claim": "Most existing surveys are dominated by ChatGPT and GitHub Copilot, leaving understanding of other assistants limited",
       "evidence": "Section 3.3 and Figure 2a show ChatGPT and GitHub Copilot dominate the distribution of studied tools across 35 surveys.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["observational", "meta-analysis", "qualitative"],
   "key_findings": "A survey of 57 enterprise developers at IBM found 88% perceived productivity gains from AI coding assistants, with significant variation in motivations and valued features across business divisions. A companion review of 35 prior surveys revealed that existing research is dominated by ChatGPT and GitHub Copilot studies, focuses on short-term productivity rather than long-term quality, and draws from homogeneous participant pools. Users requested deeper context awareness, full customization, and evolution toward autonomous agent capabilities as key requirements for future AI coding assistants.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All six authors are from IBM Research. IBM's watsonx Code Assistant is the most-used tool among the surveyed 57 IBM employees (Figure 5). The paper does not acknowledge this conflict of interest."
     },
     {
       "flag": "Single-company non-representative sample presented broadly",
       "detail": "The 57 respondents are all from a single tech company (IBM), yet the title claims 'Enterprise' generality. No limitations section acknowledges this restriction. The sample is heavily skewed toward experienced developers (42% have >20 years experience)."
     },
     {
       "flag": "Self-reported productivity treated as actual productivity",
       "detail": "The paper reports perceived productivity gains (88%) as evidence of actual productivity improvement. Section 2 acknowledges 'discrepancies between perceived and measured productivity for experienced developers' but does not apply this caveat to its own findings."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section despite multiple obvious threats: single-company sample, self-selection bias, demand characteristics from employer-administered survey, self-reported outcomes."
     },
     {
       "flag": "Arithmetic error in reported percentages",
       "detail": "Section 4.3.3 states '44 out of 57 users (97.8% of the users who answered on the perceived gain)'. 44/57 = 77.2%, not 97.8%. The denominator for 97.8% is unclear."
     },
     {
       "flag": "LLM-assisted analysis without validation methodology",
       "detail": "The paper used Gemini 2.5 pro and Claude Sonnet 4 to extract information from 35 papers, then 'manually verified' the results. No inter-rater reliability, no description of what manual verification entailed, and no error rate reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding discrepancies between perceived and measured productivity for experienced developers using AI coding tools."
     },
     {
       "title": "Dear Diary: A randomized controlled trial of Generative AI coding tools in the workplace",
       "authors": ["Jenna Butler", "Jina Suh", "Sankeerti Haniyur", "Constance Hadley"],
       "year": 2024,
       "arxiv_id": "2410.18334",
       "relevance": "RCT of GitHub Copilot finding increased belief in usefulness but no statistically significant changes in objective telemetry metrics."
     },
     {
       "title": "How Much Does AI Impact Development Speed? an Enterprise-Based Randomized Controlled Trial",
       "authors": ["Elise Paradis", "Kate Grey", "Quinn Madison"],
       "year": 2025,
       "doi": "10.1109/ICSE-SEIP66354.2025.00060",
       "relevance": "Enterprise RCT finding developers were ~21% faster with AI but statistical significance reduced after controlling for other factors."
     },
     {
       "title": "Security Weaknesses of Copilot-Generated Code in GitHub Projects: An Empirical Study",
       "authors": ["Yujia Fu", "Peng Liang", "Amjed Tahir"],
       "year": 2025,
       "doi": "10.1145/3716848",
       "relevance": "Empirical study of security vulnerabilities in Copilot-generated code in real GitHub projects."
     },
     {
       "title": "The Effects of Generative AI on High-Skilled Work: Evidence from Three Field Experiments with Software Developers",
       "authors": ["Zheyuan Cui", "Mert Demirer", "Sonia Jaffe"],
       "year": 2025,
       "doi": "10.2139/ssrn.4945566",
       "relevance": "Three field experiments showing less experienced developers benefit more from AI coding tools."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2023,
       "doi": "10.1145/3576915.3623157",
       "relevance": "CCS study finding participants with AI access wrote significantly less secure code and were more overconfident."
     },
     {
       "title": "Examining the Use and Impact of an AI Code Assistant on Developer Productivity and Experience in the Enterprise",
       "authors": ["Justin D. Weisz", "Shraddha Kumar", "Michael Muller"],
       "year": 2025,
       "arxiv_id": "2412.06603",
       "relevance": "Large-scale enterprise study of IBM watsonx Code Assistant finding unevenly distributed productivity gains."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["Agnia Sergeyuk", "Yaroslav Golubev", "Timofey Bryksin", "Iftekhar Ahmed"],
       "year": 2025,
       "doi": "10.1016/j.infsof.2024.107610",
       "relevance": "Large survey (481 programmers) on AI coding assistant usage patterns and reasons for non-adoption."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": ["Jenny T. Liang", "Chenyang Yang", "Brad A. Myers"],
       "year": 2024,
       "doi": "10.1145/3597503.3608128",
       "relevance": "410-developer survey on usability factors and challenges of AI programming assistants."
     },
     {
       "title": "Measuring GitHub Copilot's Impact on Productivity",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou"],
       "year": 2024,
       "doi": "10.1145/3633453",
       "relevance": "Large-scale study (17,420 users) measuring Copilot's productivity impact through acceptance rates."
     },
     {
       "title": "Maybe We Need Some More Examples: Individual and Team Drivers of Developer GenAI Tool Use",
       "authors": ["Courtney Miller", "Rudrajit Choudhuri"],
       "year": 2025,
       "arxiv_id": "2507.21280",
       "relevance": "Study finding organizational expectations for rapid AI productivity gains limited by uneven adoption across teams."
     }
   ]
 }