ai-research-survey

scan.json (23175B)

---

 {
   "paper": {
     "title": "An Empirical Study of Generative AI Adoption in Software Engineering",
     "authors": ["Görkem Giray", "Onur Demirörs", "Marcos Kalinowski", "Daniel Mendez"],
     "year": 2025,
     "venue": "arXiv (submitted to ACM)",
     "arxiv_id": "2512.23327",
     "doi": "10.48550/arXiv.2512.23327"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The questionnaire, the collected data, and the quantitative and qualitative data analysis artifacts, including Python scripts... are available in our online open science repository [to be published on Zenodo].' This is a promise of future release, not an actual release — no URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as above — data availability is stated as '[to be published on Zenodo]' which is a future promise, not an actual release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided for the analysis scripts."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper describes methods but does not provide a guide to replicate the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports bootstrap confidence intervals for percentages, e.g., 'P = 79.44% [79.27, 79.61]' (Section 4.2), using S=1,000 bootstrap resamples as described in Section 3.5."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports descriptive statistics and confidence intervals but does not perform significance tests for any comparative claims (e.g., differences between groups or experience levels)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Results are presented as percentages with confidence intervals but without effect size measures for comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or formal justification for the sample size of 204 is provided. The paper does not discuss whether 204 is adequate for the analyses conducted."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "For experience, the paper reports mean, median, IQR (Q1=5.0, Q3=17.0) in Section 4.1. Bootstrap confidence intervals provide spread measures for survey percentages."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares its findings against prior surveys (Table 1) and industry reports (StackOverflow, Capgemini, DORA, MIT) throughout Section 5 Discussion."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons are made against contemporary sources: StackOverflow 2025, Google DORA 2025, MIT 2025, Capgemini 2024 — all recent and relevant."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey study with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey, not an evaluation of a system. Percentages and confidence intervals describe the sample; there are no competing evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study is a survey collecting self-reported data, not evaluating a system's outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no train/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by SE activity (Figure 9), tool (Figure 10), usage frequency (Figure 11), benefits (Figure 12), challenges (Figure 16), country (Figure 1), role (Figure 4), organization size (Figure 6), and more."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Challenges and reasons for non-adoption are extensively discussed (Sections 4.2.1 and 4.3.5), including specific failure modes like hallucinations, validation overhead, and context understanding failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that ~20% do not use GenAI, 58% don't use objective metrics, ~9% disagree that quality improves, and 1% report negative productivity impact. Challenges are reported in detail."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 80% adoption, benefits (cycle time, quality, productivity), challenges (incorrect outputs, prompt engineering, validation overhead), and institutionalization patterns are all supported by the results in Sections 4.2-4.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language in places, e.g., 'GenAI tools enable me to achieve better quality' (Q18), 'reduction in cycle time' as a 'benefit' of GenAI. The cross-sectional survey design cannot support causal inference, and this is not explicitly acknowledged in the framing of benefits."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) explicitly avoids generalizability claims: 'we avoided further generalizability claims throughout the paper due to the aforementioned limitations. Replications should be conducted.' The paper frames results as 'from our sample.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion (Section 5) considers alternative interpretations, e.g., that perceived productivity gains may be unreliable due to lack of objective metrics, that non-adopters may face skill gaps rather than tool trust issues, and cites Becker et al. (2025) showing AI can slow experienced developers."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes perceived from objective productivity/quality: 'Despite the high perceived productivity and quality improvements, objective measurement of GenAI's impact seems very limited' (Section 5) and 'these reported perceived improvements become even more questionable' given limited metric usage (Section 4.3.4)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no AI models are evaluated."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no prompting is used."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no model hyperparameters applicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes the data cleaning pipeline: 10 responses removed for declining consent, 9 removed for not providing a valid SE activity, completeness check on remaining 204 responses. Qualitative coding procedures are described in detail including selective and open coding approaches."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated and substantive discussion of face/content validity, criterion validity, construct validity, and reliability threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: convenience/snowball sampling bias, risk of misunderstood questions, purposive sampling to limit geographic over-representation (stopped collecting in Brazil/Türkiye at ~20), and the random sampling limitation addressed via bootstrapping."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it focuses on 'GenAI for SE' (not SE for GenAI), that units of analysis are 'software professionals... not organizations or projects' (Section 3.4), and avoids generalizability claims due to non-probability sampling."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Data is promised for Zenodo '[to be published]' but not actually available. No working URL or archive is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes the data collection in detail: online questionnaire on surveyjs.io, May-November 2025, using convenience + purposive + snowball sampling via emails and LinkedIn."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes three sampling strategies: convenience (professional network, LinkedIn), purposive (targeting different countries, max 2 per organization), and snowball sampling through contacts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 documents the pipeline: 223 responses received → 10 removed (consent declined) → 9 removed (no valid SE activity) → 204 used. Qualitative coding process is described with initial cycles, theme extraction, and validation by third/fourth authors."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Eindhoven University of Technology, Izmir Institute of Technology, PUC-Rio, Blekinge Institute of Technology, and fortiss."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no pre-trained model is evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no pre-trained model is evaluated on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey study — no pre-trained model is evaluated on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any registry."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states: 'The Research Ethics Committee at Izmir Institute of Technology approved the questionnaire.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 provides extensive demographics: country (37 countries, Figure 1), education field and degree (Figure 2), experience distribution (Figure 3), roles (Figure 4), sector (Figure 5), organization and team size (Figure 6), and project management approach (Figure 7)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states the target population is 'professionals performing SE-related activities.' Section 3.5 describes exclusion: 10 removed for declining consent, 9 removed for not providing a valid SE activity. The screening question C1 gates entry."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Cross-sectional survey — no experimental conditions or randomization applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Cross-sectional survey — blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 reports: 223 total responses, 10 removed for declining consent, 9 removed for quality (no valid SE activity), leaving 204 for analysis. Per-question N values vary and are reported (e.g., N=162 for users, N=38 for non-users)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no computational method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no significant computation required."
       }
     }
   },
   "claims": [
     {
       "claim": "Approximately 80% of SE practitioners use GenAI tools in their SE activities.",
       "evidence": "P = 79.44% [79.27, 79.61] from 204 respondents (Section 4.2). Aligns with StackOverflow 2025 (84% using or planning to use).",
       "supported": "moderate"
     },
     {
       "claim": "Implementation is the primary use case for GenAI in SE (71%), followed by verification & validation (24%).",
       "evidence": "Figure 9, Section 4.2.2. Based on 162 GenAI-using respondents reporting ~2 tasks each, coded to ISO/IEC/IEEE 12207 processes.",
       "supported": "moderate"
     },
     {
       "claim": "Approximately 95% of GenAI-using practitioners report a productivity increase.",
       "evidence": "Section 4.3.2, Figure 13: 43% report 50% time reduction, 27% report 75% reduction, 26% report moderate increase. Only 3.5% report no gain and 1.2% negative impact.",
       "supported": "weak"
     },
     {
       "claim": "82% of respondents agree that GenAI tools enhance their work quality.",
       "evidence": "Section 4.3.3, Figure 14: 46.72% strongly agree, 35.21% somewhat agree. Based on 5-point Likert scale (N=161).",
       "supported": "weak"
     },
     {
       "claim": "Objective measurement of productivity and quality remains very limited — 58% of respondents use no objective metric.",
       "evidence": "Section 4.3.4, Figure 15: 58.15% [57.85, 58.45] explicitly stated no objective metrics used. Only story points (19%) and velocity (12%) reported as common metrics.",
       "supported": "strong"
     },
     {
       "claim": "Incorrect outputs/hallucinations are the top challenge (48%), followed by prompt engineering difficulties (31%).",
       "evidence": "Section 4.3.5, Figure 16: inaccurate output at 47.70%, interaction/prompting at 31.48%, validation overhead at 25.89%. Based on N=130 coded responses.",
       "supported": "moderate"
     },
     {
       "claim": "79% of practitioners expect GenAI to redefine rather than replace their roles; 62% disagree that GenAI will replace them.",
       "evidence": "Section 4.5, Figure 18: Based on Likert scale responses from N=198 and N=198 respectively.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "Survey of 204 SE practitioners from 37 countries finds ~80% adoption of GenAI tools, primarily for implementation (71%) and verification/validation (24%). While 95% report perceived productivity gains and 82% report quality improvements, 58% use no objective metrics to measure these claims. Top challenges are hallucinations (48%), prompt engineering (31%), and output validation overhead (26%). Most practitioners (79%) expect role redefinition rather than replacement.",
   "red_flags": [
     {
       "flag": "Self-reported benefits without objective measurement",
       "detail": "95% claim productivity gains and 82% claim quality improvements, but 58% of respondents use no objective metrics. The paper acknowledges this tension but the headline findings (especially in the abstract) still lead with perceived benefits."
     },
     {
       "flag": "Non-probability sampling with narrow CIs",
       "detail": "The bootstrap confidence intervals are extremely narrow (e.g., [79.27, 79.61] for adoption rate) because bootstrapping quantifies sampling variability of the estimator, not coverage of the target population. With convenience + snowball sampling, the real uncertainty is much larger than these CIs suggest. The paper acknowledges sampling limitations but the narrow CIs may give a false sense of precision."
     },
     {
       "flag": "Potential selection bias toward GenAI enthusiasts",
       "detail": "Survey distributed through professional networks, LinkedIn, and snowball sampling to SE practitioners. People who respond to a survey about GenAI adoption are likely more engaged with GenAI than non-respondents, potentially inflating the 80% adoption figure."
     },
     {
       "flag": "Data and code not yet released",
       "detail": "The paper promises open science data on Zenodo but the repository is '[to be published]'. Claims of transparency cannot be verified without the actual data release."
     }
   ],
   "cited_papers": [
     {
       "title": "Copiloting the Future: How Generative AI Transforms Software Engineering",
       "authors": ["L. Banh", "F. Holldack", "G. Strobel"],
       "year": 2025,
       "relevance": "Survey of GenAI impact on SE across 17 European companies, covering use cases, benefits, and challenges."
     },
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Empirical study finding AI can slow experienced developers, contradicting perceived productivity gains — key contrarian evidence."
     },
     {
       "title": "A large-scale survey on the usability of ai programming assistants: Successes and challenges",
       "authors": ["J. T. Liang", "C. Yang", "B. A. Myers"],
       "year": 2024,
       "relevance": "Survey of 410+ developers on programming assistant usability, covering reasons for non-use and challenges."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["A. Sergeyuk", "Y. Golubev", "T. Bryksin", "I. Ahmed"],
       "year": 2025,
       "relevance": "Survey of 481 developers across 71 countries on AI coding assistant usage patterns and perceptions."
     },
     {
       "title": "Navigating the complexity of generative ai adoption in software engineering",
       "authors": ["D. Russo"],
       "year": 2024,
       "relevance": "Survey of 100 practitioners on GenAI adoption complexity in SE, covering all knowledge areas."
     },
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Controlled experiment showing Copilot enables 55.8% faster task completion, a key productivity claim in the field."
     },
     {
       "title": "The impact of generative AI on collaborative open-source software development: Evidence from GitHub Copilot",
       "authors": ["F. Song", "A. Agarwal", "W. Wen"],
       "year": 2024,
       "arxiv_id": "2410.02091",
       "relevance": "Evaluates Copilot's impact on OSS project-level productivity, finding benefits especially for core developers."
     },
     {
       "title": "Toward Effective AI Support for Developers: A survey of desires and concerns",
       "authors": ["M. Khemka", "B. Houck"],
       "year": 2024,
       "relevance": "Large-scale Microsoft survey (737 developers) on AI support desires and concerns, including trust and accuracy issues."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["A. Ziegler", "E. Kalliamvakou", "X. A. Li"],
       "year": 2022,
       "relevance": "GitHub Copilot productivity study surveying 2,000+ developers on perceived productivity impact."
     },
     {
       "title": "How secure is AI-generated code: a large-scale comparison of large language models",
       "authors": ["N. Tihanyi", "T. Bisztray", "M. A. Ferrag"],
       "year": 2025,
       "relevance": "Finds 62% of LLM-generated code is vulnerable, directly relevant to security challenges of GenAI adoption."
     },
     {
       "title": "Perceived Impact of AI-Based Tooling on Software Development Code Quality",
       "authors": ["B. Martinović", "R. Rozić"],
       "year": 2025,
       "relevance": "Survey finding AI tool users 67% likely to produce vulnerable code vs 27% without, relevant to quality impact claims."
     },
     {
       "title": "Naming the pain in machine learning-enabled systems engineering",
       "authors": ["M. Kalinowski", "D. Mendez", "G. Giray"],
       "year": 2025,
       "relevance": "Large-scale survey on ML-enabled systems engineering challenges, related methodology and authorship overlap."
     }
   ]
 }