ai-research-survey

scan-v4.json (26209B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "An Empirical Study of Generative AI Adoption in Software Engineering",
     "authors": [
       "G. Giray",
       "Onur Demirörs",
       "Marcos Kalinowski",
       "Daniel Méndez"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.23327",
     "doi": "10.48550/arXiv.2512.23327"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 80% adoption, benefits (cycle time, quality, productivity), challenges (incorrect outputs, prompt engineering, validation overhead), and institutionalization patterns are all supported by the results in Sections 4.2-4.5.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language in places, e.g., 'GenAI tools enable me to achieve better quality' (Q18), 'reduction in cycle time' as a 'benefit' of GenAI. The cross-sectional survey design cannot support causal inference, and this is not explicitly acknowledged in the framing of benefits.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) explicitly avoids generalizability claims: 'we avoided further generalizability claims throughout the paper due to the aforementioned limitations. Replications should be conducted.' The paper frames results as 'from our sample.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion (Section 5) considers alternative interpretations, e.g., that perceived productivity gains may be unreliable due to lack of objective metrics, that non-adopters may face skill gaps rather than tool trust issues, and cites Becker et al. (2025) showing AI can slow experienced developers.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes perceived from objective productivity/quality: 'Despite the high perceived productivity and quality improvements, objective measurement of GenAI's impact seems very limited' (Section 5) and 'these reported perceived improvements become even more questionable' given limited metric usage (Section 4.3.4).",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated and substantive discussion of face/content validity, criterion validity, construct validity, and reliability threats.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: convenience/snowball sampling bias, risk of misunderstood questions, purposive sampling to limit geographic over-representation (stopped collecting in Brazil/Türkiye at ~20), and the random sampling limitation addressed via bootstrapping.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it focuses on 'GenAI for SE' (not SE for GenAI), that units of analysis are 'software professionals... not organizations or projects' (Section 3.4), and avoids generalizability claims due to non-probability sampling.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or funding sources.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Eindhoven University of Technology, Izmir Institute of Technology, PUC-Rio, Blekinge Institute of Technology, and fortiss.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly defines AI, GenAI, SE for GenAI, and GenAI for SE; SE activities are mapped to the ISO/IEC/IEEE 12207 taxonomy providing precise vocabulary.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper states its aim to 'provide an overview of the status of GenAI adoption in SE' via four explicit RQs, and Table 1 shows the 16 dimensions this study covers that prior work does not fully address.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 systematically compares this study against 17 prior surveys across 16 dimensions; Section 5 explicitly contrasts findings with StackOverflow Developer Survey, Capgemini, Google DORA, McKinsey, and Liang et al.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 'The questionnaire, the collected data, and the quantitative and qualitative data analysis artifacts, including Python scripts... are available in our online open science repository [to be published on Zenodo].' This is a promise of future release, not an actual release — no URL is provided.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "Same as above — data availability is stated as '[to be published on Zenodo]' which is a future promise, not an actual release.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No environment specifications, requirements files, or dependency details are provided for the analysis scripts.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are included. The paper describes methods but does not provide a guide to replicate the analysis.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "The paper consistently reports bootstrap confidence intervals for percentages, e.g., 'P = 79.44% [79.27, 79.61]' (Section 4.2), using S=1,000 bootstrap resamples as described in Section 3.5.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper reports descriptive statistics and confidence intervals but does not perform significance tests for any comparative claims (e.g., differences between groups or experience levels).",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": false,
           "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Results are presented as percentages with confidence intervals but without effect size measures for comparisons.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or formal justification for the sample size of 204 is provided. The paper does not discuss whether 204 is adequate for the analyses conducted.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "For experience, the paper reports mean, median, IQR (Q1=5.0, Q3=17.0) in Section 4.1. Bootstrap confidence intervals provide spread measures for survey percentages.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The paper systematically compares its findings against prior surveys (Table 1) and industry reports (StackOverflow, Capgemini, DORA, MIT) throughout Section 5 Discussion.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Comparisons are made against contemporary sources: StackOverflow 2025, Google DORA 2025, MIT 2025, Capgemini 2024 — all recent and relevant.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "Survey study with no system components to ablate.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": false,
           "answer": false,
           "justification": "This is a descriptive survey, not an evaluation of a system. Percentages and confidence intervals describe the sample; there are no competing evaluation metrics.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable — the study is a survey collecting self-reported data, not evaluating a system's outputs.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no train/test split applicable.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by SE activity (Figure 9), tool (Figure 10), usage frequency (Figure 11), benefits (Figure 12), challenges (Figure 16), country (Figure 1), role (Figure 4), organization size (Figure 6), and more.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Challenges and reasons for non-adoption are extensively discussed (Sections 4.2.1 and 4.3.5), including specific failure modes like hallucinations, validation overhead, and context understanding failures.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that ~20% do not use GenAI, 58% don't use objective metrics, ~9% disagree that quality improves, and 1% report negative productivity impact. Challenges are reported in detail.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no AI models are evaluated.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no prompting is used.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no model hyperparameters applicable.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used in this survey study.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.5 describes the data cleaning pipeline: 10 responses removed for declining consent, 9 removed for not providing a valid SE activity, completeness check on remaining 204 responses. Qualitative coding procedures are described in detail including selective and open coding approaches.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Data is promised for Zenodo '[to be published]' but not actually available. No working URL or archive is provided.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.4 describes the data collection in detail: online questionnaire on surveyjs.io, May-November 2025, using convenience + purposive + snowball sampling via emails and LinkedIn.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.4 describes three sampling strategies: convenience (professional network, LinkedIn), purposive (targeting different countries, max 2 per organization), and snowball sampling through contacts.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.5 documents the pipeline: 223 responses received → 10 removed (consent declined) → 9 removed (no valid SE activity) → 204 used. Qualitative coding process is described with initial cycles, theme extraction, and validation by third/fourth authors.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no pre-trained model is evaluated on any benchmark.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no pre-trained model is evaluated on any benchmark.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Survey study — no pre-trained model is evaluated on any benchmark.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any registry.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 states: 'The Research Ethics Committee at Izmir Institute of Technology approved the questionnaire.'",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 provides extensive demographics: country (37 countries, Figure 1), education field and degree (Figure 2), experience distribution (Figure 3), roles (Figure 4), sector (Figure 5), organization and team size (Figure 6), and project management approach (Figure 7).",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.4 states the target population is 'professionals performing SE-related activities.' Section 3.5 describes exclusion: 10 removed for declining consent, 9 removed for not providing a valid SE activity. The screening question C1 gates entry.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "Cross-sectional survey — no experimental conditions or randomization applicable.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "Cross-sectional survey — blinding is not applicable.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.5 reports: 223 total responses, 10 removed for declining consent, 9 removed for quality (no valid SE activity), leaving 204 for analysis. Per-question N values vary and are reported (e.g., N=162 for users, N=38 for non-users).",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": false,
           "answer": false,
           "justification": "Survey paper — no computational method with inference costs.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": false,
           "answer": false,
           "justification": "Survey paper — no significant computation required.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Approximately 80% of SE practitioners use GenAI tools for their SE activities",
       "evidence": "162 of 204 respondents reported GenAI use; bootstrapped P=79.44% [79.27, 79.61] from convenience/snowball sample in 37 countries",
       "supported": "moderate"
     },
     {
       "claim": "Implementation is the primary GenAI use case (71%), far exceeding V&V (24%) and personal assistance (23%)",
       "evidence": "Frequency analysis of free-text Q13 responses coded to ISO 12207 taxonomy; N=162 with bootstrapped CIs in Figure 9",
       "supported": "moderate"
     },
     {
       "claim": "~95% of GenAI users report productivity increases, with 43% estimating an 8-hour task now takes 4 hours",
       "evidence": "Self-reported Likert scale Q17 with N=162; no objective measurement—58% of respondents explicitly use no objective metrics to verify this",
       "supported": "weak"
     },
     {
       "claim": "Incorrect output/hallucination is the dominant challenge (47.7%), followed by prompt engineering difficulties (31.5%) and validation overhead (25.9%)",
       "evidence": "Free-text Q20 (N=130) coded via grounded theory and mapped to ISO/IEC 25059 quality characteristics; reviewed by all four authors",
       "supported": "moderate"
     },
     {
       "claim": "58% of practitioners use no objective metric to measure size, productivity, or quality",
       "evidence": "Direct survey question Q19 with N=115 valid responses; explicitly acknowledged as undermining self-reported benefit claims",
       "supported": "strong"
     },
     {
       "claim": "79% of practitioners expect GenAI to redefine rather than replace their roles within five years",
       "evidence": "Likert Q24 with N=198; bootstrapped CIs reported; aligns with Capgemini (59% no displacement concern) and StackOverflow (64%) surveys",
       "supported": "moderate"
     },
     {
       "claim": "54% of practitioners anticipate job market contraction despite personal job security confidence",
       "evidence": "Likert Q25 with N=194; creates tension with Q24 finding about individual role preservation",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "qualitative"
   ],
   "key_findings": "A questionnaire-based survey of 204 SE practitioners from 37 countries found that ~80% use GenAI tools, primarily for implementation (71%), with ChatGPT dominating (62%). Despite ~95% reporting perceived productivity increases and 82% perceiving quality gains, 58% use no objective metrics to verify these claims—a critical gap the authors explicitly flag. The top challenge is incorrect or hallucinated output (48%), followed by prompt engineering difficulties (31%) and output validation overhead (26%). Most practitioners (79%) expect GenAI to redefine rather than replace their roles, though 54% anticipate broader job market contraction from efficiency gains, revealing a divergence between individual and market-level outlooks.",
   "red_flags": [
     {
       "flag": "Self-report productivity inflation",
       "detail": "All productivity and quality improvement claims are self-reported with no objective verification; 58% of respondents explicitly admit using no metrics, yet the paper presents '95% report productivity increases' as a headline result without adequate qualification."
     },
     {
       "flag": "Non-probability sampling biases toward adopters",
       "detail": "Convenience and snowball sampling via authors' professional LinkedIn/email networks likely overrepresents GenAI enthusiasts and early adopters; practitioners who avoid GenAI tools or are skeptical are systematically underrepresented, inflating adoption rates."
     },
     {
       "flag": "Misleadingly narrow bootstrapped CIs",
       "detail": "Bootstrapped CIs are extremely narrow (e.g., [79.27, 79.61]) reflecting within-sample precision, but bootstrapping cannot correct for non-representative sampling; the false precision may be misread as population-level accuracy."
     },
     {
       "flag": "Data and code not yet available",
       "detail": "Data availability statement says 'to be published on Zenodo'—the questionnaire, raw data, and Python analysis scripts are not verifiable at submission time, making independent replication impossible."
     },
     {
       "flag": "No pre-registration",
       "detail": "A survey with 31 questions and multiple open-text coding schemes was not pre-registered, allowing post-hoc selection of which patterns and categories to highlight without a verifiable analysis plan."
     }
   ],
   "cited_papers": [
     {
       "title": "A large-scale survey on the usability of AI programming assistants: Successes and challenges",
       "relevance": "Direct comparator surveying 410 developers on programming assistant use; finding on non-use due to functional failures (54%) contrasts with this study's skill/time barrier finding"
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "relevance": "Closest comparator; 481 developers in 71 countries on programming assistant usage—Table 1 uses it as a benchmark for coverage comparison"
     },
     {
       "title": "Productivity assessment of neural code completion",
       "relevance": "Seminal survey of 2,047 Copilot users on perceived productivity; used as baseline comparison for scale and methodology"
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "relevance": "RCT showing 55.8% faster task completion with Copilot; referenced as a more rigorous counterpart to self-reported productivity estimates"
     },
     {
       "title": "Navigating the complexity of generative AI adoption in software engineering",
       "relevance": "Prior survey on GenAI adoption in SE across 100 practitioners; Table 1 systematically compares coverage against this study"
     },
     {
       "title": "Toward Effective AI Support for Developers: A survey of desires and concerns",
       "relevance": "Large survey of 737 Microsoft developers on AI support desires; compared directly in non-adoption analysis (trust gap vs. skill gap finding)"
     },
     {
       "title": "Rethinking software engineering in the era of foundation models: A curated catalogue of challenges",
       "relevance": "Foundational framing paper on FMware engineering challenges; provides theoretical context for organizational governance findings"
     },
     {
       "title": "Sampling in software engineering research: A critical review and guidelines",
       "relevance": "Methodological reference used to justify sampling approach and bootstrapping decisions in Section 3.4"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for software teams, CTOs, and policy makers making GenAI adoption decisions—covers tools, use cases, governance mechanisms, and workforce impacts across 37 countries."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 58% no-metrics finding is notable, and the skill-gap vs. trust-gap non-adoption dichotomy adds nuance, but most headline numbers confirm existing industry surveys."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises concerns about overreliance and skill atrophy, 54% expecting job market contraction, and cloud-based security/privacy risks from proprietary code exposure."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy; results broadly align with StackOverflow and Capgemini industry reports, with the tension between individual confidence and market contraction concern being the main narrative hook."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Survey paper with no tool, system, or interactive component to demonstrate."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from established academic institutions (TU/e, PUC-Rio, BTH/fortiss) but not famous industry labs; no brand recognition beyond academic SE circles."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }