ai-research-survey

scan.json (23331B)

---

 {
   "paper": {
     "title": "More code, less validation: Risk factors for over-reliance on AI coding tools among scientists",
     "authors": ["Gabrielle O'Brien", "Alexis Parker", "Nasir U. Eisty", "Jeffrey Carver"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.19644",
     "doi": "10.48550/arXiv.2512.19644"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states 'R scripts for processing data from the survey are provided in the accompanying GitHub repository' and a GitHub URL is provided in Section 4.5 (Code Availability)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Data Availability section states 'raw survey responses are not made public. Anonymized data is available to peer reviewers at request.' Available upon request is not public release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or library versions mentioned. Only that analyses were conducted in R."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided. Scripts are said to be in a GitHub repository but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results report point estimates (e.g., r = -0.14, p = 0.0009) but no confidence intervals or error bars on main results. The linear model in Table 3 reports standard errors on coefficients but no CIs on the main findings."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Extensive use of significance tests: ANOVA (F tests with p-values), polyserial/polychoric correlations with p-values, t-tests for group comparisons, chi-squared tests, and linear model F-statistics throughout Section 2."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Correlation coefficients reported throughout (e.g., polyserial r = -0.12, r = 0.31), R² values for models (R² = 0.04, adjusted R² = 0.035), and percentage of variance explained (9.6%). These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification. The sample of 868 is a convenience sample; no discussion of whether this is sufficient for the analyses conducted."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported for key variables (e.g., programming experience σ = 9.20, perceived productivity σ = 0.70) and residual standard error in regression models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares across groups (experienced vs inexperienced, high vs low development practice adoption, adopters vs non-adopters) and references prior work (Ziegler et al.'s GitHub Copilot study) as comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons reference contemporary studies: Ziegler et al. (2022), Kumar et al. (2025), O'Brien (2025), Chugunova et al. (2025). The SPACE scale is adapted from recent work."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple outcome variables analyzed: usage frequency, tool choice, composite perceived productivity score, individual SPACE dimensions, development practice scores, lines of code accepted."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable to a survey study."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey study — there is no train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by research area (Table 1, Figure 2), position (Figure 3), gender (Table B2, Figure A3), tool type (Table 2), and individual development practices (Figure 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the low explanatory power of their model (R² = 0.04) explicitly, and Section 2.5 presents reasons for non-adoption including failures and frustrations with tools."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: no significant overall effect of gender (p = 0.069), no significant effect of research area on perceived productivity (p = 0.125), and low R² of the main model acknowledged."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about adoption patterns, tool preferences, interaction of experience and practices, and lines-of-code as strongest predictor are all supported by results in Sections 2.2-2.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses language like 'factors associated with' and 'predictors' which is appropriately correlational, but the title 'Risk factors for over-reliance' and discussion of 'over-reliance' imply causal mechanisms from cross-sectional survey data. Section 3.2 acknowledges 'cross-sectional study design precludes strong causal claims' but the framing throughout suggests causation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly limits generalizability: 'our survey sample is non-random, mostly academic, and heavily U.S.-centric, limiting generalizability to non-academic researchers and scientists in the global south.' Section 2.1 emphasizes 'this is a non-random, convenience sample.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses alternative explanations: users accepting many lines may still modify code substantially; perceived productivity may genuinely reflect benefit for novices, not just automation bias; the SPACE scale may have imperfect reliability. Section 2.4.3 offers an 'ad hoc explanation' for the interaction effect."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently distinguishes 'perceived productivity' from actual productivity. Section 3.1 states 'our survey lacks direct measures of code quality or developer activity' and discusses the proxy gap extensively. The title itself flags 'over-reliance' as the real concern behind perceived productivity gains."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study; no AI models were used by the researchers in the methodology."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting used in the methodology. The paper studies others' use of AI tools."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models or hyperparameters involved in the study methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used in this survey study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 documents data handling: inclusion criteria filtering (1272 → 868), log transformations, ordered factor conversion, standardized dictionary for job title recoding. Section 4.2.1 details each exclusion step with counts (283 incomplete, 8 no consent, 102 non-programmers, 9 non-research, 1 quality control)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 is titled 'Limitations' and provides substantive discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses specific threats: cross-sectional design precluding causal inference, non-random U.S.-centric academic sample, recruitment through RSE communities underrepresenting casual programmers, self-report divergence from actual productivity (citing specific studies showing this gap)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly states what results do NOT show: 'limiting generalizability to non-academic researchers and scientists in the global south.' Section 3.1 states 'our survey lacks direct measures of code quality or developer activity, we cannot definitively determine how perceived productivity relates to these constructs.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data not publicly available. 'Anonymized data is available to peer reviewers at request' — not public access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the survey instrument in detail (seven sections, Qualtrics platform, 10-15 minute estimated completion). Section 4.2 describes recruitment channels. The full survey is provided as supplementary material."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes recruitment in detail: US-RSE Slack and newsletter, pyOpenSci community, University of Michigan mailing list (18,851 addresses), Alfred P. Sloan Foundation newsletter, Learning Engineering Google group. Incentivization via cash drawing ($100 × 5)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 documents the full pipeline: 1272 collected → 283 incomplete removed → 8 no consent → 102 non-programmers screened → 1 quality control → 9 non-research screened → 868 final. Section 4.3 documents all transformations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work is supported by a grant from the Alfred P. Sloan foundation to G.O.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: University of Michigan, University of Tennessee, University of Alabama. No commercial affiliations that would create conflicts."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Alfred P. Sloan Foundation is a philanthropic organization with no financial stake in whether genAI tools help or harm scientific programming."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "Ethics declarations section states: 'The authors declare no competing interests.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not evaluating a pre-trained model on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned. The study was reviewed by IRB but no pre-registration link (OSF, AsPredicted, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'the survey and study plan were reviewed by the University of Michigan Institutional Review Board and deemed to be of minimal risk to participants.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive demographics: career stage (Section 2.1), programming experience (median 7 years, µ=9.50, σ=9.20), gender (441 men, 384 women, 24 non-binary), research area (Table 1), institution type, geographic distribution."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 details inclusion criteria: must program at least sometimes, 18+, informed consent, research group publishes peer-reviewed works. Exclusion counts documented for each criterion."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with treatment/control conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a cross-sectional survey study."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 reports: 1272 total responses, 283 incomplete removed, 8 no consent, 102 non-programmers, 9 non-research, 1 quality control = 868 final. Additionally, per-question response counts are reported throughout (e.g., n=760 for usage frequency, n=609 for tool choice)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no AI inference costs apply."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no significant compute involved."
       }
     }
   },
   "claims": [
     {
       "claim": "GenAI adoption for programming is highest among students and less experienced programmers.",
       "evidence": "Section 2.2: Student research assistants showed highest adoption (Figure 3). Log-years of programming experience inversely related to usage frequency (polyserial r = -0.12, p = 0.001).",
       "supported": "strong"
     },
     {
       "claim": "Scientific programmers overwhelmingly prefer general-purpose conversational tools (ChatGPT) over developer-specific tools.",
       "evidence": "Section 2.3: 472/609 (77.5%) selected general-purpose tools vs 88/609 (14.4%) developer-specific. ChatGPT was primary tool for 391/609 (64.2%). Table 2.",
       "supported": "strong"
     },
     {
       "claim": "Both inexperience and limited development practices are associated with greater perceived productivity, and these factors interact.",
       "evidence": "Table 3: Main effects of development practice score (β = -0.217, p < 0.01) and log-years experience (β = -0.554, p < 0.01) with significant interaction (β = 0.207, p < 0.01). However, overall R² = 0.04.",
       "supported": "moderate"
     },
     {
       "claim": "The strongest predictor of perceived productivity is the number of lines of generated code typically accepted at once.",
       "evidence": "Section 2.4.3: polyserial r = 0.31, p = 3.12e-17, explaining 9.6% of variance. Robustness check excluding activity item: r = 0.28, p = 1.01e-13.",
       "supported": "strong"
     },
     {
       "claim": "Women tend to report less genAI usage for programming than men.",
       "evidence": "Section 2.2: β = -0.20778, SE = 0.09006, p = 0.0213. Omnibus effect of gender did not meet significance threshold (p = 0.069). Described as exploratory.",
       "supported": "moderate"
     },
     {
       "claim": "Scientific programmers using genAI may gauge productivity by code generation rather than validation.",
       "evidence": "Section 3.1: Lines accepted negatively correlates with all development practice measures (Figure 5). Users accepting >100 lines report highest productivity but have lowest practice adoption. However, this is inferential — the study has no direct measures of code quality.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "Survey of 868 scientists who program reveals genAI adoption is highest among students and less experienced programmers, with ChatGPT dominating (64.2%) over developer-specific tools (14.4%). Both programming inexperience and limited use of development practices (testing, code review, version control) are associated with greater perceived productivity, with a significant interaction suggesting practices partially compensate for inexperience. The strongest single predictor of perceived productivity is the volume of code accepted at once (r = 0.31), raising concerns that scientific programmers equate code generation with productivity rather than validation. About 25% of respondents rejected genAI tools, citing concerns about skill development, ethics, no perceived need, inefficiency, and accuracy problems.",
   "red_flags": [
     {
       "flag": "Non-random convenience sample",
       "detail": "The sample is heavily skewed toward U.S. academic researchers recruited through RSE mailing lists and one university's employee list (18,851 addresses at University of Michigan). 843/868 respondents are from the U.S. The paper acknowledges this but the title and abstract do not bound the generalization to this population."
     },
     {
       "flag": "Low explanatory power of main model",
       "detail": "The main hypothesis-testing model explains only 4% of variance in perceived productivity (R² = 0.04). While the paper acknowledges this, it still frames the results as supporting its hypotheses about risk factors for over-reliance."
     },
     {
       "flag": "Self-report measures only",
       "detail": "All measures are self-reported with no behavioral validation. The paper frames findings as 'risk factors for over-reliance' but has no measure of actual over-reliance, code quality, or real productivity. The leap from survey correlations to 'over-reliance' in the title is inferential."
     }
   ],
   "cited_papers": [
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler"],
       "year": 2022,
       "relevance": "Foundation for the SPACE perceived productivity scale used in this study; large study of GitHub Copilot users linking experience to perceived productivity."
     },
     {
       "title": "Intuition to Evidence: Measuring AI's True Impact on Developer Productivity",
       "authors": ["Aman Kumar"],
       "year": 2025,
       "arxiv_id": "2509.19708",
       "relevance": "Study linking perceived productivity with genAI tools to measurable increases in objective programming activity."
     },
     {
       "title": "The Widening Gap: The Benefits and Harms of Generative AI for Novice Programmers",
       "authors": ["James Prather"],
       "year": 2024,
       "relevance": "Describes the 'illusion of competence' where novice programmers overestimate their understanding of AI-generated code."
     },
     {
       "title": "GitHub Copilot AI pair programmer: Asset or Liability?",
       "authors": ["Arghavan Moradi Dakhel"],
       "year": 2023,
       "relevance": "Empirical evaluation of Copilot's code quality and risks of uncritical acceptance."
     },
     {
       "title": "Speed at the Cost of Quality? The Impact of LLM Agent Assistance on Software Development",
       "authors": ["Hao He", "Cody Miller", "Siddharth Agarwal", "Christian Kästner", "Bogdan Vasilescu"],
       "year": 2025,
       "relevance": "Study finding that Cursor-associated projects show initial activity bursts but declining quality and maintainability."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Josh Becker", "Neil Rush", "Brian Barnes", "David Rein"],
       "relevance": "Study that failed to find meaningful productivity increases with AI tools, providing counterevidence to productivity claims."
     },
     {
       "title": "Developer Productivity With and Without GitHub Copilot: A Longitudinal Mixed-Methods Case Study",
       "authors": ["Viktoria Stray"],
       "year": 2025,
       "arxiv_id": "2509.20353",
       "relevance": "Longitudinal case study examining Copilot's impact on developer productivity with mixed-methods approach."
     },
     {
       "title": "How Scientists Use Large Language Models to Program",
       "authors": ["Gabrielle O'Brien"],
       "year": 2025,
       "relevance": "Prior study by first author on scientific programmers' use of LLMs, informing the survey design."
     },
     {
       "title": "Vibe Coding in Practice: Motivations, Challenges, and a Future Outlook - a Grey Literature Review",
       "authors": ["Alaa Fawzy", "Amjed Tahir", "Kelly Blincoe"],
       "year": 2025,
       "relevance": "Grey literature review on vibe coding practices with genAI tools, relevant to how users interact with AI coding assistants."
     },
     {
       "title": "Self-Admitted GenAI Usage in Open-Source Software",
       "authors": ["Tao Xiao"],
       "year": 2025,
       "arxiv_id": "2507.10422",
       "relevance": "Study on identifying genAI-originated code in open-source, relevant to detecting AI tool usage patterns."
     },
     {
       "title": "A survey of the state of the practice for research software in the United States",
       "authors": ["Jeffrey C. Carver", "Nic Weber", "Karthik Ram", "Sandra Gesing", "Daniel S. Katz"],
       "year": 2022,
       "relevance": "Foundation survey instrument for development practice adoption among scientific programmers, directly adapted in this study."
     }
   ]
 }