ai-research-survey

scan.json (25183B)

---

 {
   "paper": {
     "title": "\"Maybe We Need Some More Examples:\" Individual and Team Drivers of Developer GenAI Tool Use",
     "authors": ["Courtney Miller", "Rudrajit Choudhuri", "Mara Ulloa", "Sankeerti Haniyur", "Robert DeLine", "Margaret-Anne Storey", "Emerson Murphy-Hill", "Christian Bird", "Jenna L. Butler"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.21280",
     "doi": "10.48550/arXiv.2507.21280"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["qualitative"],
   "key_findings": "Through paired interviews with 54 developers (27 matched pairs from the same teams), the study identifies that frequent GenAI tool users view tools as collaborators, adopt experimental approaches, and demonstrate adaptive persistence when facing challenges, while infrequent users view tools as features, adopt conservative approaches, and quickly abandon tools after failures. The study identifies a 'Productivity Pressure Paradox' where organizational expectations for rapid productivity gains without learning support undermine the very benefits that motivate adoption. Team and organizational factors (leadership messaging, context-specific resources, social learning structures, protected learning time) actively shape individual adoption patterns.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or analysis scripts are released. The paper mentions supplementary materials (interview guide, codebook) will be posted on Zenodo with camera ready, but no repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Interview transcripts are not released (understandable for privacy). The paper states interview guide, codebook, and codebook manual are available in supplemental materials on HotCRP, with plans to post on Zenodo, but no data download link is provided."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the methodology section describes the interview protocol and analysis approach in reasonable detail, no step-by-step reproduction instructions are provided. The supplementary materials (interview guide, codebook) are promised but not yet available."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative study. The authors explicitly state they 'avoid discussing frequency counts or percentages in line with established guidance against quantifying qualitative data' (Section 3.4)."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative thematic analysis study; no statistical hypothesis testing is performed or claimed."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effect sizes are relevant to this qualitative study design."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The authors describe their saturation criterion: 'two consecutive interview pairs without learning any new major insights (i.e., 4 interviews)' (Section 3.4), citing Francis et al. (2010) for operationalizing data saturation."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative measurements requiring variance reporting in this qualitative study."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not an evaluation of a system. The paired design comparing frequent vs. infrequent users serves as the comparative structure."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a qualitative interview study."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a qualitative interview study."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a qualitative interview study."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study IS a human interview study, not a system evaluation requiring human judges."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to qualitative research."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are organized by theme categories (mindset formation, approach determination, encounters with challenges, integration and evolution) with separate analysis of frequent vs. infrequent users. Figure 3 provides a visual breakdown of factor distributions across user types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the framework's patterns do not hold — e.g., PID35 remained infrequent despite a collaborative team environment (Section 5.3), and Section 5 shows that organizational factors affect team members differently despite shared context."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that peer influence 'operates differently even within teams' (Section 5.3) and that formal organizational messaging alone is insufficient — PID37's team received minimal guidance reinforcing limited usage. These counter-narratives complicate the framework rather than being suppressed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about collaborator vs. feature perception, experimental vs. conservative approaches, adaptive persistence vs. quick abandonment, and the Productivity Pressure Paradox are all supported with extensive interview evidence in Sections 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges in Section 3.6: 'we occasionally use causal language (e.g., \"leads to\") for clarity and readability. These phrasings should not be interpreted as statistical causal claims, as our qualitative design does not support formal causal inference.' This is an appropriate hedge."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 explicitly states limitations to generalizability: single large company, GitHub Copilot only, excludes Germany/Norway, may not generalize to smaller companies, open source, or companies without AI promotion. The authors also acknowledge construct validity limitations around measuring only Copilot usage."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that infrequent users classified via Copilot telemetry may be heavy users of other AI tools (Section 3.6). Section 5 explicitly addresses how the same organizational factors produce divergent responses, referencing social construction theory (Fulk, Orlikowski) rather than assuming simple causal paths."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures GitHub Copilot usage days as a proxy for GenAI tool usage and explicitly acknowledges the gap: 'Engineers using general-purpose AI tools (e.g., internal OpenAI models) or non-compliant tools (e.g., Cursor, Windsurf) could be classified as infrequent users despite potentially heavy AI usage elsewhere' (Section 3.6)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used as part of the study methodology. The study is about developers using GitHub Copilot, but the researchers themselves do not use AI models for evaluation."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting of AI models is part of the study methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI model hyperparameters are relevant to this qualitative interview study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the pairing algorithm (Hungarian method via scipy.optimize), filtering criteria (same primary programming language, usage gap in top third), interview scheduling process, and exclusion of 2 participants whose pair partners dropped out (Section 3.2, 3.4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 'Limitations' provides a dedicated, substantive discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 discusses specific threats: self-selection bias, single-company limitation with strong pro-AI culture, construct validity issue of measuring only Copilot usage, and potential response bias from organizational pressure. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 explicitly states what the results do not show: 'This limits generalizability to engineers at smaller or larger companies without AI promotion, or to open source development contexts.' The paper also notes Copilot-only measurement does not capture other tool usage."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Interview transcripts are not available (reasonable for privacy, but limits independent verification). Only interview guide and codebook are promised as supplementary materials."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2-3.4 describes data collection in detail: telemetry extraction, pairing algorithm, 8-week observation window (April 1 - May 31, 2025), video conferencing interviews, 670 invitations sent, 315 agreed, 152 from 76 pairs, 56 interviews conducted."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes recruitment: internal company chat, reaching out to individual developers from algorithmically-identified pairs, excluding Germany and Norway for regulatory reasons. Potential recruitment bias from self-selection is acknowledged in limitations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 670 invitations → 315 agreed → 152 from 76 complete pairs → 56 interviews conducted → 2 excluded (partner dropout) → 54 participants (27 pairs) in final analysis. The coding process is also documented: open coding → 8-interview framework review → iterative refinement → full re-coding."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure or acknowledgment of grants. The acknowledgments mention internships at Microsoft and visiting researcher status but do not disclose formal funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: 4 authors at Microsoft (the company whose tool, GitHub Copilot, is studied), plus affiliations at CMU, Oregon State, Northwestern, and University of Victoria. Microsoft affiliation is prominent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft employees are studying Microsoft's own product (GitHub Copilot). Microsoft has a direct financial interest in demonstrating that adoption barriers can be overcome and that the tool provides value. The funder/employer is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Several authors are Microsoft employees studying a Microsoft product (GitHub Copilot), creating an inherent financial interest that is not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not a benchmark evaluation of a pre-trained model's capabilities."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model is being evaluated on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration. The interview protocol evolved iteratively (Section 3.3), which is standard for qualitative research but means the study was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval anywhere in the paper. Section 3.1 discusses ethical considerations (privacy, anonymity, non-judgmental environments) but does not mention formal ethics review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports participant demographics: location (7 countries), role (Senior SWE, SWE II, SWE), and Copilot usage duration. Figure 2 shows usage distributions for frequent and infrequent groups."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes inclusion criteria: same primary programming language, usage gap in top third, same career stage/title/level/country/manager. Exclusion: Germany and Norway for regulatory reasons, pairs where one partner didn't agree."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is an observational/qualitative study, not an experiment with random assignment to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a qualitative interview study, not an experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 reports: 670 invitations → 315 agreed → 152 from 76 pairs → 56 interviews conducted → 2 excluded when pair partners dropped out → 54 final participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no computational method whose cost would be relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Frequent GenAI tool users view tools as collaborative partners while infrequent users view them as utility features, and this perception shapes subsequent usage patterns.",
       "evidence": "Section 4.1 with quotes from multiple participants: PID2 uses Copilot 'as a colleague', PID7 compares it to 'a stack overflow search'. Contrasting pairs (PID7/PID23) illustrate how perception differences lead to different engagement patterns within the same team.",
       "supported": "moderate"
     },
     {
       "claim": "Frequent users demonstrate adaptive persistence when facing challenges while infrequent users more often quickly abandon the tool.",
       "evidence": "Section 4.3 with paired comparisons: PID27 breaks tasks into smaller components when facing hallucination; PID24 gives up after compile errors and reverts to manual methods. PID24/PID26 pair directly contrasts responses to similar prompt engineering challenges. Figure 3 shows distribution differences.",
       "supported": "moderate"
     },
     {
       "claim": "A 'Productivity Pressure Paradox' exists where organizational expectations for rapid productivity gains without learning support undermines the skill development needed to achieve those gains.",
       "evidence": "Section 5.4 with quotes from multiple participants: PID20 notes rising expectations, PID46 describes the 'chicken or egg problem' of investing time in prompt crafting, PID6 describes pressure-induced reversion to familiar methods, PID44 describes being overburdened. Figure 4 illustrates the cycle.",
       "supported": "moderate"
     },
     {
       "claim": "Team-specific demonstrations and context-specific resources are more valuable than generic AI guidance for promoting adoption.",
       "evidence": "Section 5.2: PID48 calls for 'a comprehensive guide of how to use it for specific projects'; PID6 describes a colleague's shared-screen demo of using agent mode for a common team task as 'very helpful'. PID33 describes being overwhelmed by 10+ generic AI meetings per week.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Four of nine authors are Microsoft employees. The study examines GitHub Copilot adoption at Microsoft. While the findings are largely about adoption barriers (not product promotion), the framing positions challenges as organizational rather than product-related, potentially deflecting from product quality issues. No formal competing interests statement is provided."
     },
     {
       "flag": "No IRB/ethics approval mentioned",
       "detail": "The study interviews 54 human participants about sensitive workplace topics (tool usage patterns, performance pressure, job security fears) at their employer, yet no IRB or ethics board approval is mentioned despite a detailed ethical considerations section."
     },
     {
       "flag": "Potential social desirability bias",
       "detail": "Section 3.6 acknowledges the company's 'strong pro-AI culture may have influenced participants' responses.' Infrequent users at a company 'vocally committed to AI adoption' may understate resistance or overstate willingness to adopt. The study frames this as a limitation but does not attempt to measure or mitigate it beyond neutral question wording."
     },
     {
       "flag": "No quantification of qualitative patterns",
       "detail": "The paper explicitly avoids frequency counts, citing guidance against quantifying qualitative data. While methodologically defensible, this means claims like 'many frequent-AI users' vs 'many infrequent-AI users' cannot be verified for how dominant these patterns actually were. Figure 3 shows proportional distributions but without counts."
     }
   ],
   "cited_papers": [
     {
       "title": "Dear Diary: A randomized controlled trial of Generative AI coding tools in the workplace",
       "authors": ["Jenna Butler", "Jina Suh", "Sankeerti Haniyur", "Constance Hadley"],
       "year": 2025,
       "relevance": "RCT of GenAI coding tools in the workplace, finding mixed outcomes including decreased efficiency for some developers."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Empirical measurement of AI impact on developer productivity showing mixed results."
     },
     {
       "title": "Navigating the complexity of generative ai adoption in software engineering",
       "authors": ["Daniel Russo"],
       "year": 2024,
       "relevance": "Study of GenAI adoption factors in SE finding compatibility outweighs traditional UTAUT factors."
     },
     {
       "title": "Exploring Individual Factors in the Adoption of LLMs for Specific Software Engineering Tasks",
       "authors": ["Stefano Lambiase", "Gemma Catolino", "Fabio Palomba", "Filomena Ferrucci", "Daniel Russo"],
       "year": 2025,
       "arxiv_id": "2504.02553",
       "relevance": "Examines how adoption drivers vary by task type, finding peer opinions can backfire by creating unrealistic expectations."
     },
     {
       "title": "A large-scale survey on the usability of ai programming assistants: Successes and challenges",
       "authors": ["Jenny T Liang", "Chenyang Yang", "Brad A Myers"],
       "year": 2024,
       "relevance": "Large-scale survey documenting usability challenges developers face with AI programming assistants."
     },
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key productivity study estimating Copilot's impact on developer task completion speed."
     },
     {
       "title": "Navigating the jagged technological frontier: Field experimental evidence of the effects of AI on knowledge worker productivity and quality",
       "authors": ["Fabrizio Dell'Acqua", "Edward McFowland III", "Ethan R Mollick"],
       "year": 2023,
       "relevance": "Influential field experiment showing AI improves some tasks but harms performance on others (jagged frontier concept)."
     },
     {
       "title": "What Guides Our Choices? Modeling Developers' Trust and Behavioral Intentions Towards GenAI",
       "authors": ["Rudrajit Choudhuri", "Bianca Trinkenreich", "Rahul Pandita"],
       "year": 2024,
       "arxiv_id": "2409.04099",
       "relevance": "Models trust factors in developer GenAI adoption including system quality, functional value, and goal alignment."
     },
     {
       "title": "Copiloting the future: How generative AI transforms Software Engineering",
       "authors": ["Leonardo Banh", "Florian Holldack", "Gero Strobel"],
       "year": 2025,
       "relevance": "Survey on how GenAI transforms SE, documenting enablers, barriers, and ethical concerns."
     },
     {
       "title": "Human-AI Experience in Integrated Development Environments: A Systematic Literature Review",
       "authors": ["Agnia Sergeyuk", "Ilya Zakharov", "Ekaterina Koshchenko", "Maliheh Izadi"],
       "year": 2025,
       "arxiv_id": "2503.06195",
       "relevance": "Systematic review of human-AI interaction in IDEs, relevant to understanding developer experience with AI tools."
     },
     {
       "title": "\"It would work for me too\": How online communities shape software developers' trust in AI-powered code generation tools",
       "authors": ["Ruijia Cheng", "Ruotong Wang", "Thomas Zimmermann", "Denae Ford"],
       "year": 2024,
       "relevance": "Studies how peer influence in online communities shapes developer trust in AI code generation tools."
     },
     {
       "title": "AI tool use and adoption in software development by individuals and organizations: a grounded theory study",
       "authors": ["Ze Shi Li", "Nowshin Nawar Arony", "Ahmed Musa Awon", "Daniela Damian", "Bowen Xu"],
       "year": 2024,
       "arxiv_id": "2406.17325",
       "relevance": "Grounded theory study of AI tool adoption in software development at individual and organizational levels."
     }
   ]
 }