ai-research-survey

scan.json (22858B)

---

 {
   "paper": {
     "title": "Developer Productivity With and Without GitHub Copilot: A Longitudinal Mixed-Methods Case Study",
     "authors": ["Viktoria Stray", "Elias Goldmann Brandtzæg", "Viggo Tellefsen Wivestad", "Astri Barbala", "Nils Brede Moe"],
     "year": 2025,
     "venue": "Unknown (appears to be a conference or journal submission)",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Scripts for mining GitHub data, cleaning, analyzing, statistical tests, and generating plots are available at https://figshare.com/s/736af5662435675e7914 (Appendix A)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The commit dataset and survey/interview data are not released. Only analysis scripts are shared via Figshare. The underlying data (26,317 commits from 39 developers) is not publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, library versions, etc.) is mentioned in the paper. Only Python and PyDriller are named without versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The Figshare link contains scripts but the paper does not describe how to run them to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are shown in Figures 2 and 3 (shaded areas and error bars). Explicitly stated: 'The shaded areas around each line represent the 95% confidence intervals.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U tests are used to compare Copilot users vs. non-users (p < 0.00555). Spearman correlation with p-values reported (ρ ≈ 0.17, p = 0.406). Bonferroni correction applied for six tests (α = 0.0083)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Spearman ρ values reported (ρ ≈ 0.17 for commit count, ρ ≈ 0.09 for net lines). Raw differences in weekly activity provided (e.g., 188 vs 80 lines added, 200 vs 70 after). These provide sufficient context for magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes (N=39 developers, N=25 Copilot users, N=14 non-users, N=13 interviewees). No power analysis. The small N is not discussed as a limitation of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported in Figures 2 and 3, providing spread measures for the weekly activity data across the observation period."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Non-users (n=14) serve as a comparison group. Pre-adoption period serves as a temporal baseline for Copilot users. Both before/after and user/non-user comparisons are made."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is between contemporaneous groups (Copilot users vs. non-users) within the same organization during the same time period."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational case study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: commit frequency, lines added, lines removed, net lines changed, plus perceived productivity (Likert scale), plus qualitative interview data. Six activity metrics examined for correlation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes survey responses on perceived productivity and 13 semi-structured interviews with developers, providing human evaluation of the tool's impact."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is an observational/case study, not a machine learning evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by Copilot users vs. non-users, before vs. after adoption, and across multiple metrics (insertions, deletions, net diff, commits). Roles are shown in Figure 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses developers who were skeptical or declined Copilot, developers who tried it and stopped, and the discrepancy between perceived and measured productivity. Developer concerns about hidden mistakes and subtle bugs are noted."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main finding is essentially a negative result: no statistically significant change in commit-based activity after Copilot adoption, and no significant correlation between perceived and measured productivity."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: Copilot users were more active pre-adoption (Figures 2-3, Mann-Whitney U), no statistically significant change post-adoption, discrepancy between metrics and subjective experience (Figure 4, Spearman correlation)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to avoid strong causal claims. It uses language like 'suggests' and explicitly notes the self-selection bias. Section 5 acknowledges the longitudinal view shows the activity gap pre-existed Copilot, cautioning against causal interpretation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper frames results within NAV IT specifically and notes differences from controlled experiments (Peng et al., Cui et al.). It acknowledges this is a single organization case study in a Norwegian public sector context."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Self-selection bias is discussed in detail. Alternative explanations for perceived productivity (subject-expectancy effect, reduced cognitive load vs. actual output) are explored. Role differences, seasonal patterns, and early adoption phase are all considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GitHub Copilot' without specifying the version. No mention of which Copilot model version or underlying LLM was used during the study period."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates GitHub Copilot as a third-party black-box tool used by developers in their workflow. No custom prompts were used as part of the research methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates Copilot as a black-box tool in a naturalistic setting. There are no hyperparameters to report — developers used Copilot with default settings in their normal workflow."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a third-party tool (GitHub Copilot) as a black box. Authors cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data pipeline: 37,974 original commits → duplicate removal → outlier trimming at 95th percentile → exclusion of low-activity users (< 1 commit/week) → 26,317 final commits. Weekly aggregation and zero-imputation documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Some limitations are discussed inline (e.g., self-selection bias in Section 5, metrics limitations), but there is no substantive dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: self-selection bias (Copilot adopters were already more active), subject-expectancy effect for perceived productivity, commit-based metrics failing to capture real productivity, and seasonal fluctuations. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it implicitly limits scope to NAV IT, it does not make explicit statements about what populations, settings, or claims are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw commit data and survey/interview transcripts are not publicly available. Only analysis scripts are shared. The underlying data cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: GitHub API for repo list, PyDriller for commit mining, survey distributed March-April 2024, 13 interviews in Nov-Dec 2023 (30-60 min each, avg 47 min), AI transcription with manual review."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Recruitment is described: 100 developers at NAV IT volunteered for early Copilot access in September 2023. Survey respondents who shared GitHub usernames (63 total) were linked. 39 remained after filtering. Self-selection into Copilot group acknowledged."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented in Section 3: 37,974 commits → duplicate removal → outlier trimming at 95th percentile → low-activity user exclusion → 26,317 commits from 39 developers across 703 repos. Weekly aggregation into N=4095 observations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed in Acknowledgments: 'This work was supported by the Research Council of Norway through the projects TransformIT (grant 321477) and Kairos (grant 357147).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: University of Oslo and SINTEF. These are academic/research institutions, not GitHub/Microsoft."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Research Council of Norway is a public funding agency with no financial stake in whether GitHub Copilot improves productivity."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on a benchmark. It observes developer behavior with a tool in a naturalistic setting."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not evaluate model capability on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or similar). The study involves human participants (surveys and interviews) but was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval despite collecting survey data and conducting interviews with identifiable developers."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Participant roles are reported in Figure 1 (front-end developer, back-end developer, tech lead, platform engineer, data scientist/analyst, data engineer). The organizational context (NAV IT, Norwegian public sector) is described."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria stated: developers who provided GitHub usernames in survey (63 initially). Exclusion: users with fewer than 1 commit/week average excluded. Final sample: 39 developers (25 Copilot users, 14 non-users)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not an experiment. Copilot adoption was voluntary (self-selected), not randomized. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is an observational study. Blinding is not feasible when studying voluntary tool adoption."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition documented: 63 survey respondents provided GitHub usernames, reduced to 39 after filtering low-activity users. Original 37,974 commits reduced to 26,317 after cleaning. The pipeline from initial to final sample is clear."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational case study of tool adoption, not proposing a method with computational costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational case study. No significant computation was involved beyond data mining scripts."
       }
     }
   },
   "claims": [
     {
       "claim": "Copilot users were consistently more active than non-users even before Copilot's introduction, indicating self-selection bias.",
       "evidence": "Mann-Whitney U tests (p < 0.00555) confirm significantly higher activity for Copilot users both before and after adoption. Figures 2 and 3 show the pre-existing gap. Section 4.1.",
       "supported": "strong"
     },
     {
       "claim": "No statistically significant changes in commit-based activity were found for Copilot users after adoption.",
       "evidence": "Section 4.1: Minor increases observed (e.g., +16 net lines/week) but not statistically significant. Figures 2-3 show no visible spike at adoption.",
       "supported": "strong"
     },
     {
       "claim": "There is no significant correlation between perceived productivity and commit-based activity metrics among Copilot users.",
       "evidence": "Section 4.2: Spearman ρ ≈ 0.17 (p = 0.406) for commit count. None of six activity metrics showed significant correlation with perceived productivity after Bonferroni correction (α = 0.0083).",
       "supported": "strong"
     },
     {
       "claim": "Copilot did not negatively impact code quality metrics.",
       "evidence": "Section 4.1: 'structural metrics (e.g., function complexity, average module size) remained virtually unchanged and showed no significant differences between users and non-users.' No detailed analysis or statistics provided for this claim.",
       "supported": "weak"
     },
     {
       "claim": "Copilot's value may lie in reducing cognitive load and improving developer experience rather than increasing raw output.",
       "evidence": "Interview quotes from I12, I13 about smoother flow and faster starting points. Survey data showing no decreases in perceived productivity. Section 5 discussion.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational", "case-study", "qualitative"],
   "key_findings": "In a longitudinal study of 39 developers at NAV IT over two years, Copilot users showed no statistically significant change in commit-based activity after adoption, despite perceiving productivity improvements. Copilot adopters were already more active developers before the tool's introduction, indicating self-selection bias. No significant correlation was found between perceived productivity and any of six commit-based activity metrics, suggesting that traditional output metrics fail to capture the subjective benefits developers report from AI coding assistants.",
   "red_flags": [
     {
       "flag": "Small sample size",
       "detail": "Only 39 developers (25 Copilot users, 14 non-users) from a single organization. No power analysis. The small N limits statistical power for detecting effects and makes subgroup analysis unreliable."
     },
     {
       "flag": "Self-selection bias acknowledged but not controlled",
       "detail": "Copilot adoption was voluntary, creating inherent self-selection. The paper acknowledges this but cannot control for it since there was no randomization. Propensity score matching or similar techniques were not attempted."
     },
     {
       "flag": "Code quality claim weakly supported",
       "detail": "The claim that Copilot did not negatively impact code quality is stated without detailed analysis or statistics. Only a brief mention of structural metrics being 'virtually unchanged' with no supporting data shown."
     },
     {
       "flag": "No ethics approval mentioned",
       "detail": "The study collects survey data and conducts interviews with identifiable developers (linked to GitHub usernames) but does not mention IRB or ethics board approval."
     }
   ],
   "cited_papers": [
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["S. Peng", "E. Kalliamvakou", "P. Cihon", "M. Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Foundational RCT on Copilot productivity (55.8% faster task completion), directly compared in this study."
     },
     {
       "title": "The effects of generative AI on high skilled work: Evidence from three field experiments with software developers",
       "authors": ["Z. K. Cui", "M. Demirer", "S. Jaffe", "L. Musolff", "S. Peng", "T. Salz"],
       "year": 2024,
       "relevance": "Field experiments at Microsoft/Accenture showing 26% task completion increase with Copilot, contrasted with this study's null finding."
     },
     {
       "title": "Large language models, small labor market effects",
       "authors": ["A. Humlum", "E. Vestergaard"],
       "year": 2025,
       "relevance": "Large-scale Danish study finding only 3% time savings from GenAI tools, aligning with this study's modest findings."
     },
     {
       "title": "Generative AI at work",
       "authors": ["E. Brynjolfsson", "D. Li", "L. Raymond"],
       "year": 2025,
       "relevance": "Reports relatively small productivity gains from GenAI in customer support, supporting pattern of modest real-world effects."
     },
     {
       "title": "Ironies of generative AI: Understanding and mitigating productivity loss in human-AI interaction",
       "authors": ["A. Simkute", "L. Tankelevitch", "V. Kewenig", "A. E. Scott", "A. Sellen", "S. Rintel"],
       "year": 2025,
       "relevance": "Studies role shift from creating to supervising AI outputs, relevant to understanding perceived vs measured productivity."
     },
     {
       "title": "Generative AI and developer workflows: How GitHub Copilot and ChatGPT influence solo and pair programming",
       "authors": ["V. Stray", "N. B. Moe", "N. Ganeshan", "S. Kobbenes"],
       "year": 2025,
       "relevance": "Companion study from same research group examining Copilot's influence on developer workflows and collaboration."
     },
     {
       "title": "The SPACE of developer productivity: There's more to it than you think",
       "authors": ["N. Forsgren", "M.-A. Storey", "C. Maddila", "T. Zimmermann", "B. Houck", "J. Butler"],
       "year": 2021,
       "relevance": "Framework for multidimensional developer productivity measurement (satisfaction, performance, activity, communication, efficiency) referenced in this study's discussion."
     },
     {
       "title": "Exploring GenAI in software development: Insights from a case study in a large Brazilian company",
       "authors": ["G. V. Pereira", "V. Jackson", "R. Prikladnicki"],
       "year": 2025,
       "relevance": "Mixed-methods study of GenAI adoption in corporate setting finding perceived task completion improvements."
     },
     {
       "title": "Is GitHub Copilot a substitute for human pair-programming? An empirical study",
       "authors": ["S. Imai"],
       "year": 2022,
       "relevance": "Early empirical study examining Copilot's role relative to pair programming practices."
     },
     {
       "title": "Practices and challenges of using GitHub Copilot: An empirical study",
       "authors": ["B. Zhang", "P. Liang", "X. Zhou", "A. Ahmad", "M. Waseem"],
       "year": 2023,
       "relevance": "Empirical study of Copilot usage practices and challenges, relevant to understanding adoption patterns."
     }
   ]
 }