ai-research-survey

scan.json (26473B)

---

 {
   "paper": {
     "title": "Generative AI for Pull Request Descriptions: Adoption, Impact, and Developer Interventions",
     "authors": [
       "Tao Xiao",
       "Hideaki Hata",
       "Christoph Treude",
       "Kenichi Matsumoto"
     ],
     "year": 2024,
     "venue": "Proc. ACM Softw. Eng.",
     "arxiv_id": "2402.08967",
     "doi": "10.1145/3643773"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "This study of 18,256 GitHub PRs finds growing adoption of Copilot for Pull Requests during its early access period (March–August 2023), with copilot:summary as the most popular marker tag. Using propensity score weighting with 17 covariates and entropy balancing, the authors estimate Copilot for PRs reduces review time by 19.3 hours and increases merge likelihood by 1.57× (OR=1.57, 95% CI 1.35–1.84). Qualitative analysis of 1,437 revisions reveals developers frequently complement AI-generated descriptions with templates and links, while also partially deleting (22.9%) or refining (20.8%) Copilot's suggestions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided on GitHub (https://github.com/NAIST-SE/CopilotForPRsEarlyAdoption) and archived on Zenodo (doi: 10.5281/zenodo.10656106). The Data Availability section states it includes 'lists of studied PRs, features of PRs, coding results for RQ3, and scripts.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package includes lists of studied PRs from GitHub (both Copilot and non-Copilot), PR features used in RQ2, and coding results for RQ3. The underlying GitHub PR data is publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. The replication package contents described do not include environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The replication package includes scripts but no documented procedure for reproducing the main results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The odds ratio for RQ2.2 is reported with a 95% confidence interval (1.35 to 1.84). Table 4 reports standard errors for all regression coefficients."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports p-values for all regression coefficients. The treatment effect is significant at p=1.64e-17 for review time (RQ2.1) and p<0.001 for merge likelihood (RQ2.2). Inter-rater agreement is reported via Cohen's kappa."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Average Treatment Effect on the Treated (ATT) of -19.3 hours is reported for review time. For merge likelihood, an odds ratio of 1.57 is reported with baseline context (84% vs 71% merge rates). These provide meaningful magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or formal sample size justification is provided. The study uses all available data from the observation period (18,256 Copilot PRs, 54,188 non-Copilot PRs) but does not discuss whether this is adequate for the claimed effects."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported for all regression coefficients in Table 4. For the qualitative analysis, inter-rater reliability is quantified via Cohen's kappa (0.64 for RQ3.1, 0.62 for RQ3.2). These are the appropriate uncertainty measures for this study design."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares 17,177 merged/closed Copilot PRs (treatment) against 50,695 merged/closed non-Copilot PRs (control) from the same 146 GitHub repositories during the same time period."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Control PRs are from the same time period (March–August 2023) and same repositories as the treatment PRs, ensuring temporal comparability."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of a third-party service, not a system with decomposable components. There is no system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct outcome variables are used: review time in hours (RQ2.1) and binary merge likelihood (RQ2.2). The qualitative analysis (RQ3) adds complementary findings via categorical coding."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 involves qualitative analysis by three human raters who independently coded PR revisions. Two rounds of coding achieved substantial inter-rater agreement (κ=0.64 for RQ3.1, κ=0.62 for RQ3.2)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not a predictive modeling task. There is no train/test split or held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by marker tag type (Table 3), 13 categories of supplementary information (Table 5), and 7 categories of editorial actions with subcategories (Table 6). Table 4 breaks down regression coefficients by language, purpose, and other covariates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The qualitative analysis documents cases where developers deleted Copilot output (22.9%), replaced it (14.7%), or explicitly rejected it (e.g., developer commenting 'Nope, you didn't get it this time'). 24 false positives in the coding are also reported (5.8%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All quantitative results show positive effects of Copilot for PRs (reduced review time, higher merge rate, growing adoption). No failed analyses, abandoned approaches, or configurations that didn't work are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three claims are supported: (1) growing adoption shown in Figures 3–4 and Table 3; (2) reduced review time (-19.3h) and higher merge likelihood (OR=1.57) in Table 4; (3) developer adaptations documented in Tables 5–6 with 13 categories of supplementary information and 7 editorial action types."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames RQ2 as causal inference and uses propensity score weighting with entropy balancing across 17 covariates. Figure 2 demonstrates covariate balance achieved post-weighting (all absolute mean differences <0.10). This is a recognized quasi-experimental design, though the paper acknowledges potential unobserved confounders in Section 6.2."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states: 'our results are not universally applicable to the broader open-source developer community, but are more pertinent to these early adopters. Developers who are less eager to adopt new technologies might use Copilot for PRs less or differently.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses specific threats: bot detection methods might miss bots (construct validity), 17 confounders are controlled but others may exist (internal validity), early adopters may differ from general developers (external validity). The PSW analysis explicitly addresses 17 potential confounding variables."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper defines 'review time' as 'the time interval between the PR creation date and closed date in hours' but does not discuss that this measures calendar time to close, not actual review effort. Time-to-close includes idle time, timezone effects, and queuing time, which are distinct from active review engagement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.1 mentions 'the GPT-4 model by OpenAI' without specifying a version (e.g., gpt-4-0314 or gpt-4-0613). Since the paper studies the outputs of this model via Copilot for PRs, the exact version affects the study's findings."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not prompt any model directly. It studies an external service (Copilot for PRs) where the interface is marker tags (copilot:summary, copilot:walkthrough, etc.), which are fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The underlying LLM hyperparameters (temperature, sampling strategy, etc.) used by Copilot for PRs are not reported. While the authors cannot control these, they affect the generated content being studied, and this limitation is not acknowledged."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates Copilot for PRs as a third-party black-box service. The authors cannot be expected to describe GitHub's internal scaffolding for the copilot4prs bot."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 thoroughly documents the data pipeline: initial GraphQL search yielding 18,858 PRs → removing false positives → excluding obsolete uses (18,322) → filtering bot PRs (18,256). For RQ3: 46,700 revisions → 18,486 post-Copilot edits → 4,391 after marker tag reapplication filter → 1,437 revisions from 311 PRs with substantive edits."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Threats to validity' provides substantive discussion organized into construct validity, internal validity, and external validity subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: bot detection methods 'might not capture every true negative' (construct); 'there may be other confounding variables not accounted for' in the 17-covariate model (internal); early adopters have 'an inherent threat to the external validity' and results are 'more pertinent to these early adopters' (external)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states: 'our results are not universally applicable to the broader open-source developer community, but are more pertinent to these early adopters. Developers who are less eager to adopt new technologies might use Copilot for PRs less or differently compared to the early adopters studied in this work.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (GitHub + Zenodo doi:10.5281/zenodo.10656106) includes 'lists of studied PRs from GitHub, both with and without the use of Copilot for PRs,' PR features for RQ2, and coding results for RQ3."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the collection procedure in detail: GitHub GraphQL search for PRs containing 'Generated by Copilot', time period (up to 31 August 2023), the strategy for handling the 1000-result API limit by halving time periods, and filtering for copilot4prs bot edits."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how PRs and repositories were identified: GraphQL search for specific phrases, filtering by bot edits, time-period constraints, and bot exclusion using methods from Golzadeh et al. (2022). The selection process for both treatment (18,256 PRs) and control (54,188 PRs) groups is documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with counts at each stage: 18,858 initial PRs → false positive removal → 18,322 non-obsolete → 18,256 after bot filtering. For RQ3: 46,700 revisions → 18,486 post-Copilot → 4,391 after reapplication filter → 1,437 from 311 PRs. Exclusion criteria are stated at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding: JSPS Grant-in-Aid for JSPS Fellows JP23KJ1589, JSPS KAKENHI Grant Numbers JP20H05706, and JST PRESTO Grant Number JPMJPR22P6."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors' affiliations are listed: Nara Institute of Science and Technology, Shinshu University, and Singapore Management University. None are affiliated with GitHub/Microsoft, so there is no product-evaluator conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from JSPS and JST (Japanese government research agencies), which have no financial interest in the success or failure of GitHub Copilot for PRs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The absence of a formal declaration is not equivalent to the absence of conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It is an observational study of the adoption and impact of a third-party AI service on software development processes."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper is an observational mining study of GitHub PRs, not a benchmark evaluation of a model."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed in this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study mines public GitHub PR data and the qualitative coding is performed by the authors themselves, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes publicly available GitHub data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Developer characteristics are described at the repository level (e.g., repo age), not at the individual level."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria for PRs and repositories are described in Section 3."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is an observational study with no experimental assignment of participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data attrition through filtering is documented in the data pipeline (Section 3)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of an existing service, not a proposed method. The paper does not invoke any model or incur inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study. The computational requirements (GraphQL queries, regression analysis) are negligible and not the focus of the work."
       }
     }
   },
   "claims": [
     {
       "claim": "Copilot for PRs shows growing adoption, with copilot:summary as the most popular marker tag (13,231 instances across 18,256 PRs).",
       "evidence": "Figure 3 shows cumulative time-series of PRs growing steadily from March to August 2023. Table 3 shows 13,231 copilot:summary instances. 50 repositories have >50% of PRs powered by Copilot. Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "Copilot for PRs reduces code review time by an average of 19.3 hours.",
       "evidence": "Table 4 reports ATT of -19.3 hours (std error 2.27, p=1.64e-17) from propensity score weighted linear regression with entropy balancing across 17 covariates. Median treatment review time is 12.17h vs 16.09h control. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "PRs generated with Copilot for PRs are approximately 1.57 times more likely to be merged (OR=1.57, 95% CI 1.35–1.84, p<0.001).",
       "evidence": "Propensity score weighted logistic regression. Raw merge rates: 84% (Copilot) vs 71% (non-Copilot). Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "Developers frequently complement AI-generated descriptions with templates (22.8%) and associated links (22.7%), while the most common editorial action is partial deletion (22.9%).",
       "evidence": "Qualitative coding of 1,437 revisions from 311 PRs by three independent raters achieving κ=0.64 (RQ3.1) and κ=0.62 (RQ3.2). Tables 5 and 6 detail frequencies. Section 5.3.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Selection bias in treatment group",
       "detail": "Early adopters of Copilot for PRs are likely more tech-savvy, productive developers in better-maintained repositories. This systematic difference could confound the treatment effect even after propensity score adjustment, as unobserved characteristics (developer skill, project management quality) may drive both Copilot adoption and faster reviews."
     },
     {
       "flag": "No sensitivity analysis for unobserved confounders",
       "detail": "The causal claims rely on propensity score weighting, which assumes all relevant confounders are observed. No Rosenbaum bounds, E-value analysis, or other sensitivity analysis is performed to assess how robust the causal estimates are to potential unobserved confounding."
     },
     {
       "flag": "Large pre-weighting covariate imbalance",
       "detail": "The median description length differs by 5× between groups (1,825 vs 343 characters, Table 2), indicating Copilot PRs are fundamentally different from non-Copilot PRs. While entropy balancing adjusts for this, extreme initial imbalances raise concerns about extrapolation and the reliability of weighted estimates."
     },
     {
       "flag": "ATT magnitude seems implausibly large",
       "detail": "The 19.3-hour reduction in review time is larger than the control group median (16.09 hours) and treatment group median (12.17 hours), suggesting the effect may be driven by outliers or heavy tails rather than a typical improvement. The paper does not report the median treatment effect or discuss distributional sensitivity."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex paper establishing LLM code generation evaluation methodology."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "arxiv_id": "2308.10620",
       "relevance": "Comprehensive systematic review of 229 papers on LLMs in software engineering from 2017–2023."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation of LLM code generation correctness, relevant to understanding AI code quality."
     },
     {
       "title": "Conversational automated program repair",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2301.13246",
       "relevance": "Applies conversational LLMs to program repair, demonstrating agentic AI programming capability."
     },
     {
       "title": "From Copilot to Pilot: Towards AI Supported Software Development",
       "authors": ["Rohith Pudari", "Neil A Ernst"],
       "year": 2023,
       "arxiv_id": "2303.04142",
       "relevance": "Directly examines the evolution of AI support in software development, including GitHub Copilot."
     },
     {
       "title": "Examining zero-shot vulnerability repair with large language models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad"],
       "year": 2023,
       "relevance": "Evaluates LLMs for security vulnerability repair, relevant to AI safety in code generation."
     },
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Proposes self-debugging for LLMs, a key agentic capability for autonomous code improvement."
     },
     {
       "title": "ChatGPT: A Study on its Utility for Ubiquitous Software Engineering Tasks",
       "authors": ["Giriprasad Sridhara", "Sourav Mazumdar"],
       "year": 2023,
       "arxiv_id": "2305.16837",
       "relevance": "Evaluates ChatGPT across multiple software engineering tasks including code review."
     },
     {
       "title": "How Effective Are Neural Networks for Fixing Security Vulnerabilities",
       "authors": ["Yi Wu", "Nan Jiang", "Hung Viet Pham"],
       "year": 2023,
       "arxiv_id": "2305.18607",
       "relevance": "Assesses neural network effectiveness for security vulnerability repair in code."
     },
     {
       "title": "Evaluating the Code Quality of AI-Assisted Code Generation Tools: An Empirical Study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT",
       "authors": ["Burak Yetiştiren", "Işık Özsoy", "Miray Ayerdem", "Eray Tüzün"],
       "year": 2023,
       "arxiv_id": "2304.10778",
       "relevance": "Comparative evaluation of major AI code generation tools including GitHub Copilot."
     },
     {
       "title": "Self-collaboration Code Generation via ChatGPT",
       "authors": ["Yihong Dong", "Xue Jiang", "Zhi Jin", "Ge Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Explores multi-agent collaboration in LLM code generation."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Examines how large pre-trained language models transform automated program repair."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable data for teams deciding whether to adopt Copilot for PRs, with specific metrics on time savings and merge rates."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results largely confirm the expected narrative that AI tools help with development tasks; the finding that developers frequently delete AI suggestions is mildly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised by the findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts; the study is a straightforward empirical investigation."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No tool or demo to try; the replication package contains analysis scripts but not an end-user tool."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Directly studies GitHub Copilot, one of the most well-known AI developer tools from Microsoft/GitHub."
     }
   }
 }