ai-research-survey

scan.json (21638B)

---

 {
   "paper": {
     "title": "The Impact of Large Language Models (LLMs) on Code Review Process",
     "authors": ["Antonio Collante", "Samuel Abedu", "SayedHassan Khatoonabadi", "Ahmad Abdellatif", "Ebube Alor", "Emad Shihab"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.11034"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public repository: https://github.com/acollant/GPT-Assistance-PR with scripts and dataset (Section 1, contributions)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is shared at the same GitHub repository. The paper states 'we publicly share our scripts and dataset online.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The repository is referenced but no specific instructions for replicating results are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. Only point estimates (medians, percentages) are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U test is used to compare GPT-assisted vs non-assisted PRs (Sections 4.1, 4.2), with p-values reported (e.g., 4.88e-11 for is_gpt-assisted in Table 2)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage reductions with baseline context: 61% reduction in merge time (9 vs 23 hours), 66.7% review time reduction, 87.5% waiting time reduction. Regression coefficients with standard errors are also provided in Table 2."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 25,473 PR dataset size is described but not justified via power analysis. The 450-PR sample for heuristic validation uses 95% confidence / 5% margin of error, but the main analysis sample size is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the main results. Only medians are given without IQR or other dispersion measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-non-assisted PRs serve as the baseline comparison group, matched using Manhattan distance on structural features (Section 3.3.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is between contemporaneous GPT-assisted and non-assisted PRs from the same time period and platform (GitHub, collected May 2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study comparing two groups, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: overall merge time, phase-specific times (review, waiting for change, change), and task type distributions across phases."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual annotation was performed: 450 PRs manually reviewed for heuristic development (3 annotators), 310 PRs manually labeled for task classification (RQ3, Section 4.3)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study, not a predictive modeling task requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by PR phase (submission, review, waiting, change, post-acceptance) in Table 3, and by task type (enhancement, implementation, bug fix, testing, documentation, other) in Table 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where reviewers rejected GPT-generated code (e.g., 'I will not blindly accept any AI-generated code' in Section 4.3) and phases where GPT had no impact (submission, post-acceptance)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports no significant difference in the At Change phase between GPT-assisted and non-assisted PRs (Table 3, both 1 hour), and notes GPT had no detectable impact during submission and post-acceptance phases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 61% merge time reduction, 66.7% review time reduction, and 87.5% waiting time reduction are supported by results in Tables 2-3 and Figure 3. Task distribution claims (60% optimization, 26% bug fixing, 12% documentation) are supported by Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language ('GPT-assisted PRs reduced median resolution time', 'can boost the effectiveness', 'leading to considerable time savings') but the study design is observational with matched comparisons, not a randomized experiment. The regression controls for some confounders but cannot establish causation. The authors acknowledge this in Section 6.2 but the main text uses causal framing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'The Impact of Large Language Models on Code Review Process' broadly, but the study only examines GitHub PRs where GPT usage was explicitly mentioned. The abstract and conclusion make broad claims about 'GPT to accelerate code reviews' without bounding to the specific population of PRs that openly reference GPT."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses multiple alternative explanations: task simplicity, developer experience, team workflow dynamics, and code change complexity as potential confounders. Section 6.1 specifically notes that 'other factors, such as task simplicity or team dynamics, could also influence outcomes.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures PR merge time and phase durations but frames results as 'efficiency', 'effectiveness', and 'code quality' improvement without distinguishing between the proxy (time metrics) and the claimed outcomes. Faster reviews do not necessarily mean better reviews, but this gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper studies GPT usage by developers in the wild, not a specific model version controlled by the researchers. The paper refers generically to 'GPT' and 'ChatGPT' without controlling which version developers used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its methodology — it mines existing PRs where developers used GPT externally."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters for the statistical models (e.g., R's lm function configuration) or the n-gram extraction are reported beyond the 5-gram maximum."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: filtering criteria (≥10 stars, ≥2 contributors, ≥1 event), keyword search methodology, n-gram extraction approach, heuristic refinement rounds, and the Manhattan distance matching procedure (Sections 3.1-3.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated multi-subsection discussion covering construct, internal, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: false negatives from keyword-based detection, GitHub-only platform bias, the ≥10 stars selection criterion introducing bias, and the inability to capture undisclosed GPT use (Section 6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While threats to validity are discussed, the paper does not explicitly state what the results do NOT show. The conclusion makes broad claims without bounding them to the specific population studied (PRs that explicitly mention GPT on GitHub)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly shared at https://github.com/acollant/GPT-Assistance-PR, allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection: GitHub REST API, keyword 'GPT', collected May 9, 2024, yielding 81,234 PRs filtered to 25,473. Fields collected are enumerated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study mines public GitHub repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Section 3 document the full pipeline: search (81,234 PRs) → filter (25,473) → heuristic labeling (1,600 GPT-assisted) → similarity matching → phase mapping. Counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work was supported by the NSERC CREATE grant number 555406, 2021.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Concordia University and University of Calgary. No authors are affiliated with GPT/OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a Canadian government research funding agency with no financial interest in GPT or code review outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study analyzing developer behavior, not evaluating a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study with no human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — the study mines public GitHub data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — inclusion/exclusion criteria apply to repositories and PRs, not people."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study, not a system with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study with standard statistical analysis; compute budget is not a meaningful concern."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-assisted PRs reduced median resolution time by more than 61% (from 23 hours to 9 hours).",
       "evidence": "Multiple linear regression model (Table 2) shows is_gpt-assisted coefficient of -0.4921 with p=4.88e-11. Figure 3 shows median merge times of 9 vs 23 hours.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-assisted PRs reduce review time by 66.7% (from 3 hours to 1 hour median).",
       "evidence": "Table 3 shows median At Review time of 1 hour for GPT-assisted vs 3 hours for non-assisted. Mann-Whitney U test confirms statistical significance (p < 0.05).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-assisted PRs reduce waiting time before acceptance by 87.5% (from 24 hours to 3 hours median).",
       "evidence": "Table 3 shows median At Waiting for Change of 3 hours for GPT-assisted vs 24 hours for non-assisted. Mann-Whitney U test confirms statistical significance.",
       "supported": "moderate"
     },
     {
       "claim": "Developers predominantly use GPT for code optimization (60%), bug fixing (26%), and documentation updates (12%).",
       "evidence": "Table 4 reports task frequencies from manual analysis of 310 GPT-assisted PRs. Enhancement is 60.26% at Review phase, bug fix is 25.64% at Review.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational"],
   "key_findings": "This observational study of 25,473 GitHub PRs finds that GPT-assisted PRs have 61% shorter median merge times (9 vs 23 hours), with the largest reductions in review time (66.7%) and waiting-before-change time (87.5%). Manual analysis of 310 GPT-assisted PRs shows developers primarily use GPT for code enhancement (60%), bug fixing (26%), and documentation (12%), with minimal use for implementation from scratch or testing. However, these are correlational findings from PRs that explicitly mention GPT, not causal estimates.",
   "red_flags": [
     {
       "flag": "Severe selection bias in treatment group",
       "detail": "GPT-assisted PRs are identified by keyword mentions of 'GPT' in PR text. This captures only PRs where developers openly disclose GPT use, which is a tiny, non-representative subset. Developers who mention GPT may be more experienced, more transparent, or working on simpler tasks. The 1,600 GPT-assisted PRs out of 25,473 (6.3%) likely represent a biased sample."
     },
     {
       "flag": "Causal language with observational design",
       "detail": "The paper uses causal language ('reduced', 'boost effectiveness', 'leading to time savings') throughout, but the observational design with matched comparisons cannot establish causation. The Manhattan distance matching on 4 features is insufficient to rule out confounders like task complexity, developer skill, or project maturity."
     },
     {
       "flag": "No variance or uncertainty quantification on main results",
       "detail": "All main findings are reported as median point estimates (9 hours, 1 hour, 3 hours) without confidence intervals, IQR, or any spread measure. The reader cannot assess result stability."
     },
     {
       "flag": "Matching methodology is weak",
       "detail": "The Manhattan distance matching uses only 4 features (no_commits, PR_size, no_changed_files, project_age) and selects only the single closest match per GPT-assisted PR. Important confounders like programming language, task type, developer experience, project domain, and team size are not controlled for."
     },
     {
       "flag": "R² not reported for regression model",
       "detail": "The paper mentions calculating R² but does not report it, making it impossible to assess how much variance the model explains. The methodology section says they evaluate R² but the results section omits it."
     }
   ],
   "cited_papers": [
     {
       "title": "Unveiling chatgpt's usage in open source projects: A mining-based study",
       "authors": ["R. Tufano", "A. Mastropaolo", "F. Pepe", "O. Dabić", "M. Di Penta", "G. Bavota"],
       "year": 2024,
       "relevance": "Directly relevant study on ChatGPT usage patterns in open source, including commit message generation and PR descriptions."
     },
     {
       "title": "Generative ai for pull request descriptions: Adoption, impact, and developer interventions",
       "authors": ["T. Xiao", "H. Hata", "C. Treude", "K. Matsumoto"],
       "year": 2024,
       "relevance": "Studies Copilot impact on PR merging likelihood, finding 1.57x higher merge rate for Copilot-assisted PRs."
     },
     {
       "title": "AI-powered code review with llms: Early results",
       "authors": ["Z. Rasheed", "M. A. Sami", "M. Waseem"],
       "year": 2024,
       "arxiv_id": "2404.18496",
       "relevance": "Examines LLM-powered code review agents and their dual benefit of enhancing code quality and developer learning."
     },
     {
       "title": "An empirical study on developers' shared conversations with chatgpt in github pull requests and issues",
       "authors": ["H. Hao", "K. A. Hasan", "H. Qin"],
       "year": 2024,
       "relevance": "Studies developer-ChatGPT interactions in GitHub PRs and issues, finding common request types including code generation and review."
     },
     {
       "title": "Automated code review in practice",
       "authors": ["U. Cihan", "V. Haratian", "A. İçöz"],
       "year": 2025,
       "relevance": "Empirical study of LLM-based automated code review across 4,335 PRs in industry, finding 74% application rate but longer closure times."
     },
     {
       "title": "Automated code review using large language models at ericsson: An experience report",
       "authors": ["S. Ramesh", "J. Bose", "H. Singh"],
       "year": 2025,
       "arxiv_id": "2507.19115",
       "relevance": "Industry case study of LLM-augmented code review combining static analysis with prompt engineering at Ericsson."
     },
     {
       "title": "Tales from the trenches: Expectations and challenges from practice for code review in the generative ai era",
       "authors": ["N. Davila", "J. Melegati", "I. Wiese"],
       "year": 2024,
       "relevance": "Identifies five challenges to adopting generative AI in code review: trustworthiness, context, misleading comments, security, token limits."
     },
     {
       "title": "Analyzing developer use of chatgpt generated code in open source github projects",
       "authors": ["B. Grewal", "W. Lu", "S. Nadi", "C.-P. Bezemer"],
       "year": 2024,
       "relevance": "Studies ChatGPT-generated code adoption in open source projects."
     },
     {
       "title": "AI-assisted assessment of coding practices in modern code review",
       "authors": ["M. Vijayvergiya", "M. Salawa"],
       "year": 2024,
       "relevance": "Describes AutoCommenter, an LLM-powered code review tool enforcing coding best practices at scale."
     },
     {
       "title": "How do software developers use chatgpt? an exploratory study on github pull requests",
       "authors": ["M. Chouchen", "N. Bessghaier", "M. Begoug"],
       "year": 2024,
       "relevance": "Exploratory study of ChatGPT usage patterns in GitHub PRs."
     }
   ]
 }