ai-research-survey

scan.json (29208B)

---

 {
   "paper": {
     "title": "Does AI Code Review Lead to Code Changes? A Case Study of GitHub Actions",
     "authors": [
       "Kexin Sun",
       "Hongyu Kuang",
       "Sebastian Baltes",
       "Xin Zhou",
       "He Zhang",
       "Xiaoxing Ma",
       "Guoping Rong",
       "Dong Shao",
       "Christoph Treude"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.18771",
     "doi": "10.48550/arXiv.2508.18771"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["observational"],
   "key_findings": "Among 178 mature GitHub repositories with AI-based code review actions, only 0.9%–19.2% of valid AI-generated review comments led to code changes, compared to 60% for human-authored comments. Usage is highly concentrated on four popular actions accounting for 98.9% of comments. SHAP analysis reveals that hunk-level granularity, manual triggering, code-rich concise comments, and targeting less experienced contributors are the strongest predictors of comment addressing. The best-performing tool (coderabbitai/ai-pr-reviewer) achieved a 19.2% addressing rate, suggesting design choices directly affect developer response.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Reference [12] provides a GitHub repository URL (brinnarlyne8585/AIReviewActionAnalysis) containing 'dataset, annotations, and scripts for LLM-assisted analysis.' The paper states it was accessed 30-05-2025, indicating the repository was live."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The online appendix (reference [12]) includes the dataset and annotations. Section III states: 'We provide an online appendix, including our dataset, annotations, and scripts.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is described. The paper mentions using PyYAML, FastText, difflib, and the GitHub REST API, but provides no comprehensive dependency or environment setup information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Scripts are available in the online appendix but the paper does not describe step-by-step reproduction instructions. No README with commands or a 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates. Table VII reports accuracy and Cohen's κ without confidence intervals or error bars. Despite running LLM evaluations five times, no uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Fisher's exact test is used in Table XI to compare addressing rates across trigger modes and LLM series, with p-values reported (p≤0.05 and p>0.05)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with sufficient context throughout: absolute addressing rates by tool (0.9%–19.2% for AI vs 60% for human), Cohen's κ for inter-rater agreement (0.674–0.764), and Macro-F1 (0.854) for the Random Forest. Comparisons include base rates enabling magnitude assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The annotation sample of 150 comments (50 per category) is not justified with a power analysis or explicit rationale for why 50 per category is sufficient. The ≥50 PRs maturity threshold references prior work [19] but no justification is given for the annotation sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "LLM evaluations were run five times 'for robust evaluation' but no variance, standard deviation, or range across runs is reported. Table VII shows only point estimates. The Random Forest reports single accuracy/F1 values without cross-validation variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-authored review comments serve as a comparison baseline, collected from the same 51 repositories during the same time period (Section IV-B, Phase II). Cross-tool comparisons across four actions also serve as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Human review comments are drawn from the same repositories and time windows as the AI comments (Section IV-B). The four AI tools studied are current and popular as of January 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. Different LLM models are compared for the classification framework (model selection), but no components of the two-stage framework or the feature engineering pipeline are systematically removed to assess their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: overall accuracy and Cohen's κ for the LLM classification (Table VII), and accuracy and Macro-F1 for the Random Forest (Section IV-C). SHAP importance and directionality are also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "150 comments were manually annotated by two independent raters with a third resolving disagreements (Section IV-B, Phase III). Inter-rater agreement (Cohen's κ) reached 0.674–0.764. An additional 250 samples were examined by the first author for representativeness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the Random Forest classifier in RQ3, an 80/20 train/test split was used: 'we trained a Random Forest classifier (80% training data) that achieved 88.5% overall accuracy on the test set' (Section IV-C)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by review granularity (PR/file/hunk-level), by individual action (ID-1 through ID-4), by trigger type (auto/manual), by LLM series (GPT-3.5/GPT-4), by code-text ratio bins, and by author experience bins (Tables IV, V, VIII, XI, XII)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A discusses common failure modes: vague comments like 'Without more context, it is difficult to provide further suggestions,' overly generic summaries, hallucinated style warnings, and redundant reviews. The causes of invalid comments and the 'one-in-one-out paradigm' are analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are prominently reported: mattzcarey/code-review-gpt had only 0.9% valid comments addressed; 37.1% of repositories declared an action but showed no generated comments; most AI comments are not addressed; and automatically triggered comments perform worse than manually triggered ones."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about growing adoption (Table IV), effectiveness variation (Table VIII), and that concise, code-rich, manually triggered hunk-level comments are more effective (Table X, SHAP analysis) are all supported by the results in Sections IV-A through IV-C."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'Does AI Code Review Lead to Code Changes?' — causal framing. The paper uses associational language ('associated with,' 'more likely') but also causal language throughout ('lead to,' 'influence,' 'impact'). The study design is purely observational with no causal identification strategy. While the authors acknowledge in Section VI that 'these interpretations describe associations...not causal effects,' the overall framing exceeds what the observational design supports."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly bounds generalization: English-language comments only (75% excluded), repositories with ≥50 PRs, primarily small-to-medium projects (≤50 non-bot contributors), only GitHub Actions platform, data from early February 2025. The authors state 'our findings may not generalize to very large-scale projects.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses multiple alternative explanations: github-actions[bot] misattribution, file-level change detection insufficiency, language filtering bias, concentration on four popular actions, incomplete feature engineering, Random Forest vs logistic regression model choice, and association vs causation distinction."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures file-level code changes as a proxy for comment effectiveness. Section V explicitly acknowledges: 'not all valid comments necessarily need to result in immediate code changes to be useful. Some AI-generated suggestions, even if ultimately not adopted, may still prompt reflection, discussion, or future improvements.' Section VI also notes that 'comment addressing based on file-level code changes may be insufficient for certain edge cases.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists model names — gpt-4.1, gpt-4o, o4-mini, o3-mini, claude-3-sonnet, claude-3-haiku, deepseek-r1, deepseek-v3 — but defers exact version details to the appendix: 'The specific API endpoints and model versions used are documented in our online appendix scripts.' Some names (e.g., 'gpt-4o') are marketing names without snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'The details of the LLM-assisted framework with specific prompts are available in the online appendix for other researchers to use' (Section IV-B). The online appendix GitHub repository includes the scripts containing prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0 for LLM evaluations (Section IV-B). Reasoning effort is 'medium' for o3-mini and o4-mini. LDA topic count is 6, selected via hyperparameter tuning following prior work [25]. Random Forest uses 80/20 split. While not exhaustive, key generation parameters are stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study. The LLMs are called directly for classification with single prompts, not through agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in detail across multiple phases: 718 repos → 178 mature (≥50 PRs), 16,762 comments → 4,229 English (FastText), → 4,195 first-in-thread, reconstruction of reviewed changes and subsequent modifications, categorization of file changes (Table V). Each filtering step includes counts and criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a substantial section organized into Construct, Internal, External, and Conclusion validity subsections, spanning approximately one full page."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: github-actions[bot] may include unrelated comments, 150-comment annotation sample from 22,000+, English-only filtering removed 75% of comments and 47 repositories (32 Korean-only), focus on four popular actions limits generalizability, small-to-medium projects only, binary addressed/not-addressed loses nuance, feature engineering may be incomplete."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit boundaries stated: only GitHub Actions (not other CI/CD platforms), only English comments, only repositories with ≥50 PRs, only four actions analyzed in depth for RQ2-3, primarily small-to-medium projects, data collected through early February 2025, findings 'may not fully reflect the current landscape.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The online appendix (reference [12]) includes the dataset and annotations. Section III states the appendix contains 'our dataset, annotations, and scripts.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: GitHub REST API queries for workflow files, PR search queries ('repo:{repo_name} reviewed-by:github-actions[bot] is:pr'), specific API endpoints for inline and general comments, matching workflow files to actions, filtering by maturity criterion (≥50 PRs). Action selection from ~240 candidates is also described."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection: top-ranked code review actions from GitHub Marketplace sorted by popularity, 240 examined → 20 candidates → 16 after exclusion. Repositories identified via GitHub API search for workflow files referencing target actions. For annotations: one co-author and an external graduate student independently labeled, with a third co-author resolving disagreements."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with counts at each stage: 718 matched repos → 178 mature → 22,326 comments (Table IV). For addressing analysis: 16,762 from merged PRs → 4,229 English → 4,195 first-in-thread → 4,486 with valid context → 5,652 with human comments added. Table V shows file change distributions. Each filtering criterion is specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Nanjing University (China), University of Bayreuth (Germany), and Singapore Management University (Singapore). None of the authors are affiliated with the tools being studied."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The authors are academic researchers with no apparent commercial interest in the tools studied, but without explicit disclosure this cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. LLMs are used as classification tools for annotation automation, not evaluated for their intrinsic knowledge or capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper is a mining study that uses LLMs as classification tools, not a benchmark evaluation of model capability."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation of model capability is conducted. The 150-comment annotated dataset is newly created and specific to this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study of GitHub repositories with no human participants. Annotators coding data are researchers, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — the study mines publicly available GitHub data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The study uses multiple LLM APIs (gpt-4.1, o3-mini, etc.) to classify 5,652 comments, plus 5 evaluation runs on 150 comments across 7 models. No API costs, token counts, or wall-clock times are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The hardware used for running the Random Forest, LDA, and LLM API calls is not described."
       }
     }
   },
   "claims": [
     {
       "claim": "37.1% of mature repositories declared an AI code review action but showed no generated comments, indicating a gap between declaration and actual use.",
       "evidence": "Table IV: 178 mature repositories, 66 (37.1%) had no observed review activity despite having workflow files referencing AI review actions (Section IV-A).",
       "supported": "strong"
     },
     {
       "claim": "Usage is highly concentrated on four popular actions, which account for 91.1% of reviewed repositories, 95.2% of PRs, and 98.9% of generated comments.",
       "evidence": "Table IV aggregates show Actions ID-1 through ID-4 dominate across all metrics (Section IV-A).",
       "supported": "strong"
     },
     {
       "claim": "The two-stage LLM-assisted framework achieves 86.1% overall accuracy and 76.7% Cohen's κ on the full 6-class comment addressing classification task.",
       "evidence": "Table VII reports performance of gpt-4.1 (Stage-1) + o3-mini (Stage-2) combination on 150 annotated comments across three sources (Section IV-B).",
       "supported": "moderate"
     },
     {
       "claim": "Human-authored review comments are much more likely to be addressed (60%) than AI-generated ones (0.9%–19.2% depending on the tool).",
       "evidence": "Table VIII shows Valid-Fully addressed rates: human 56.0%, coderabbitai/ai-pr-reviewer 15.0%, anc95/ChatGPT-CodeReview 0.5%, mattzcarey/code-review-gpt 0.1% (Section IV-C).",
       "supported": "strong"
     },
     {
       "claim": "Comments that are concise, contain code snippets, and are manually triggered — particularly from hunk-level review tools — are more likely to result in code changes.",
       "evidence": "SHAP analysis in Table X shows Source Features (Is Human, Trigger auto, Is File Level Action) and Comment Features (Code Text Ratio, Text Length) are top predictors. Table XII shows addressing rate jumps when Code Text Ratio exceeds 0.5 (Section IV-C).",
       "supported": "moderate"
     },
     {
       "claim": "AI-generated reviews may be more helpful for newcomers: the addressing rate for comments on code by the least experienced contributors was five times higher than for the most experienced.",
       "evidence": "Table XII shows 16.1% addressing rate for Author Prior Commits ≤30 vs 3.3% for >1013 commits. SHAP directionality for Author Prior Commits is ρ=-0.67 (Section IV-C).",
       "supported": "moderate"
     },
     {
       "claim": "82.6% of mature repositories customized at least one optional parameter, and configuration evolution is an ongoing process with 22.5% continuing adjustments beyond one month.",
       "evidence": "Section IV-A reports 147/178 repositories customized parameters; temporal analysis shows 40 repositories continued adjustments beyond one month.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small annotation sample relative to dataset",
       "detail": "Only 150 comments were manually annotated from a pool of 22,326 AI-generated and 1,166 human comments (<1%). The 150 comments were sampled only from those with subsequent file modifications, introducing survivorship bias in the ground truth. The first author examined an additional 250 samples informally but this is not a formal validation."
     },
     {
       "flag": "Language filtering removes 75% of data",
       "detail": "Filtering to English-only comments removed approximately 75% of comments (12,533 from 16,762) and 47 of 100 repositories. Thirty-two excluded repositories contained only Korean comments. The remaining sample may not represent the broader user population of these tools."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "LLM evaluations were run five times each 'for robust evaluation,' but no variance, standard deviation, or range across runs is reported. Only point estimates appear in Table VII. This makes it impossible to assess the stability of the automated classification results."
     },
     {
       "flag": "Causal framing exceeds observational evidence",
       "detail": "The title asks 'Does AI Code Review Lead to Code Changes?' using causal language, but the study is purely observational. While the authors acknowledge this in Section VI, the framing throughout (including 'factors that influence,' 'impact the likelihood') implies causal relationships that the study design cannot establish."
     }
   ],
   "cited_papers": [
     {
       "title": "Llama-reviewer: Advancing code review automation with large language models through parameter-efficient fine-tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "relevance": "Directly relevant as an example of LLM-based automated code review, demonstrating parameter-efficient fine-tuning of LLaMA for review comment generation."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Z. Li", "S. Lu", "D. Guo", "N. Duan"],
       "year": 2022,
       "relevance": "Proposes large-scale pre-training for automating code review activities including determining if changes need review and generating comments."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["R. Tufano", "S. Masiero", "A. Mastropaolo", "L. Pascarella", "D. Poshyvanyk", "G. Bavota"],
       "year": 2022,
       "relevance": "Explores pre-trained models (T5) for code review automation including comment generation and revised code generation."
     },
     {
       "title": "Improving automated code reviews: Learning from experience",
       "authors": ["H. Y. Lin", "P. Thongtanunam", "C. Treude", "W. Charoenwet"],
       "year": 2024,
       "relevance": "Proposes improvements to automated code review through experience-based learning, contributing benchmark datasets."
     },
     {
       "title": "Vulrepair: a t5-based automated software vulnerability repair",
       "authors": ["M. Fu", "C. Tantithamthavorn", "T. Le", "V. Nguyen", "D. Phung"],
       "year": 2022,
       "relevance": "T5-based automated vulnerability repair demonstrating LLM application in code quality improvement."
     },
     {
       "title": "Inferfix: End-to-end program repair with llms",
       "authors": ["M. Jin", "S. Shahriar", "M. Tufano", "X. Shi", "S. Lu", "N. Sundaresan", "A. Svyatkovskiy"],
       "year": 2023,
       "relevance": "End-to-end LLM-based program repair system demonstrating practical deployment of AI code fixing."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["P. Vaithilingam", "T. Zhang", "E. L. Glassman"],
       "year": 2022,
       "relevance": "Studies developer expectations and experiences with GitHub Copilot, directly relevant to understanding developer response to AI coding tools."
     },
     {
       "title": "Automated code review in practice",
       "authors": ["U. Cihan", "V. Haratian", "A. İçöz", "M. K. Gül"],
       "year": 2024,
       "arxiv_id": "2412.18531",
       "relevance": "Industrial study of automated code review tools providing complementary evidence on AI code review adoption and effectiveness in practice."
     },
     {
       "title": "Characteristics of useful code reviews: An empirical study at microsoft",
       "authors": ["A. Bosu", "M. Greiler", "C. Bird"],
       "year": 2015,
       "relevance": "Foundational study on what makes code review comments useful, providing baseline understanding of human code review effectiveness."
     },
     {
       "title": "Out of sight, out of mind: Better automatic vulnerability repair by broadening input ranges and sources",
       "authors": ["X. Zhou", "K. Kim", "B. Xu", "D. Han", "D. Lo"],
       "year": 2024,
       "relevance": "Advances LLM-based vulnerability repair methodology relevant to automated code quality improvement."
     },
     {
       "title": "The code review comprehension assessment for large language models",
       "authors": ["H. Y. Lin", "C. Liu", "H. Gao", "P. Thongtanunam", "C. Treude"],
       "year": 2025,
       "relevance": "Assesses LLM comprehension capability for code review tasks, directly relevant to understanding AI code review quality."
     },
     {
       "title": "Predicting usefulness of code review comments using textual features and developer experience",
       "authors": ["M. M. Rahman", "C. K. Roy", "R. G. Kula"],
       "year": 2017,
       "relevance": "Predicts code review comment usefulness using features similar to those in this study, providing methodological foundation for the SHAP analysis approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides concrete, actionable design recommendations for AI code review tool builders and adopters, including specific feature priorities like hunk-level granularity and manual triggering."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that most AI code review tools have near-zero impact (0.9% addressing) is mildly surprising, though the general suspicion about AI tool effectiveness is not new."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk, security, or safety concerns are raised by this study."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Reveals a large gap between AI code review marketing promises and actual developer response (0.9%–19.2% vs 60% human addressing rate), which could generate discussion about AI tool hype."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Scripts and dataset are released in an online appendix but there is no interactive demo or installable tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Studies well-known GitHub ecosystem tools (CodeRabbit, ChatGPT-CodeReview) but authors are from universities without major brand recognition."
     }
   }
 }