ai-research-survey

scan.json (26326B)

---

 {
   "paper": {
     "title": "Automated Code Review In Practice",
     "authors": [
       "Umut Cihan",
       "Vahid Haratian",
       "Arda İçöz",
       "Mert Kaan Gül",
       "Ömercan Devran",
       "Emircan Furkan Bayendur",
       "Baykal Mehmet Uçar",
       "Eray Tüzün"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2412.18531",
     "doi": "10.5281/zenodo.13917481"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a replication package via Zenodo (https://doi.org/10.5281/zenodo.13917481), referenced in Section III-E as containing 'data analysis scripts, survey questions, and results.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at Zenodo (doi.org/10.5281/zenodo.13917481) includes survey results and data analysis scripts. The paper states in Section III-E: 'Our data analysis scripts, survey questions, and results are shared in our replication package.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, library versions, Python version) are mentioned in the paper beyond a brief reference to using the pandas library."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The replication package is mentioned but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates (e.g., '73.8% resolved', average closure durations) without confidence intervals or error bars. Standard deviation is mentioned once (rating std dev of 1.79) but no CIs on the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses independent samples t-tests for closure duration comparisons (p-value < 0.001) and Poisson regression for human comment volume analysis (Section IV-C), reporting p-values for each project."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: closure durations changed from 5h52m to 8h20m overall, with per-project breakdowns (e.g., Project #1 from 2h48m to 4h38m). Human comments went from 0.31 to 0.28 per PR. These raw differences with baselines provide sufficient context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The general opinion survey has only 22 respondents and the code review survey only 10 respondents, with no discussion of whether these are adequate sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviation is reported once for review ratings (1.79) but variance/spread is not reported for the main quantitative results (closure durations, comment counts). Figures show averages without spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a before/after design comparing metrics (closure duration, human comment volume) before and after CodeReviewBot deployment, serving as a natural baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the same projects' own pre-intervention data, which is the appropriate contemporary baseline for this type of industry case study."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an industry case study evaluating a single tool (CodeReviewBot). There is no multi-component system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study uses multiple metrics: comment resolution labels (73.8% resolved), PR closure duration, number of human reviewer comments, developer survey ratings, and commits after review."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes human evaluation through two surveys: a per-PR code review survey (10 respondents rating automated comments) and a general opinion survey of 22 practitioners evaluating the tool's impact on code quality, speed, and knowledge sharing."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an industry case study, not a benchmark evaluation. There is no train/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per project (Projects #1, #2, #3) for all major metrics: comment labels (Figure 9), closure durations (Figure 11), and human comments (Figure 12), revealing significant variation across projects."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses failure cases including unnecessary review comments (26.2% not acted upon), out-of-scope suggestions, irrelevant comments, and concerns about over-reliance. Developer quotes illustrate specific failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: PR closure duration increased overall (5h52m to 8h20m), human review volume did not significantly decrease, and 26.2% of automated comments were labeled 'Won't Fix' or 'Closed'. Developers reported disadvantages including out-of-scope suggestions."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 73.8% resolved comments (Section IV-C), increased closure duration from 5h52m to 8h20m (Section IV-C), and minor perceived code quality improvement (Section IV-B). The claims are hedged appropriately."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims (e.g., 'automated code reviews led to longer pull request closure times') but the before/after design without controlling for confounds (seasonality, team changes, project maturity) is weak for causal inference. The paper acknowledges seasonality as a threat but does not use causal identification strategies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper appropriately bounds its findings to the specific company and tool. Section VI-C explicitly states: 'Since we used this specific tool and model, we acknowledge that other LLMs and automated code review tools might exhibit different behavior.' Section VI-D notes: 'This study might have led to different results in a different company setting.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several alternative explanations in the threats to validity: seasonality effects (summer vacations), the mandatory comment resolution policy potentially causing unreliable labels, and bot accounts contaminating data. The discussion section considers that closure time increases could be due to developers engaging more with feedback rather than tool inefficiency."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-4-32K Model' (Section III-A) but does not provide a specific version identifier or snapshot date (e.g., 'gpt-4-32k-0613'). The model behavior could differ across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The tool is based on open-source Qodo PR Agent, but the paper does not provide the actual prompts or system instructions used by CodeReviewBot. No prompt text is shown in the paper or described as being in the replication package."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) for the GPT-4-32K API calls are reported. The paper only mentions '3,937 tokens per pull request' as average usage, not the API configuration."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions CodeReviewBot is 'based on the open-source Qodo PR-Agent' but does not describe the scaffolding, workflow, or how the tool processes PRs beyond a brief flow diagram (Figure 3). The customizations Beko made are not detailed."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C-4 describes data preprocessing in detail: filtering late-arriving comments using elbow evaluation (93% threshold), removing bot accounts (SonarQube), excluding deleted comments and non-main-branch PRs, and collaborative data cleaning sessions. The pipeline from raw Azure DevOps data to final dataset is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a dedicated, substantive discussion of limitations organized into construct, internal, external, and conclusion validity subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: seasonality of data collection (summer vacations at Beko), one author being a manager introducing potential bias, mandatory comment resolution policy potentially causing unreliable labels, specific tool/model limitations, and two bot accounts identified in the data."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C states the results apply only to this specific tool (Qodo PR Agent) and model (GPT-4-32K). Section VI-D explicitly states 'This study might have led to different results in a different company setting' and 'we limited our scope and did not aim for statistical generalizations.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (Zenodo, doi.org/10.5281/zenodo.13917481) is referenced as containing data analysis scripts and results. However, the raw Azure DevOps data (individual PR data, comments) may be partially restricted due to industry confidentiality."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C describes data collection from Azure DevOps API in detail, including what data was extracted (PRs, comments, commits, labels), the timeline (Figure 1), and the three data sources (repository data, PR surveys, general opinion surveys)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes that 238 practitioners across 10 projects had access to the tool, 22 developers who contributed to the 3 study projects were surveyed (Table III provides demographics), and PR survey recipients were authors of PRs that received automated reviews."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C-4 documents the pipeline: Azure DevOps API extraction → relational database → CSV conversion → pandas processing → filtering (elbow evaluation for late comments, bot account removal, deleted comment exclusion). The pipeline stages and cleaning decisions are described with rationale."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII 'Acknowledgements' states: 'This work has been supported by the ITEA4 GENIUS project, which has been funded by the national funding authorities of the participating countries.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly show four authors from Bilkent University and four from Beko (the company being studied). The paper header lists these affiliations with institutional markers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is ITEA4 GENIUS project (a European research consortium), which has no direct financial interest in whether automated code review tools succeed or fail. The funder appears independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Four authors are Beko employees, and the study evaluates a tool adopted by Beko, but no explicit declaration of potential conflicts or financial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on a benchmark. It evaluates an automated code review tool in practice (industry deployment), so training cutoff is not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is an industry case study examining tool impact, not a benchmark evaluation. Train/test overlap is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The study measures real-world usage metrics and developer perceptions, so contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or similar) is found in the paper. The study involves human participants through surveys."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval is found in the paper, despite involving human participants through surveys and collecting data about developer behavior."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table III reports participant demographics for the general opinion survey: experience levels (0-2, 2-5, 5-10, 10+ years) and positions (individual contributor vs. lead/manager), with counts for each category."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "The paper states surveys were sent to 'developers who contributed to the three projects within our research scope' (Section III-D-2). For quantitative data, three projects were selected based on 'their longer duration of tool usage' while others were excluded for insufficient data."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational case study, not an experimental study with random assignment to conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational case study. Blinding is not feasible — developers knew they were using CodeReviewBot, and this is inherent to the study design."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes 38 pull requests received automated reviews during the survey period, but only 10 people completed the full survey. For the general opinion survey, 23 responses were received, with 22 consenting to published results and 2 non-useful responses excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V states: 'the CodeReviewBot used an average of 3,937 tokens per pull request at a cost of 0.48$', providing per-PR cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget (total API spend across all PRs, Azure DevOps hosting, etc.) is not stated. Only the per-PR token cost is provided."
       }
     }
   },
   "claims": [
     {
       "claim": "73.8% of automated code review comments were labeled as 'Resolved' by developers, indicating they implemented the tool's suggestions.",
       "evidence": "Section IV-C-1 reports analysis of 1,408 CodeReviewBot comments on merged PRs: 73.8% Resolved, 21.3% Won't Fix. Figure 9 shows per-project breakdown.",
       "supported": "moderate"
     },
     {
       "claim": "The overall average pull request closure duration increased significantly from 5 hours 52 minutes to 8 hours 20 minutes after CodeReviewBot introduction.",
       "evidence": "Section IV-C-3 reports the increase with independent samples t-test (p < 0.001). Per-project results vary: Project #1 increased (2h48m to 4h38m), Project #2 decreased (6h06m to 3h07m), Project #3 increased (20h22m to 30h51m).",
       "supported": "moderate"
     },
     {
       "claim": "The number of human review comments per pull request did not significantly change after introducing automated code review.",
       "evidence": "Section IV-C-4: overall human comments decreased from 0.31 to 0.28 per PR, but Poisson regression showed this was not statistically significant (p >= 0.05). Per-project trends differed (Project #1 increased, Project #2 decreased, Project #3 unchanged).",
       "supported": "strong"
     },
     {
       "claim": "Most practitioners perceived a minor improvement in code quality as a result of automated code reviews.",
       "evidence": "Section IV-B: 14 of 22 survey respondents indicated minor improvement in code quality. 68.8% perceived minor improvement per Section V discussion.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based automated code reviews are highly useful in this company's context.",
       "evidence": "RQ1 answer box cites 73.8% resolved comments, 88 commits after bot reviews, and survey results showing perceived code quality improvement.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "case-study",
     "observational",
     "qualitative"
   ],
   "key_findings": "An industry case study at Beko found that 73.8% of automated code review comments (from a Qodo PR Agent-based tool using GPT-4-32K) were labeled as resolved by developers. However, overall PR closure duration significantly increased from 5h52m to 8h20m, with substantial variation across projects. Human review volume did not significantly change. Most of the 22 surveyed practitioners perceived minor code quality improvements but raised concerns about irrelevant suggestions and potential over-reliance on automation.",
   "red_flags": [
     {
       "flag": "Insider evaluation",
       "detail": "Four of eight authors are Beko employees, including one in a managerial position. The study evaluates a tool adopted by Beko. While the paper notes results were written by non-practitioner authors, the potential for bias is significant and no formal competing interests statement is provided."
     },
     {
       "flag": "Very small survey sample",
       "detail": "The code review survey received only 10 responses and the general opinion survey only 22 responses. The paper acknowledges dissatisfied developers may have been less motivated to complete surveys, creating potential response bias, but does not quantify this threat."
     },
     {
       "flag": "No causal identification strategy",
       "detail": "The before/after comparison lacks controls for temporal confounds. The paper acknowledges seasonality (summer data collection) but does not use difference-in-differences, matching, or other quasi-experimental methods to isolate the tool's causal effect from other changes over time."
     },
     {
       "flag": "Contradictory project-level results",
       "detail": "Results vary substantially across the three projects (e.g., closure duration increased in two projects but decreased in one; human comments increased in one, decreased in another, unchanged in a third). The paper aggregates to overall statistics that may obscure the heterogeneity, and the 'highly useful' conclusion for RQ1 is based primarily on comment resolution labels, which may not reflect genuine usefulness."
     },
     {
       "flag": "Comment resolution label reliability",
       "detail": "The 73.8% 'Resolved' rate is a key finding, but the paper acknowledges developers may not have followed the labeling policy correctly ('developers might also disregard the expected labeling policy and use Resolved instead of Closed or Won't Fix'). This undermines the primary usefulness metric."
     }
   ],
   "cited_papers": [
     {
       "title": "AI-Assisted Assessment of Coding Practices in Modern Code Review",
       "authors": ["M. Vijayvergiya", "M. Salawa", "I. Budiselic"],
       "year": 2024,
       "relevance": "Industry study from Google on AutoCommenter, an LLM-backed automated code review system at scale — directly relevant to evaluating LLM code review tools."
     },
     {
       "title": "Exploring the Capabilities of LLMs for Code Change Related Tasks",
       "authors": ["L. Fan", "J. Liu", "Z. Liu", "D. Lo", "X. Xia", "S. Li"],
       "year": 2024,
       "arxiv_id": "2407.02824",
       "relevance": "Evaluates LLM capabilities on code review generation, commit message generation, and just-in-time comment update — core LLM code generation evaluation."
     },
     {
       "title": "Security Code Review by Large Language Models",
       "authors": ["J. Yu", "P. Liang", "Y. Fu", "A. Tahir", "M. Shahin", "C. Wang", "Y. Cai"],
       "year": 2024,
       "arxiv_id": "2401.16310",
       "relevance": "Explores LLM capabilities for security-focused code review, relevant to AI safety and code quality assessment."
     },
     {
       "title": "Code Review Automation: Strengths and Weaknesses of the State of the Art",
       "authors": ["R. Tufano", "O. Dabic", "A. Mastropaolo", "M. Ciniselli", "G. Bavota"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3348172",
       "relevance": "Systematic evaluation of code review automation approaches including ChatGPT, directly relevant to understanding automated code review effectiveness."
     },
     {
       "title": "Fine-Tuning and Prompt Engineering for Large Language Models-Based Code Review Automation",
       "authors": ["C. Pornprasit", "C. Tantithamthavorn"],
       "year": 2024,
       "relevance": "Investigates prompt engineering and fine-tuning for LLM-based code review, relevant to LLM methodology in SE tasks."
     },
     {
       "title": "AI-Powered Code Review with LLMs: Early Results",
       "authors": ["Z. Rasheed", "M. A. Sami", "M. Waseem", "K.-K. Kemell", "X. Wang", "A. Nguyen", "K. Systa", "P. Abrahamsson"],
       "year": 2024,
       "relevance": "Early results on AI-powered code review with LLMs, directly within the scope of automated code review evaluation."
     },
     {
       "title": "On the Use of ChatGPT for Code Review: Do Developers Like Reviews by ChatGPT?",
       "authors": ["M. Watanabe", "Y. Kashiwa", "B. Lin", "T. Hirao", "K. Yamaguchi", "H. Iida"],
       "year": 2024,
       "doi": "10.1145/3661167.3661183",
       "relevance": "Investigates developer reception of ChatGPT-generated code reviews in GitHub projects, directly comparable to this study's developer perception findings."
     },
     {
       "title": "Tales from the Trenches: Expectations and Challenges from Practice for Code Review in the Generative AI Era",
       "authors": ["N. Davila", "J. Melegati", "I. Wiese"],
       "year": 2024,
       "relevance": "Grey literature review on generative AI for code reviews in practice, complementary survey of the same topic area."
     },
     {
       "title": "CodeAgent: Collaborative Agents for Software Engineering",
       "authors": ["D. Tang", "K. Kim", "Y. Song", "C. Lothritz", "B. Li", "S. Ezzini", "H. Tian", "J. Klein", "T. F. Bissyande"],
       "year": 2024,
       "relevance": "LLM agent approach to software engineering tasks including code review, relevant to agentic AI workflows."
     },
     {
       "title": "Towards Automating Code Review Activities",
       "authors": ["R. Tufano", "L. Pascarella", "M. Tufano", "D. Poshyvanyk", "G. Bavota"],
       "year": 2021,
       "doi": "10.1109/ICSE43902.2021.00027",
       "relevance": "Foundational work on automating code review activities, cited as motivation for the time-saving potential of automated code review."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, the underlying model used in the CodeReviewBot tool evaluated in this study."
     }
   ]
 }