ai-research-survey

scan.json (27380B)

---

 {
   "paper": {
     "title": "AI-Assisted Assessment of Coding Practices in Modern Code Review",
     "authors": [
       "Manushree Vijayvergiya",
       "Małgorzata Salawa",
       "Ivan Budiselić",
       "Dan Zheng",
       "Pascal Lamblin",
       "Marko Ivanković",
       "Juanjo Carin",
       "Mateusz Lewko",
       "Jovan Andonov",
       "Goran Petrović",
       "Daniel Tarlow",
       "Petros Maniatis",
       "René Just"
     ],
     "year": 2024,
     "venue": "AIware '24 (1st ACM International Conference on AI-Powered Software)",
     "arxiv_id": "2405.13565",
     "doi": "10.1145/3664646.3665664"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided. The system is internal to Google and no code release is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper states 'due to industrial confidentiality reasons we are unable to disclose absolute numbers' (Section 4). Training data comes from Google's internal code review system and is not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions T5X framework, TPU pods, and TensorFlow but does not provide a requirements file, dependency versions, or detailed environment specifications. No reproducible environment setup is given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The system is proprietary to Google and cannot be replicated externally."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any of the main results (useful ratio, comment resolution rate, URL coverage). All results are point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We did not detect any statistically significant change' in the A/B experiment (Section 4.4) but does not report the specific test used, test statistics, p-values, or sample sizes for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes are reported. The paper provides percentages (e.g., 40% resolution rate, 68% URL coverage) but these are descriptive statistics, not effect sizes with baseline context for comparative claims."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No sample size justification or power analysis is provided. The rater study uses 15 raters and ~370 comments; the comment resolution analysis uses 6000 snapshot pairs; the manual inspection uses 40 pairs. None of these sample sizes are justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. All metrics are single point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AutoCommenter against human reviewer comments (Section 5.2) and traditional linters (Section 5.3). The A/B experiment (Section 4.4) includes a control group."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are appropriate: human reviewers represent the current standard, and traditional linters are the existing automated approach. The A/B experiment's control group (no AutoCommenter) is the most relevant baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The system has multiple components (model, thresholds, URL filtering, summarization, beam search) but no systematic analysis of individual component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: useful ratio (positive/negative feedback), comment resolution rate (~40%), URL coverage vs. human comments (68%), URL coverage vs. linters (66% beyond linters), precision/recall on validation/test datasets."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "An independent human rating study was conducted (Section 4.3) with 15 raters evaluating ~370 posted comments. Developer feedback via thumbs up/down buttons also constitutes ongoing human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 describes temporal splitting: 'We temporally split the dataset to ensure that the model has not been trained on future code-review snapshots of the code comments in the validation and test datasets.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides a breakdown of the top-50 most frequently predicted URLs by best-practice type (code idioms, documentation, formatting, language, naming) and whether a linter exists. Figure 5 shows per-URL cumulative distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides detailed failure analysis from the rater study, including specific categories: 'Several topics or complex topic,' 'Importance of high-quality summaries,' 'Subjective and potentially contentious topic,' 'Systematic model error for some guidelines,' and 'Correct but low-value comments.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The A/B experiment reports null results: 'We did not detect any statistically significant change in any of the following: total duration of code reviews, time developers actively spent on the code review, the number of comment-response iterations' (Section 4.4). Also reports per-language threshold failure (Section 4.1.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the system is 'feasible and has a positive impact on the developer workflow.' The paper supports feasibility through deployment details and positive impact through the useful ratio (>80%) and comment resolution (~40%). The claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim (no adverse effects on code review process) is supported by a randomized A/B experiment (Section 4.4) with random assignment verified for balance. The 'slight improvement in coding speed' is reported as a conjecture, not a definitive causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to Google's setting: 'in an industrial setting at Google' and 'in a large industrial setting at Google' (Sections 1, 2). It does not claim the results generalize to other companies or open-source settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the 40% comment resolution rate could be inflated by unrelated code changes. While they note that 'unrelated code changes could have led to a specific comment no longer being predicted' (Section 5.1), they do not discuss this as a systematic alternative explanation for the overall claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'a traditional transformer approach based on T5, using T5X' (Section 3.1) but does not specify the model size (number of parameters) or exact version/checkpoint used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides the actual input/target format including the task prompt ('Check language best practices') and a complete worked example for Go. The prompt is a fixed-text code comment, and the full format is shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Very few hyperparameters are reported. Beam search n=4 is mentioned (Section 4.1.2), and initial confidence threshold t=0.98 is stated (Section 4.1.1). However, training hyperparameters (learning rate, batch size, number of training steps, etc.) are not reported. Context window is mentioned as 2048 tokens but labeled as a limitation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "AutoCommenter is not an agentic system. It is a single-pass inference model that takes source code as input and outputs predicted violations. No agentic scaffolding (tool use, retry logic, multi-turn interaction) is involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.2.1 and 3.2.2 describe the data pipeline: large-scale preprocessing identifies relevant code comments containing best-practice URLs, then dataset curation converts them into TensorFlow Examples. The temporal split strategy is described (Section 3.3.1). Training corpus size (~3 billion examples total, ~800k best practice examples) is stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Some limitations are mentioned in passing (e.g., incomplete ground truth in Section 3.3.1, confidentiality constraints in Section 4) but there is no structured discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper mentions that precision/recall measures are noisy due to incomplete ground truth (Section 3.3.1) but does not discuss this as a formal threat to validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not discuss limitations regarding generalizability beyond Google, the specific languages tested, or the types of best practices the system cannot handle."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The paper explicitly states confidentiality prevents disclosure of absolute numbers (Section 4)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes how training data is collected: extracting human-authored code review comments containing best-practice URLs from Google's code review system, along with corresponding source code and metadata including creation timestamps."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For the rater study (Section 4.3), recruitment is described: '15 raters—developers from partner teams' were recruited. For the A/B experiment (Section 4.4), random assignment via SHA256 hash of developer email addresses is described, with verification that groups were balanced."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented across Sections 3.2.1-3.2.3 (preprocessing, curation, training) and Section 5.1 (comment resolution analysis with 6000 snapshot pairs, 50% absent from submitted snapshot, 80% of manual sample confirmed resolution)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement is provided. The acknowledgements section thanks individuals but does not disclose funding sources. Given that this is a Google product evaluated by Google employees, funding disclosure is important."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed. 12 of 13 authors are Google employees, and one (René Just) is at University of Washington with a note 'Work done at Google.' The affiliation with the evaluated product's company is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google funded this work (all but one author are Google employees) and Google is the entity whose product (AutoCommenter) is being evaluated. Google has a direct financial interest in demonstrating that its code review automation is effective."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The authors are employees of the company whose product is being evaluated, but this conflict is not explicitly acknowledged as such."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state a specific training data cutoff date. It mentions training data 'stretches before 2022' (Section 4.2) but does not give a precise cutoff. The model was described as 'state of the art' in 2022."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 describes temporal splitting to avoid data leakage: 'We temporally split the dataset to ensure that the model has not been trained on future code-review snapshots of the code comments in the validation and test datasets.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The temporal split strategy (Section 3.3.1) directly addresses contamination. The evaluation on full historical code reviews (Section 3.3.2) and A/B experiment (Section 4.4) use live/contemporaneous data that could not have been in training, further mitigating contamination concerns."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the rater study or the A/B experiment."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned, despite the rater study involving human participants (15 raters) and the A/B experiment involving tens of thousands of developers."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "For the rater study, participants are described only as 'developers from partner teams' (Section 4.3). For the A/B experiment, the paper confirms groups were balanced on 'tenure, seniority, programming languages and business units' (Section 4.4) but does not report the actual demographic distribution."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria are stated for the rater study participants. For the A/B experiment, all developers were included but selection criteria for the control vs. experiment group are described only as hash-based randomization."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "The A/B experiment randomization is described: 'We randomly assigned developers to an experiment group (AutoCommenter enabled) and a control group (AutoCommenter disabled). We randomized based on the last few digits of the SHA256 hash of the developers email address' (Section 4.4)."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "For the rater study: 'We did not show the original user feedback to the rater, to avoid biasing their evaluation' (Section 4.3). The raters were blinded to the developer feedback on the comments they were rating."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported for either the rater study or the A/B experiment. It is not stated whether all 15 raters completed all assigned evaluations or whether any developers dropped out of the A/B experiment."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Latency is partially reported (median 2 seconds for beam search, sub-second for greedy in IDE, Section 4.1.2), but no compute cost, API cost, or cost per prediction is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The paper mentions using TPU pods for training (Section 3.2.3) and storing checkpoints every 1000 steps, but does not quantify training time, number of TPUs, or total compute cost."
       }
     }
   },
   "claims": [
     {
       "claim": "An end-to-end system for learning and enforcing coding best practices is feasible and has a positive impact on the developer workflow.",
       "evidence": "The system was deployed to all developers at Google. Developer feedback useful ratio exceeded 80% after refinements (Section 4.3). Comment resolution rate is approximately 40% (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "AutoCommenter's comments cover 68% of best practice documents that human reviewers reference.",
       "evidence": "Section 5.2 and Figure 5 show the cumulative distribution of comments per URL for automated vs. human comments. AutoCommenter has created comments for 330 distinct URLs covering 68% of historical human comments.",
       "supported": "moderate"
     },
     {
       "claim": "For 66% of the top-50 most frequently predicted best practices, violation detection is beyond the scope of traditional static analysis.",
       "evidence": "Section 5.3 and Figure 6: three expert authors independently categorized the 50 most frequently predicted URLs. For 33/50, a linter does not exist and cannot easily be built.",
       "supported": "strong"
     },
     {
       "claim": "The comment resolution rate is approximately 40%.",
       "evidence": "Section 5.1: Automated analysis of 6000 snapshot pairs found comments absent in 50% of submitted snapshots. Manual inspection of 40 such pairs found 80% were directly resolved. 50% * 80% = 40%.",
       "supported": "moderate"
     },
     {
       "claim": "AutoCommenter did not adversely affect the code review process (total duration, active time, comment-response iterations).",
       "evidence": "Section 4.4: A/B experiment with random assignment of about half of all developers. 'We did not detect any statistically significant change in any of the following: total duration of code reviews, time developers actively spent on the code review, the number of comment-response iterations.'",
       "supported": "moderate"
     },
     {
       "claim": "A slight improvement in coding speed was detected when using AutoCommenter.",
       "evidence": "Section 4.4: 'We did, however, detect a slight improvement in coding speed.' No magnitude, test statistic, or p-value is reported. The authors 'conjecture that the reduction in context switches to documentation leads to this positive effect.'",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "rct",
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "AutoCommenter, a T5-based LLM system for detecting coding best-practice violations in code review, was deployed to tens of thousands of developers at Google. After iterative refinements including per-URL thresholds, beam search decoding, and suppression of non-actionable URLs, the system achieved over 80% useful ratio on developer feedback. An A/B experiment found no adverse effects on code review duration or iteration count, and an estimated 40% of automated comments were resolved by code authors. The system covers 68% of best practices referenced by human reviewers, with 66% of its top-50 predicted violations being beyond the scope of traditional linters.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "12 of 13 authors are Google employees evaluating Google's AutoCommenter system. One external author (René Just, UW) also worked at Google during this research. No conflict of interest statement is provided despite this clear conflict."
     },
     {
       "flag": "Confidentiality prevents verification",
       "detail": "The paper states 'due to industrial confidentiality reasons we are unable to disclose absolute numbers of code reviews, developers, files, comments, or distribution of duration of code reviews' (Section 4). This makes all quantitative claims unverifiable by external parties."
     },
     {
       "flag": "No statistical details for A/B experiment",
       "detail": "The A/B experiment (Section 4.4) claims no statistically significant adverse effects and a 'slight improvement in coding speed' but reports no test statistics, p-values, sample sizes, effect sizes, or confidence intervals. The reader cannot assess the power of the null results or the strength of the positive finding."
     },
     {
       "flag": "Small manual verification sample",
       "detail": "The 40% comment resolution rate is derived from a chain of estimates: 50% absence rate from 6000 automated pairs, then 80% true resolution from manual inspection of only 40 pairs. The 40-pair manual sample is small and the multiplication of estimates compounds uncertainty."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, which is unusual for a paper making deployment claims at this scale."
     }
   ],
   "cited_papers": [
     {
       "title": "Expectations, outcomes, and challenges of modern code review",
       "authors": ["Alberto Bacchelli", "Christian Bird"],
       "year": 2013,
       "doi": "10.1109/ICSE.2013.6606617",
       "relevance": "Foundational study on modern code review practices and challenges, relevant to understanding the context for AI-assisted code review."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo", "Nan Duan", "Shailesh Jannu", "Grant Jenks", "Deep Majumder", "Jared Green", "Alexey Svyatkovskiy", "Shengyu Fu", "Neel Sundaresan"],
       "year": 2022,
       "doi": "10.1145/3540250.3549081",
       "relevance": "Pre-trained model approach to code review automation using LLMs, directly comparable methodology to AutoCommenter."
     },
     {
       "title": "Code Review Automation: Strengths and Weaknesses of the State of the Art",
       "authors": ["Rosalia Tufano", "Ozren Dabić", "Antonio Mastropaolo", "Matteo Ciniselli", "Gabriele Bavota"],
       "year": 2024,
       "relevance": "Survey of code review automation approaches evaluating strengths and weaknesses, relevant to understanding the state of AI-assisted code review."
     },
     {
       "title": "Resolving Code Review Comments with Machine Learning",
       "authors": ["Alexander Frömmgen", "Jacob Austin", "Peter Choy"],
       "year": 2024,
       "relevance": "Companion work evaluating a live ML system for the inverse task (creating code from review comments), providing relevant methodology for evaluating deployed ML-for-code systems."
     },
     {
       "title": "Modern Code Review: A Case Study at Google",
       "authors": ["Caitlin Sadowski", "Emma Söderberg", "Luke Church", "Michal Sipko", "Alberto Bacchelli"],
       "year": 2018,
       "relevance": "Prior case study of Google's code review process, establishing the baseline environment into which AutoCommenter was deployed."
     },
     {
       "title": "CommentFinder: a simpler, faster, more accurate code review comments recommendation",
       "authors": ["Yang Hong", "Chakkrit Tantithamthavorn", "Patanamon Thongtanunam", "Aldeida Aleti"],
       "year": 2022,
       "relevance": "Alternative approach to automated code review comment recommendation, relevant as a comparison point for AI-based code review tools."
     },
     {
       "title": "Auger: Automatically generating review comments with pre-training models",
       "authors": ["Lingwei Li", "Li Yang", "Huaxi Jiang"],
       "year": 2022,
       "relevance": "Pre-training approach for automated code review comment generation, directly related to AI-assisted code review methodology."
     },
     {
       "title": "AutoTransform: Automated code transformation to support modern code review process",
       "authors": ["Patanamon Thongtanunam", "Chanathip Pornprasit", "Chakkrit Tantithamthavorn"],
       "year": 2022,
       "relevance": "Automated code transformation for code review, relevant methodology for evaluating AI tools in the code review workflow."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["Rosalia Tufano", "Simone Masiero", "Antonio Mastropaolo", "Luca Pascarella", "Denys Poshyvanyk", "Gabriele Bavota"],
       "year": 2022,
       "relevance": "Evaluation of pre-trained models for code review tasks, relevant methodology and baselines for AI-assisted code review research."
     },
     {
       "title": "Scaling up models and data with t5x and seqio",
       "authors": ["Adam Roberts", "Hyung Won Chung"],
       "year": 2023,
       "relevance": "The T5X framework used as the foundation for AutoCommenter's model, relevant to understanding LLM infrastructure for code tasks."
     },
     {
       "title": "Please fix this mutant: How do developers resolve mutants surfaced during code review?",
       "authors": ["Goran Petrović", "Marko Ivanković", "Gordon Fraser", "René Just"],
       "year": 2023,
       "relevance": "Prior work on developer interaction with automated code review analysis at Google, providing methodology for measuring comment resolution."
     }
   ]
 }