ai-research-survey

scan.json (23366B)

---

 {
   "paper": {
     "title": "Code Review Automation using Retrieval Augmented Generation",
     "authors": ["Qianru Meng", "Xiao Zhang", "Zhaochun Ren", "Joost Visser"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2511.05302"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository link is provided: https://anonymous.4open.science/r/GAR-9EE2, mentioned in both Section 1 (contributions) and Section 10 (Data Availability)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: the CodeReviewer dataset and the Tufano dataset, both cited with original references. The anonymous repository also claims to contain the data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 4 NVIDIA A100 GPUs (Section 4.6) and specific model links on HuggingFace, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymous repository is referenced but no README or reproduction guide is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, and 6 are reported as point estimates (e.g., BLEU-4 of 12.32) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RARe 'outperforms state-of-the-art methods' and reports percentage improvements, but no statistical significance tests (p-values, t-tests, etc.) are conducted to validate these differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '153% increase in BLEU-4' from 5.12 to 12.96 (Table 5), and absolute scores for all methods, allowing readers to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 100 samples were chosen for the human evaluation (Section 6.2), nor is there a power analysis or acknowledgment that this may be insufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.6 states 'we averaged the results across three runs for each experiment,' but no standard deviations, error bars, or variance measures are reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 compares RARe against six baselines: Tufano et al., CodeT5, CodeReviewer, CommentFinder, AUGER, and LLaMA-Reviewer."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods such as LLaMA-Reviewer (2023), CommentFinder (2022), AUGER (2022), and CodeReviewer (2022), which represent the state of the art for this task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides a detailed ablation comparing the contributions of retrieval augmentation across three models and two settings (direct inference vs. fine-tuning). Table 6 also ablates retrieval strategy (random, top-1, top-3, top-5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: BLEU-4, ROUGE-L, and METEOR (Section 4.3). Human evaluation adds a fourth dimension."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 reports a human evaluation on 100 samples from each dataset, with two expert annotators categorizing reviews as Perfect Prediction, Semantically Equivalent, Alternative Solution, or Incorrect."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows explicit train/test/validation splits for both datasets. Section 4.2 states 'The data split ratios are consistent with those used in previous works.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (CRer. vs. Tuf.), by retriever type (Table 4), by generator model and setting (Table 5), and by retrieval strategy (Table 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses cases where BLEU scores are misleading, Section 6.2 shows 'Incorrect' review counts, and Section 9 (Conclusion) discusses dependencies on retrieval accuracy as a failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CodeGemma benefits very little from retrieval augmentation (Section 5.2: 'the benefits of retrieval are very limited'), and that increasing retrieved reviews from 1 to 5 decreases performance (Section 5.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims BLEU-4 scores of 12.32 and 12.96 on two benchmarks, which match Table 6 (RARe top1: 12.32 on CRer., 12.96 on Tuf.). Human evaluation and interpretability claims are supported in Section 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about retrieval augmentation improving performance. These are supported by controlled ablation: same models with and without retrieval (Table 5), and random vs. top-k retrieval (Table 6), constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (External Validity) explicitly states that 'the data for each language in the multi-language dataset are relatively limited' and raises 'concerns about the model's ability to generalize effectively to other programming languages or broader contexts.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Internal Validity) discusses alternative explanations: pre-training data biases, quality of the external knowledge base, and the construct validity of BLEU/ROUGE/METEOR metrics for measuring review quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5.2 provides specific HuggingFace model links: meta-llama/Meta-Llama-3.1-8B-Instruct, mistralai/Mistral-7B-Instruct-v0.3, and google/codegemma-7b-it, which identify exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides the actual prompt templates for both direct inference and RARe, including the instruction text and the structure of the retrieved review placeholder."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.6 reports LoRA fine-tuning hyperparameters (5 epochs, learning rate 10^-4, batch size 16), but does not report inference-time parameters such as temperature, top-p, or max tokens for the LLM generation."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full RARe pipeline: the retriever component (NDR, GDR, DPR) and the generator component, with prompt construction detailed in Section 4.5.3 and Table 3. The architecture is shown in Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states 'We do not apply any preprocessing to these datasets, since they have been already cleaned to reduce duplicates. The data split ratios are consistent with those used in previous works.' This is sufficient transparency about preprocessing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 includes a dedicated 'Limitations' subsection, and Section 8 provides a full 'Threats to Validity' section covering internal, construct, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: pre-training data opacity for the LLMs used, reliance on training data as knowledge base rather than a purpose-built corpus, limitations of BLEU/ROUGE/METEOR for measuring review quality, and limited language coverage in the multi-language dataset."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) explicitly states scope boundaries: limited to three LLMs and dense retrieval only, limited case studies, and only two annotators. Section 8 (External Validity) notes limited per-language data in the multi-language dataset."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Data Availability) states 'The data and code that support the findings of this study are available in anonymous repository.' The datasets are publicly available benchmarks (CodeReviewer and Tufano datasets)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the datasets: CRer. collects code from GitHub across nine languages; Tuf. consists of Java code from GitHub and Gerrit. Dataset statistics with train/test/val splits are in Table 1."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation (Section 6.2), two expert annotators are mentioned (6 and 8 years of SE experience), but their recruitment method is not described — how they were selected and whether this could introduce bias is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the pipeline: datasets are taken as-is from prior work (no preprocessing), the retriever selects top-k reviews, these are fed to the generator via a prompt. Each stage is described in Section 3 and 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Leiden University and University of Groningen. No commercial product is being evaluated, so there is no product-affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, and the authors are at academic institutions with no apparent financial stake in the outcome. Likely unfunded academic work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3.1, Mistral, and CodeGemma without stating their training data cutoff dates. These models may have been pre-trained on the benchmark datasets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the CodeReviewer or Tufano datasets (both publicly available since 2022) appeared in the pre-training data of the LLMs used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both benchmark datasets were published in 2022 and are publicly available on GitHub. All three LLMs were trained after 2022 and could have seen this data. The paper does not address this contamination risk. Section 8 (Internal Validity) mentions pre-training data opacity but does not specifically address benchmark contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation in Section 6.2 involves two annotators rating generated reviews. No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 reports that the two annotators have '6 or 8 years of software engineering experience,' providing relevant professional demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for the annotators are stated. The paper does not describe how they were selected or what qualifications were required beyond SE experience."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control groups for human participants. The annotators are evaluating outputs, not being assigned to experimental conditions."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "It is not stated whether the annotators knew which method (DI, RADI, FT, RAFT) generated each review. If they did, this could bias their ratings."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "With only two annotators performing a rating task, attrition is not a meaningful concern. This is not a participant study with dropout risk."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The paper does not mention the practical cost of running RARe despite it requiring both retrieval and LLM generation steps."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.6 mentions '4 NVIDIA A100 GPUs' for training, but does not state total GPU hours, training time, or total computational budget."
       }
     }
   },
   "claims": [
     {
       "claim": "RARe achieves state-of-the-art BLEU-4 scores of 12.32 and 12.96 on the CodeReviewer and Tufano datasets, respectively.",
       "evidence": "Table 6 shows RARe (top1) achieving 12.32 BLEU-4 on CRer. and 12.96 on Tuf., compared to the next best baselines CommentFinder (9.47/12.71) and Tufano et al. (-/12.31).",
       "supported": "strong"
     },
     {
       "claim": "Retrieval augmentation improves Llama 3.1 direct inference by 153% BLEU-4, 240% ROUGE-L, and 228% METEOR on the Tufano dataset.",
       "evidence": "Table 5 shows Llama 3.1 DI scores of 5.12/2.55/2.67 increasing to 12.96/8.67/8.76 with retrieval augmentation on the Tuf. dataset.",
       "supported": "moderate"
     },
     {
       "claim": "RARe significantly increases the proportion of valuable human-evaluated reviews, with 275% increase on CRer. and 266% on Tuf. for direct inference.",
       "evidence": "Table 8 shows valuable reviews (PP+SE+AS) increasing from 12 to 45 on CRer. (275%) and from 15 to 55 on Tuf. (267%) when adding retrieval to direct inference.",
       "supported": "moderate"
     },
     {
       "claim": "The most relevant single retrieved review (top-1) contributes most to RARe's performance; more retrieved reviews decrease performance.",
       "evidence": "Table 6 shows RARe (top1) outperforming RARe (top3) and RARe (top5) on all metrics across both datasets.",
       "supported": "strong"
     },
     {
       "claim": "RARe is the first to apply RAG to the code review generation task.",
       "evidence": "Section 1 contributions and Section 2 literature review claim no prior work combines retrieval-based and generative methods for code review. This is a novelty claim that cannot be fully verified.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RARe, a retrieval-augmented generation system for automated code review, outperforms existing retrieval-only and generation-only baselines on two benchmark datasets (CodeReviewer and Tufano), achieving BLEU-4 scores of 12.32 and 12.96 respectively. Retrieval augmentation yields the largest improvements when applied to direct inference (up to 153% BLEU-4 increase), while fine-tuned models see more modest gains (up to 24%). Human evaluation on 200 samples confirms that retrieval augmentation substantially reduces incorrect reviews and increases the proportion of semantically equivalent or alternative solution reviews. Notably, using only the single most relevant retrieved review (top-1) outperforms using multiple retrieved reviews (top-3, top-5).",
   "red_flags": [
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "Section 4.6 states results are averaged over three runs, but no standard deviation, confidence interval, or variance measure is reported in any table. This makes it impossible to assess result stability or whether differences between methods are meaningful."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('outperforms', 'significantly outperforms') are based on comparing point estimates without any significance tests. Given the lack of variance reporting, the claimed improvements may not be statistically significant."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "Both datasets (CodeReviewer 2022, Tufano 2022) are publicly available and were published before the training of Llama 3.1, Mistral, and CodeGemma. The paper does not discuss whether these models may have seen the benchmark data during pre-training, which could inflate scores."
     },
     {
       "flag": "Very small human evaluation sample",
       "detail": "Only 100 samples per dataset with only 2 annotators. No inter-annotator agreement metric (e.g., Cohen's kappa) is reported. Disagreements are resolved by discussion, which may introduce bias. No power analysis justifies this sample size."
     },
     {
       "flag": "Missing inference hyperparameters",
       "detail": "Temperature, top-p, and max token settings for LLM inference are not reported. These parameters significantly affect generation quality and reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo", "Nan Duan"],
       "year": 2022,
       "relevance": "Key baseline for automated code review using pre-trained models; provides one of the two benchmark datasets used."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["Rosalia Tufano", "Simone Masiero", "Antonio Mastropaolo"],
       "year": 2022,
       "relevance": "Provides the Tufano benchmark dataset and T5-based code review generation baseline."
     },
     {
       "title": "LLaMA-Reviewer: Advancing Code Review Automation with Large Language Models through Parameter-Efficient Fine-Tuning",
       "authors": ["Junyi Lu", "Lei Yu", "Xiaojia Li"],
       "year": 2023,
       "relevance": "Baseline using LLM fine-tuning (LoRA) for code review generation; demonstrates LLM applicability to code review."
     },
     {
       "title": "AUGER: automatically generating review comments with pre-training models",
       "authors": ["Lingwei Li", "Li Yang", "Huaxi Jiang"],
       "year": 2022,
       "relevance": "Pre-training-based code review generation baseline, part of the state-of-the-art comparison."
     },
     {
       "title": "Commentfinder: a simpler, faster, more accurate code review comments recommendation",
       "authors": ["Yang Hong", "Chakkrit Tantithamthavorn", "Patanamon Thongtanunam", "Aldeida Aleti"],
       "year": 2022,
       "relevance": "Retrieval-based code review baseline; strongest competitor to RARe on the Tufano dataset."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Survey of RAG techniques for LLMs; provides theoretical foundation for the RAG approach used in RARe."
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "authors": ["Md Rizwan Parvez", "Wasi Ahmad", "Saikat Chakraborty"],
       "year": 2021,
       "relevance": "Applies RAG to code generation and summarization tasks; provides DPR hyperparameters used in this work."
     },
     {
       "title": "Retrieval-based prompt selection for code-related few-shot learning",
       "authors": ["Noor Nashid", "Mifta Sintaha", "Ali Mesbah"],
       "year": 2023,
       "relevance": "CEDAR framework using retrieval-augmented strategies for code-related tasks including program repair."
     },
     {
       "title": "No more fine-tuning? an experimental evaluation of prompt tuning in code intelligence",
       "authors": ["Chaozheng Wang", "Yuanhang Yang", "Cuiyun Gao"],
       "year": 2022,
       "relevance": "Evaluates prompt tuning vs. fine-tuning for code intelligence tasks; informs the generation approach in RARe."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Pre-trained code model used as the encoder in the NDR and DPR retrievers in this work."
     }
   ]
 }