ai-research-survey

scan.json (25104B)

---

 {
   "paper": {
     "title": "Combining Large Language Models with Static Analyzers for Code Review Generation",
     "authors": [
       "Imen Jaoua",
       "Oussama Ben Sghaier",
       "Houari Sahraoui"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2502.06633"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at https://github.com/ImenJaoua/Hybrid-Code-Review, mentioned in the footnote on page 1 and in references [48]."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is available at https://zenodo.org/records/14061110, mentioned in the footnote on page 1 and in references [49]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions four NVIDIA RTX 3090 GPUs and 4-bit quantization with QLoRA, but does not provide a requirements.txt, Dockerfile, or detailed library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the replication package is referenced, the paper itself does not contain step-by-step reproduction instructions. The reader must rely on the external repository without guidance in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as percentages in bar charts (Figures 8, 9, 10) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'RAG significantly outperforms Mi') but does not use any statistical significance tests. Cohen's kappa is used for inter-rater agreement (RQ2), but no significance tests compare the approaches."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as bar charts showing percentage breakdowns of accuracy categories, but no formal effect sizes (Cohen's d, odds ratios, etc.) are reported. The differences are shown visually but not quantified with magnitude measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set of 1,245 samples and the 10% manual evaluation subset (~125 samples) are used without justification of why these sizes are sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experiments. The LLM-based evaluation results are single-run numbers with no indication of variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: the standalone KBS (static analyzers PMD and Checkstyle) and the standalone LBS (fine-tuned CodeLlama-7b model Mi). All three hybrid approaches are compared against both baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The LBS baseline uses CodeLlama-7b fine-tuned on the dataset from Li et al. [25] (2022). No comparison is made against more recent code review approaches or larger LLMs. PMD and Checkstyle are well-established but not the most recent static analysis tools."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The three hybrid strategies (DAT, RAG, NCO) effectively serve as ablations showing different integration points for combining KBS and LBS. Each strategy modifies a different component of the pipeline (training data, inference prompt, post-processing)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct evaluation dimensions are used: accuracy (RQ1, RQ3) classifying reviews as accurate/partially accurate/not accurate, and coverage (RQ4) ranking approaches by issue detection breadth."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 involves manual evaluation by two human evaluators on a 10% sample of the test set, classifying reviews as accurate, partially accurate, or not accurate. Reviews were anonymized and randomized."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The augmented dataset Da was split into 80% training, 10% validation, and 10% test. The evaluation was conducted on a test set of 1,245 samples unseen by the fine-tuned model MFT (Section IV.B)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as aggregate distributions across all samples. There is no breakdown by issue type, rule category, project, or code complexity."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.D discusses cases where NCO's coverage suffers due to conflicting LBS and KBS feedback, with a concrete example of an unused import case. The bimodal distribution of DAT is also discussed as a failure pattern."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that DAT and NCO did not improve accuracy over Mi (RQ1, RQ3), and that DAT shows a polarized bimodal performance pattern with many Rank 5 results (Section IV.C.4). NCO showed only marginal coverage improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'hybrid strategies enhance the relevance, completeness, and overall quality of review comments.' The results support this for RAG (accuracy improvement) and DAT/RAG (coverage improvement), though NCO shows only modest gains. The abstract is somewhat generous but not contradicted."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation-style comparisons (DAT, RAG, NCO each modify one aspect of the pipeline). The controlled experimental design — same test set, same base model, varying only the integration strategy — supports these causal claims adequately."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Review Generation' broadly, but the study tests only on Java code with PMD and Checkstyle using CodeLlama-7b. The threats to validity mention Java specificity, but the title and abstract do not bound the claims to this setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, the accuracy improvement from RAG could be partly due to the static analyzer output providing correct answers that the model parrots without understanding. The threats section discusses evaluator bias and tool choice but not alternative explanations for the observed performance differences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'CodeLlama-7b' and 'Llama3-70B' but does not provide exact model versions, snapshot dates, or API versions. These are marketing-level names without version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows the prompt used for RAG-based review generation. Figure 4 shows the LLM-as-judge prompt structure. The prompts include the actual text with placeholders clearly indicated and filled from the dataset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B reports detailed hyperparameters: batch size 4 per device, gradient accumulation step size 4, 4-bit quantization, QLoRA with r=16, alpha=32, dropout=0.05. The LLM-as-judge uses a 10-point scale with threshold 8."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach uses standard fine-tuning and inference pipelines without agent loops, tool use, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C describes the data augmentation pipeline in detail: generation of 4 reviews per code change, static analysis application, merging, filtering using Llama3-70B with a threshold of 8/10, balancing KBS/LBS reviews, and discarding overrepresented rules. The final dataset Da has 78,776 samples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.E 'Threats to Validity' is a dedicated subsection discussing evaluator bias, generalizability, dependency on LLM judgment, choice of static analyzers, and evaluation metrics limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: Java-only evaluation limiting generalizability, reliance on Llama3-70B as evaluator with potential misalignment, the specific choice of PMD and Checkstyle, and the win-tie-loss ranking system requiring a two-level difference threshold."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The threats section mentions Java specificity as a limitation, but does not explicitly state what the results do NOT show. There is no explicit bounding of claims to the tested models, dataset, or languages — the conclusion speaks broadly about 'combining static analysis tools with LLMs' without qualifying."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is available at https://zenodo.org/records/14061110, and the replication package at https://github.com/ImenJaoua/Hybrid-Code-Review, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is described: the original dataset DMi from Li et al. [25] is a real-world code review dataset. The augmented dataset Da is generated through the pipeline in Section III.C with explicit filtering criteria."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Two human evaluators assessed accuracy in RQ1, described only as having 'good expertise in code review.' No details on how they were recruited, their backgrounds, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is well-documented: from the original 27,267 Java entries in Do, to augmented data generation, Llama3-70B filtering with threshold 8, rule balancing, and final 78,776 samples in Da (Section III.C, Figures 3, 5). The test set filtering to 1,245 common samples is also explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned in the paper. There is no acknowledgments section disclosing grants or funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with Université de Montréal, clearly disclosed on the first page. They are not evaluating a commercial product they are affiliated with."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, and the authors are at a university with no apparent commercial stake in the outcome. This appears to be unfunded academic work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes CodeLlama-7b and uses Llama3-70B as an evaluator, but does not state the training data cutoff dates for either model. This is relevant since the code review dataset could overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeLlama-7b or Llama3-70B may have seen the code review dataset during pre-training. The test data from Li et al. [25] was published in 2022, likely before these models' training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The code review dataset from Li et al. [25] was published in 2022, before the training cutoff of both CodeLlama and Llama3. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The study involves two human evaluators rating code reviews (RQ1). No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluators participate in the study but no IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The two evaluators are described only as having 'good expertise in code review.' No demographics, years of experience, or other characterization is provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for the evaluators are stated. They were not recruited from a broader pool with stated criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control assignment for participants. The evaluators both assessed the same reviews."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B states: 'All reviews were anonymized and presented to the evaluators in a randomized order. This approach minimizes confirmation bias, preventing evaluators from unconsciously favoring certain methods by being unaware of the source of each review.'"
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Only two evaluators participated in a single assessment task. Attrition is not applicable in this context."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, tokens consumed, or wall-clock time for generating reviews is reported, despite the method involving multiple LLM inference calls per code change."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions four NVIDIA RTX 3090 GPUs were used for training, but does not report total training time, GPU hours, or the computational cost of the Llama3-70B evaluation runs."
       }
     }
   },
   "claims": [
     {
       "claim": "The RAG approach significantly outperforms the standalone LBS (Mi) in terms of accuracy for code review generation.",
       "evidence": "Figure 8 (manual evaluation, RQ1) and Figure 9 (LLM evaluation, RQ3) show RAG has a higher proportion of accurate reviews compared to Mi. Cohen's kappa of 0.72 validates LLM-human agreement.",
       "supported": "moderate"
     },
     {
       "claim": "DAT and RAG achieve the best coverage, together accounting for nearly 80% of Rank 1 reviews.",
       "evidence": "Figure 10 shows DAT achieving 49% Rank 1 and RAG achieving approximately 30% Rank 1 (70% in top 2 ranks). Win-tie-loss analysis in Figure 11 confirms DAT and RAG outperform baselines in coverage.",
       "supported": "moderate"
     },
     {
       "claim": "NCO does not notably improve accuracy and offers only marginal increase in coverage compared to the standalone LBS.",
       "evidence": "Figures 8 and 9 show NCO accuracy comparable to Mi. Figure 10 shows NCO primarily in middle ranks (Rank 3). Section IV.D discusses how NCO's LBS component can contradict KBS findings.",
       "supported": "moderate"
     },
     {
       "claim": "Llama3-70B closely mirrors human evaluations with Cohen's kappa of 0.72 (substantial agreement).",
       "evidence": "Section IV.C.2 reports Cohen's kappa of 0.72 on the 10% manual evaluation subset, comparing human and LLM ratings on the same accuracy scale.",
       "supported": "moderate"
     },
     {
       "claim": "Hybrid approaches bridge the gap between rule-based tools and deep learning models for code review.",
       "evidence": "The combined results across RQ1-RQ4 show RAG improves accuracy over LBS while maintaining broader coverage than KBS. However, the claim is broad and the evidence is limited to one dataset and one programming language.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes three strategies for combining static analyzers (KBS) with fine-tuned LLMs (LBS) for automated code review: Data-Augmented Training (DAT), Retrieval-Augmented Generation (RAG), and Naive Concatenation of Outputs (NCO). RAG emerged as the most effective strategy, improving both accuracy and coverage over standalone LBS by incorporating static analysis results into inference prompts. DAT achieved broad coverage (49% Rank 1) but showed a polarized bimodal performance pattern, while NCO provided only marginal improvements. The evaluation used both human assessment and LLM-as-a-judge (Llama3-70B) with Cohen's kappa of 0.72 indicating substantial agreement between the two evaluation methods.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims RAG 'significantly outperforms' Mi but provides no statistical tests to support this. All comparisons are based on visual inspection of bar charts and percentage distributions without any hypothesis testing."
     },
     {
       "flag": "Small evaluator pool",
       "detail": "Only two human evaluators assessed accuracy, with no characterization beyond 'good expertise in code review.' This is insufficient for establishing the reliability of the manual evaluation, and inter-rater agreement between the two humans is not reported (only human-LLM agreement via Cohen's kappa)."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The code review dataset from Li et al. (2022) was published before CodeLlama and Llama3 training cutoffs. Neither model's training cutoff is stated, and no contamination analysis is performed. The evaluation results could be inflated if the models saw this data during pre-training."
     },
     {
       "flag": "LLM evaluating LLM outputs",
       "detail": "Llama3-70B is used both as the data filtering judge (threshold 8/10 for DAT data quality) and as the evaluation judge (RQ3, RQ4). This creates a circular dependency: the training data was curated by a model from the same family as the evaluator, potentially inflating scores for approaches trained on LLM-curated data."
     },
     {
       "flag": "No variance or reproducibility measures",
       "detail": "All results appear to be single-run. No standard deviations, confidence intervals, or multi-seed experiments are reported, making it impossible to assess result stability."
     }
   ],
   "cited_papers": [
     {
       "title": "Code review automation: strengths and weaknesses of the state of the art",
       "authors": ["R. Tufano", "O. Dabic", "A. Mastropaolo", "M. Ciniselli", "G. Bavota"],
       "year": 2024,
       "relevance": "Evaluates state-of-the-art code review automation approaches, directly relevant to assessing methodology in LLM-based code review research."
     },
     {
       "title": "Core: Resolving code quality issues using llms",
       "authors": ["N. Wadhwa", "J. Pradhan", "A. Sonwane", "S. P. Sahu", "N. Natarajan", "A. Kanade", "S. Parthasarathy", "S. Rajamani"],
       "year": 2024,
       "relevance": "Uses LLMs for code quality issue resolution, relevant to understanding LLM capabilities in code review tasks."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Z. Li", "S. Lu", "D. Guo", "N. Duan", "S. Jannu", "G. Jenks", "D. Majumder", "J. Green", "A. Svyatkovskiy", "S. Fu"],
       "year": 2022,
       "relevance": "Pre-trains CodeT5 for code review tasks; the dataset used in this paper originates from this work."
     },
     {
       "title": "Llama-reviewer: Advancing code review automation with large language models through parameter-efficient fine-tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "relevance": "Applies parameter-efficient fine-tuning of LLMs specifically for code review, directly comparable methodology."
     },
     {
       "title": "RepairAgent: An autonomous, llm-based agent for program repair",
       "authors": ["I. Bouzenia", "P. Devanbu", "M. Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Combines LLMs with static analysis for autonomous program repair, demonstrating agentic approaches in software engineering."
     },
     {
       "title": "STALL+: Boosting llm-based repository-level code completion with static analysis",
       "authors": ["J. Liu", "Y. Chen", "M. Liu", "X. Peng", "Y. Lou"],
       "year": 2024,
       "arxiv_id": "2406.10018",
       "relevance": "Integrates static analyzers with LLMs for code completion through multi-phase approach, closely related hybrid methodology."
     },
     {
       "title": "Judging llm-as-a-judge with mt-bench and chatbot arena",
       "authors": ["L. Zheng", "W.-L. Chiang", "Y. Sheng", "S. Zhuang", "Z. Wu", "Y. Zhuang", "Z. Lin", "Z. Li", "D. Li", "E. Xing"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-judge methodology used in this paper's evaluation design."
     },
     {
       "title": "Improving the learning of code review successive tasks with cross-task knowledge distillation",
       "authors": ["O. Ben Sghaier", "H. Sahraoui"],
       "year": 2024,
       "relevance": "Applies knowledge distillation to code review tasks, from the same research group, relevant to code review automation methodology."
     },
     {
       "title": "SkipAnalyzer: A tool for static code analysis with large language models",
       "authors": ["M. M. Mohajer", "R. Aleithan", "N. S. Harzevili", "M. Wei", "A. B. Belle", "H. V. Pham", "S. Wang"],
       "year": 2023,
       "arxiv_id": "2310.18532",
       "relevance": "Combines static analysis with LLMs to reduce false positives in bug detection, relevant hybrid approach for code quality."
     },
     {
       "title": "No more manual tests? Evaluating and improving ChatGPT for unit test generation",
       "authors": ["Z. Yuan", "Y. Lou", "M. Liu", "S. Ding", "K. Wang", "Y. Chen", "X. Peng"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "Evaluates LLM capabilities in test generation with iterative repair, relevant to understanding LLM limitations in SE tasks."
     },
     {
       "title": "Adaptive test generation using a large language model",
       "authors": ["M. Schäfer", "S. Nadi", "A. Eghbali", "F. Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "Uses LLMs for adaptive test generation combining static analysis context, relevant hybrid approach for software testing."
     }
   ]
 }