ai-research-survey

scan.json (26265B)

---

 {
   "paper": {
     "title": "CodeReviewQA: The Code Review Comprehension Assessment for Large Language Models",
     "authors": [
       "Hong Yi Lin",
       "Chunhua Liu",
       "Haoyu Gao",
       "Patanamon Thongtanunam",
       "Christoph Treude"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.16167"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on Hugging Face (https://huggingface.co/datasets/Tomo-Melb/CodeReviewQA) as stated in the abstract footnote. However, no source code repository URL is provided for the evaluation pipeline itself. The Hugging Face link is a dataset release. Treating this as YES because the benchmark data artifact is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 900 curated examples is publicly available on Hugging Face at https://huggingface.co/datasets/Tomo-Melb/CodeReviewQA, as stated in the footnote on page 1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E mentions hardware (64 cores Intel Xeon Platinum 8462Y+, 928GB RAM, 4x NVIDIA H100-80GB SXM5) and that vLLM was used with Hugging Face models, but no requirements.txt, Dockerfile, or specific library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the experimental setup and prompts used, but does not provide step-by-step reproduction instructions, scripts, or a README with commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '50.3% exact match rate', '78.8% invariant accuracy') throughout Tables 2-6 and the appendix tables. No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims like 'performs vastly better (> 10% increase)' and 'only achieves a 3.7% increase' without statistical significance tests. Model comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports performance differences with baseline context, e.g., 'Llama-3.1-70B-Instruct only achieves a 3.7% increase over Qwen2.5-Coder-14B-Instruct, despite wielding a 5-fold increase in parameter count' (Section 6). Absolute percentages with comparisons are provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 900 examples (100 per language) but no justification is given for why this number was chosen, no power analysis is provided, and no discussion of whether 900 is sufficient for the comparative claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run numbers using greedy decoding (temperature=0). No variance, standard deviation, or spread measures across runs are reported. The invariance testing (N! permutations) is about answer order robustness, not experimental variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 72 LLMs across five scale classes and also compares its MCQA evaluation approach against traditional ACR text-matching evaluation. Existing benchmarks (CodeReviewer, CodeReview-New) are compared in Table 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 72 evaluated models include recent state-of-the-art models as of March 2025, including DeepSeek-R1 (Jan 2025), Falcon3 (Dec 2024), and QwQ-32B. The comparison benchmarks include CodeReview-New (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like analysis: (1) comparing easy vs. hard distractor difficulty levels for CL and SI tasks, (2) analyzing advanced prompting strategies (zero-shot, one-shot, two-shot, chain-of-thought) in Table 6/Appendix G, and (3) decomposing ACR failures into constituent probe failures in Table 3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: exact match rate for ACR, invariant accuracy for MCQA probes, Proportion of Plurarity Agreement (PPA) for symbol binding, perplexity scores and 5-gram accuracy for contamination evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human annotation was used to curate the benchmark dataset (manual verification with Cohen's Kappa = 0.89), but no human evaluation of the LLMs' outputs or code review comprehension was conducted. Evaluation of model performance is entirely automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 900 curated examples serve as the test set. For few-shot experiments (Appendix G), examples were 'held out from the benchmark' for use as demonstrations, indicating separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per scale class (5 classes), per task (CTR, CL easy/hard, SI easy/hard), and per model. Full results for all 72 models across all tasks are provided in Appendix F (Tables 7-13). The benchmark also covers 9 programming languages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Insights from Probing ACR Failures') is entirely dedicated to analyzing failure cases. Table 3 quantifies failure percentages across probes for non-exact match cases, and the paper discusses specific model weaknesses."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: (1) advanced prompting (chain-of-thought, few-shot) does not outperform zero-shot for most tasks (Table 6), (2) CTR capability plateaus with scale, (3) models that achieve exact match often cannot answer MCQA probes correctly (49%-99.6% of successful cases, suggesting memorization rather than comprehension)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that CodeReviewQA 'enables fine-grained assessment of model capabilities' (supported by Section 7's probe analysis), 'mitigates data contamination risks' (supported by Section 8's perplexity/n-gram analysis), and evaluates '72 recently released LLMs on 900 manually curated examples across nine programming languages' (all verified in the paper)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims like 'the inability to improve on CTR presents a fundamental limit to downstream ACR performance' (Section 6) and that exact match without probe success 'alludes to prior exposure and rote memorisation' (Section 7). These causal interpretations are not formally justified; the probe analysis is correlational, not controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 10) explicitly bounds findings: modest dataset size, line-level rather than token-level localization, reliance on specific surrogate LLM for distractors, same prompt for all models, and MCQA only mitigates surface-level not semantic contamination. Claims are largely bounded to the tested models and tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: exact match success without probe success may indicate memorization (Section 7); lower perplexity on older benchmarks may indicate training inclusion (Section 8); the 'Interaction among capabilities' limitation acknowledges they did not investigate causal relationships between probes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 in Appendix F lists all 72 models with exact Hugging Face model identifiers (e.g., 'meta-llama/Llama-3.1-70B-Instruct', 'Qwen/Qwen2.5-Coder-14B-Instruct'), parameter counts, organizations, and release dates. These are specific open-source model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for all tasks (ACR, CTR, CL, SI) are provided in Figure 3 (zero-shot) and Figures 4-6 (few-shot). The templates include placeholders with clear definitions of what fills them (e.g., {code_snippet}, {code_review}, {option_a})."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E states 'we set the temperature to zero for greedy decoding' for all tasks. Section 5.1 specifies 'max output length of one' for MCQA. Section 3.4 mentions the surrogate LLM used 'temperature of 3.5' for distractor generation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct prompting of LLMs with single-turn MCQA and generation tasks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the data pipeline in detail: source dataset (9,367 examples from Guo et al., 2024), heuristic filtering (Appendix C), manual verification by two annotators with Cohen's Kappa = 0.89, stratified sampling per language to 100 each, with 3,761 examples discarded (13% retention rate). The pipeline from raw data to final benchmark is well documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 is a dedicated 'Limitations' section with substantive discussion spanning five specific limitation areas: dataset size, distractor construction, interaction among capabilities, diversity of prompts, and semantic-level data contamination."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'our benchmark has a relatively modest size due to the difficulty of scaling rigorous manual verification', 'Codestral-22B-v0.1 is considered competitive at the time of this writing, benchmark saturation remains inevitable', 'we did not investigate the causal relationships between the capabilities tested in the probes', 'we used the same prompt and hyperparameters for each task'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states boundaries: only open-source models up to 72B parameters, MCQA only mitigates surface-level not semantic contamination, probes are designed independently rather than causally linked, same prompt for all models rather than optimized per model, line-level rather than token-level localization."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly available on Hugging Face (https://huggingface.co/datasets/Tomo-Melb/CodeReviewQA), allowing independent verification of the 900 curated examples."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data source (Guo et al., 2024 dataset of code reviews after January 1, 2022, from 259 most-starred GitHub repositories), filtering to 9 languages, and the full curation pipeline. Appendices A-C provide additional detail on noise types and heuristic filters."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study beyond the two annotator-authors who curated the dataset. The study evaluates LLMs on a benchmark, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts: 9,367 initial examples, 3,761 discarded by heuristic filtering and manual verification, 900 final examples (100 per language). The 13% retention rate is explicitly stated. Cohen's Kappa (mean 0.89, std 0.11) for inter-annotator agreement across 3k independently annotated examples and 46 conflict resolution rounds is reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Melbourne and Singapore Management University. These are academic institutions with no apparent conflict of interest with the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so it is impossible to assess funder independence. The absence of funding disclosure means this cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper extensively discusses data contamination (Section 8) and notes code reviews were from after January 1, 2022, the specific training data cutoff dates for the 72 evaluated models are not stated. Model release dates are listed in Table 7 but training cutoffs are not."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is entirely dedicated to evaluating data contamination. The paper measures perplexity and 5-gram accuracy to assess whether models have memorized benchmark examples, comparing original ACR form with MCQA probe form. The paper explicitly discusses the contamination risk from public GitHub code reviews."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Data contamination is a central concern of the paper. Section 8 compares contamination metrics across benchmarks (Table 4), demonstrating that MCQA reformulation yields higher perplexity and lower n-gram accuracy than the ACR format on the same examples, mitigating contamination effects. The paper also identifies that models achieving exact match without probe success likely memorized examples (Section 7)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study evaluates LLMs on a benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study evaluates LLMs on publicly available code review data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 72 models with N! permutation runs per question per model (6 or 24 runs per question depending on options), which is computationally substantial, but no inference cost, latency, or wall-clock time is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E specifies the hardware (4x NVIDIA H100-80GB SXM5) but does not report total GPU hours, wall-clock time, or any quantification of the computational budget required to run all 72 models."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeReviewQA enables fine-grained assessment of model capabilities disentangled from generative ACR results.",
       "evidence": "Section 7 and Table 3 show that MCQA probe failures can expose specific weaknesses (e.g., CL failure accounts for 74-99.4% of ACR failures in smaller models) that are not visible from ACR exact match rates alone. Models with similar ACR scores (e.g., Qwen2.5-Coder-14B, gemma-2-27b-it, Mistral-Small within 1%) show substantially different probe performance.",
       "supported": "strong"
     },
     {
       "claim": "MCQA reformulation mitigates data contamination effects compared to the original ACR format.",
       "evidence": "Table 4 shows MCQA format yields higher perplexity (6.0 vs 4.5 for Llama-3.1-70B) and lower 5-gram accuracy (25.1% vs 40.3%) compared to the same examples in ACR format, suggesting reduced memorization.",
       "supported": "moderate"
     },
     {
       "claim": "Llama-3.1-70B-Instruct achieved the highest exact match rate of 50.3% on ACR.",
       "evidence": "Table 2 directly reports this result.",
       "supported": "strong"
     },
     {
       "claim": "Models can achieve ACR exact match without comprehending the code review, suggesting memorization.",
       "evidence": "Section 7 reports that models could not accurately answer all probes for 49.0%-99.6% of their successful exact-match cases, with a strict inverse relationship with model size.",
       "supported": "moderate"
     },
     {
       "claim": "Change localisation is the most difficult comprehension task, with proficiency depending heavily on model scale.",
       "evidence": "Section 6 CL Results and Table 2 show most <=3B models achieve 0-3% invariant accuracy on CL, while <=72B models achieve >70% on CLE and >60% on CLH. Table 10 in Appendix F provides full results.",
       "supported": "strong"
     },
     {
       "claim": "Advanced prompting (few-shot, chain-of-thought) does not outperform zero-shot for most MCQA probes.",
       "evidence": "Table 6 in Appendix G shows chain-of-thought consistently degrades performance, and few-shot only outperforms zero-shot on CTR and CL easy. Zero-shot remains best for SI tasks.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeReviewQA decomposes automated code refinement evaluation into three MCQA probes (change type recognition, change localisation, solution identification), revealing that models with similar ACR exact-match scores have substantially different comprehension capabilities. Change localisation is the most difficult task, with most small models (<=3B) achieving near-zero accuracy. The MCQA format reduces data contamination signals compared to the original generative format, and analysis shows 49-99.6% of models' successful exact-match cases correspond to failures on at least one comprehension probe, suggesting memorization over genuine understanding.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All model comparisons across 72 LLMs on 4 tasks are based on point estimates without any statistical significance tests, confidence intervals, or error bars. Claims like '>10% increase' and 'substantially different' are made without formal testing."
     },
     {
       "flag": "Single-run evaluation with deterministic decoding",
       "detail": "All experiments use temperature=0 (greedy decoding), producing a single deterministic output. While the invariance testing permutes answer order, there is no assessment of result stability across different configurations or seeds."
     },
     {
       "flag": "No inference cost disclosure for large-scale evaluation",
       "detail": "Evaluating 72 models with N! permutations per question (900 questions x multiple probes) on 4x H100 GPUs represents substantial compute, but the total cost, GPU hours, and wall-clock time are never reported."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks; directly relevant to code generation and agentic programming evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, foundational to LLM code evaluation methodology."
     },
     {
       "title": "Exploring the Potential of ChatGPT in Automated Code Refinement: An Empirical Study",
       "authors": ["Qi Guo", "Junming Cao", "Xiaofei Xie", "Shangqing Liu", "Xiaohong Li", "Bihuan Chen", "Xin Peng"],
       "year": 2024,
       "relevance": "Source dataset for CodeReviewQA; evaluates ChatGPT on automated code refinement, directly relevant to LLM code review capabilities."
     },
     {
       "title": "Breaking the Silence: The Threats of Using LLMs in Software Engineering",
       "authors": ["June Sallou", "Thomas Durieux", "Annibale Panichella"],
       "year": 2024,
       "relevance": "Discusses data contamination threats in LLM-based software engineering evaluation, a core concern of this paper."
     },
     {
       "title": "Benchmarking Benchmark Leakage in Large Language Models",
       "authors": ["Ruijie Xu", "Zengzhi Wang", "Run-Ze Fan", "Pengfei Liu"],
       "year": 2024,
       "arxiv_id": "2404.18824",
       "relevance": "Provides the n-gram accuracy contamination detection methodology used in this paper's evaluation."
     },
     {
       "title": "Fine-Tuning and Prompt Engineering for Large Language Models-Based Code Review Automation",
       "authors": ["Chanathip Pornprasit", "Chakkrit Tantithamthavorn"],
       "year": 2024,
       "relevance": "Evaluates LLM fine-tuning and prompting for code review automation, directly relevant to AI-assisted code review."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "relevance": "Recent code generation benchmark evaluating LLMs, relevant to the broader landscape of LLM code evaluation."
     },
     {
       "title": "Automating Code Review Activities by Large-Scale Pre-Training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "relevance": "Introduces CodeReviewer benchmark used as a comparison baseline for data contamination evaluation in this paper."
     },
     {
       "title": "Code Review Automation: Strengths and Weaknesses of the State of the Art",
       "authors": ["Rosalia Tufano", "Ozren Dabic", "Antonio Mastropaolo", "Matteo Ciniselli", "Gabriele Bavota"],
       "year": 2024,
       "relevance": "Identifies critical quality issues in code review benchmarks, motivating the manual curation approach of CodeReviewQA."
     },
     {
       "title": "CLEAN-EVAL: Clean Evaluation on Contaminated Large Language Models",
       "authors": ["Wenhong Zhu"],
       "year": 2024,
       "relevance": "Addresses data contamination in LLM evaluation through task reformulation, directly related to CodeReviewQA's MCQA approach."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation, used as a model selection criterion in this work."
     },
     {
       "title": "Too Noisy to Learn: Enhancing Data Quality for Code Review Comment Generation",
       "authors": ["Chunhua Liu", "Hong Yi Lin", "Patanamon Thongtanunam"],
       "year": 2025,
       "relevance": "Identifies noise issues in code review datasets; by some of the same authors, directly related to the data quality motivation of CodeReviewQA."
     }
   ]
 }