ai-research-survey

scan.json (25176B)

---

 {
   "paper": {
     "title": "A Survey of Code Review Benchmarks and Evaluation Practices in Pre-LLM and LLM Era",
     "authors": [
       "Taufiqul Islam Khan",
       "Shaowei Wang",
       "Haoxiang Zhang",
       "Tse-Hsun Chen"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.13377",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or any code release is mentioned in the paper. The survey could have released its data extraction scripts or analysis code but did not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or supplementary data file is provided. The survey's extracted metadata (datasets, metrics, task classifications for 99 papers) is not released as a structured artifact."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey with potential analysis scripts, this could have been documented but was not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper describes the methodology at a high level but does not provide instructions for reproducing the search, screening, or classification pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. It does not make comparative claims between systems that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments and therefore has no effect sizes to report."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. The 99 papers analyzed are determined by the search and screening process, not a statistical sampling procedure."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly compares against prior survey work in Section 2.2, discussing Wang et al. [99] (2011-2019 code review studies), Paul et al. [69] (code generation benchmarks), and Wang et al. [101] (SDLC benchmarks). These serve as baselines for the survey's contribution."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related surveys discussed include Wang et al. [101] from 2025, Paul et al. [69] from 2024, and Wang et al. [99] from 2021, which are reasonably contemporary."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "A survey paper has no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper without a system evaluation requiring metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not produce outputs requiring human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive per-category breakdowns across its taxonomy: 5 high-level domains, 18 sub-tasks, with separate detailed tables for each sub-task showing datasets, metrics, and data points for both Pre-LLM and LLM eras (Tables 3-28). Figure 6 provides comparative distribution of datasets across sub-tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses limitations of existing benchmarks in the LLM era, identifying specific failure cases: missing task coverage (impact analysis, commit decomposition vanished), reliance on static metrics that fail to capture functional correctness (e.g., 'An LLM might suggest a fix that introduces a deadlock but still receives a high score'), and lack of granular benchmarking."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: Change Understanding and Analysis tasks have 'nearly vanished' in the LLM era (from 14 datasets to 1), single-language datasets decreased from 59% to 24%, and current benchmarks fail to capture functional correctness through static metrics. The dominance of Peer Review (60% of LLM-era datasets) is presented as a problematic imbalance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about analyzing 99 papers (58 Pre-LLM, 41 LLM), proposing a multi-level taxonomy with 5 domains and 18 tasks, observing a shift toward generative peer review, increasing multilingual coverage, and declining change understanding tasks are all supported with data in the body. Section 6 (RQ3) provides the comparative analysis backing abstract claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims such as 'researchers probably consider that understanding is no longer a separate research goal, instead it is a prerequisite that is now bundled into Comment Generation' (Section 6.1). This causal interpretation of why Change Understanding tasks declined is speculative and not supported by evidence (e.g., interviews with researchers or analysis of paper content). The language 'moving from helping humans understand code to having machines perform the task directly end-to-end' implies causation from LLM availability."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'A Survey of Code Review Benchmarks and Evaluation Practices in Pre-LLM and LLM Era' suggests comprehensive coverage, but the search is limited to specific databases and keywords (Section 3.1). The paper acknowledges 'the rapid growth of AI-driven code review means new datasets are published weekly, making absolute exhaustiveness a moving target' (Section 7.2) but does not bound its generalizations to the specific search scope or time window."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one interpretation for trends (e.g., Change Understanding vanished because LLMs bundle it into generation) without considering alternatives. For instance, the decline could be due to publication venue bias, keyword search limitations, or researcher familiarity. The threats-to-validity section (Section 7.2) mentions potential omissions but does not discuss alternative explanations for the observed trends."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use any AI models for its analysis."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use any models requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the four-stage paper collection process: (1) Terminology Summary with 20 search terms listed, (2) Literature Retrieval across 7 databases yielding 160 publications, (3) Literature Screening with four explicit inclusion criteria reducing to 77 papers, (4) Snowball Expansion adding 25 studies for a final 99 papers. The criteria at each stage are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Threats to Validity' discusses internal and construct validity threats in a dedicated subsection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 discusses specific threats: 'there is a risk that certain papers might have been missed' despite snowballing, 'the rapid growth of AI-driven code review means new datasets are published weekly', and 'Human bias or fatigue could lead to inconsistent labeling across the survey.' The paper describes specific mitigation steps (dual-reviewer protocol, third arbitrator, cross-referencing with prior surveys)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what is out of scope or what the results do NOT show. It does not clarify boundaries such as which types of code review work are excluded (e.g., purely industrial tools without published papers), or acknowledge that findings are bounded to the specific databases searched. The abstract and title imply comprehensive coverage without stating explicit boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw data (list of 99 papers with extracted metadata, task labels, dataset details, metric classifications) is not available for download. Only aggregated results in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data collection in detail: 20 search terms used, 7 academic databases searched (IEEE Xplore, ACM DL, Elsevier ScienceDirect, ACL Anthology, arXiv, Google Scholar, SpringerLink), time period 2015-2025, four-stage filtering process with counts at each stage (160 → 77 → 99 with snowballing)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a study with human participants. No recruitment is applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the pipeline: keyword search → 160 papers → title/abstract/full-text screening with 4 criteria → 77 papers → snowball expansion → 99 papers. Section 3.3 documents the taxonomy construction pipeline with open coding, selective coding, and inter-rater agreement (Cohen's Kappa = 0.81)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, grants, or acknowledgments section mentioning funding sources is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Manitoba (Khan, Wang), Huawei Canada Centre for Software Excellence (Zhang), Concordia University (Chen). One author is from Huawei, a major technology company with interests in code review tools."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed at all, so independence cannot be assessed. One author is affiliated with Huawei Canada, a company that develops code review tools and has a potential interest in the survey's findings about LLM-based code review."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a human subjects study. Paper inclusion/exclusion criteria are assessed under data_preprocessing_documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not an experimental study with human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not an experimental study with human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey of papers, not a human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method whose cost would be relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no significant computational requirements."
       }
     }
   },
   "claims": [
     {
       "claim": "The survey analyzed 99 research papers (58 Pre-LLM era and 41 LLM era) spanning 2015-2025.",
       "evidence": "Section 3.1 describes the collection process yielding 99 papers. Section 3.2 and Table 1 list all papers by venue. The abstract states '99 research papers (58 Pre-LLM era and 41 LLM era).'",
       "supported": "strong"
     },
     {
       "claim": "Change Understanding and Analysis has transitioned from a research cornerstone (14 datasets in Pre-LLM) to nearly absent (1 dataset in LLM era).",
       "evidence": "Figure 6 shows the comparative distribution. Section 6.1 states: 'In the Pre-LLM era, this was a cornerstone of research (14 datasets), while in the LLM era, it has nearly vanished as a standalone topic (1 dataset).'",
       "supported": "strong"
     },
     {
       "claim": "Peer Review tasks dominate the LLM era, accounting for nearly 60% of all datasets.",
       "evidence": "Section 6.1 and Figure 6 show the distribution. The paper states: 'The most obvious trend is the sheer dominance of Peer Review, which accounts for nearly 60% of datasets in LLM era.'",
       "supported": "strong"
     },
     {
       "claim": "The LLM era shows a significant shift from language-specific to cross-language generalization, with single-language datasets dropping from 59% to 24%.",
       "evidence": "Section 6.2 and Figure 7: 'In the Pre-LLM era, research was highly concentrated on single-language studies, with nearly three-fifths (59%) of all datasets restricted to a single programming language.' 'single-language datasets remain present, their relative proportion has plummeted to 24%.'",
       "supported": "strong"
     },
     {
       "claim": "Datasets covering nine or more programming languages increased from 2% (Pre-LLM) to 34% (LLM era).",
       "evidence": "Section 6.2 and Figure 7: 'datasets covering nine or more languages have become a dominant category in the LLM era (34%), whereas they were virtually non-existent (2%) in the earlier period.'",
       "supported": "strong"
     },
     {
       "claim": "The decline in Change Understanding tasks is because researchers consider understanding is now bundled into end-to-end generation.",
       "evidence": "Section 6.1 states: 'researchers probably consider that understanding is no longer a separate research goal, instead it is a prerequisite that is now bundled into Comment Generation.' The word 'probably' acknowledges this is speculative.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This survey of 99 code review papers (58 Pre-LLM, 41 LLM era, 2015-2025) proposes a multi-level taxonomy of 5 domains and 18 sub-tasks. The key finding is a dramatic structural shift: Change Understanding and Analysis collapsed from 14 datasets to 1, while Peer Review tasks grew to dominate 60% of LLM-era datasets, suggesting a move from human-centric assistance to end-to-end generation. The LLM era also shows a shift from single-language studies (59% to 24%) to highly multilingual benchmarks covering 9+ languages (2% to 34%). The authors identify critical gaps including missing macro-level review tasks, over-reliance on static text-matching metrics, and lack of runtime/dynamic evaluation in current benchmarks.",
   "red_flags": [
     {
       "flag": "Industry affiliation without conflict disclosure",
       "detail": "One author (Haoxiang Zhang) is affiliated with Huawei Canada's Centre for Software Excellence. Huawei develops code review tools and has commercial interests in LLM-based code review. No conflicts of interest statement or funding disclosure is provided."
     },
     {
       "flag": "No structured quality assessment of surveyed papers",
       "detail": "The survey collects and organizes 99 papers but does not assess their methodological quality. The paper acknowledges authors conducted a 'quality assessment' to verify papers were 'methodologically sound' (Section 3.1) but provides no details on how quality was assessed or what criteria were used. This risks laundering the signal-to-noise ratio of its sources."
     },
     {
       "flag": "Causal interpretations unsupported by evidence",
       "detail": "The paper offers causal explanations for observed trends (e.g., Change Understanding declined because LLMs bundle it into generation) without evidence. These are post-hoc rationalizations based on counting dataset frequencies, not supported by analysis of paper content or researcher interviews."
     },
     {
       "flag": "Inconsistent paper counts between abstract and body",
       "detail": "The abstract says '99 research papers (58 Pre-LLM era and 41 LLM era)' but Section 3.1 says 'we collected 61 Pre-LLM code review papers and 45 LLM-based papers' before filtering. The final count of 99 = 58 + 41 is clear, but the intermediate numbers (61 + 45 = 106) are not reconciled with the 77 from screening plus 25 from snowballing = 102. The pipeline count discrepancies (160 → 77 + 25 = 102 or 106 → 99) are confusing."
     }
   ],
   "cited_papers": [
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "relevance": "Introduces CodeReviewer, a foundational LLM pre-training approach for code review that is extensively referenced as a benchmark dataset source in this survey."
     },
     {
       "title": "CodeReviewQA: The Code Review Comprehension Assessment for Large Language Models",
       "authors": ["Hong Yi Lin", "Chunhua Liu", "Haoyu Gao"],
       "year": 2025,
       "relevance": "Proposes a code review comprehension benchmark for LLMs, directly relevant to evaluating LLM capabilities in code review tasks."
     },
     {
       "title": "CodeFuse-CR-Bench: A Comprehensiveness-aware Benchmark for End-to-End Code Review Evaluation in Python Projects",
       "authors": ["Hanyang Guo", "Xunjin Zheng", "Zihan Liao"],
       "year": 2025,
       "arxiv_id": "2509.14856",
       "relevance": "End-to-end code review benchmark for LLMs covering structured review generation across multiple dimensions."
     },
     {
       "title": "BitsAI-CR: Automated Code Review via LLM in Practice",
       "authors": ["Tao Sun", "Jian Xu", "Yuanpeng Li"],
       "year": 2025,
       "relevance": "Industrial-scale LLM-based code review system deployed in practice, relevant to understanding real-world LLM code review deployment."
     },
     {
       "title": "Deep Learning-Based Code Reviews: A Paradigm Shift or a Double-Edged Sword?",
       "authors": ["Rosalia Tufano", "Alberto Martin-Lopez", "Ahmad Tayeb"],
       "year": 2025,
       "relevance": "Empirical study on how DL-based code reviews affect human reviewer performance, relevant to evaluating AI-assisted code review impact."
     },
     {
       "title": "CodeAgent: Autonomous Communicative Agents for Code Review",
       "authors": ["Xunzhu Tang", "Kisub Kim", "Yewei Song"],
       "year": 2024,
       "relevance": "Proposes autonomous agent-based approach to code review with security focus, relevant to agentic AI code review evaluation."
     },
     {
       "title": "AI-Assisted Fixes to Code Review Comments at Scale",
       "authors": ["Chandra Maddila", "Negar Ghorbani", "James Saindon"],
       "year": 2025,
       "arxiv_id": "2507.13499",
       "relevance": "Large-scale industrial study of AI-assisted code revision from review comments, relevant to understanding AI productivity in code review workflows."
     },
     {
       "title": "Benchmarking and Studying the LLM-based Code Review",
       "authors": ["Zhengran Zeng", "Ruikai Shi", "Keke Han"],
       "year": 2025,
       "arxiv_id": "2509.01494",
       "relevance": "Benchmark study specifically for LLM-based code review with line-level issue localization evaluation."
     },
     {
       "title": "Can we benchmark Code Review studies? A systematic mapping study of methodology, dataset, and metric",
       "authors": ["Dong Wang", "Yuki Ueda", "Raula Gaikovina Kula"],
       "year": 2021,
       "doi": "10.1016/j.jss.2021.111009",
       "relevance": "Prior systematic mapping study of code review methodologies that serves as a baseline for this survey's contribution."
     },
     {
       "title": "Benchmarks and Metrics for Evaluations of Code Generation: A Critical Review",
       "authors": ["Debalina Ghosh Paul", "Hong Zhu", "Ian Bayley"],
       "year": 2024,
       "relevance": "Critical review of code generation evaluation practices, providing context for how benchmark quality issues extend beyond code review."
     },
     {
       "title": "Software Development Life Cycle Perspective: A Survey of Benchmarks for Code Large Language Models and Agents",
       "authors": ["Kaixin Wang", "Tianlin Li", "Xiaoyu Zhang"],
       "year": 2025,
       "arxiv_id": "2505.05283",
       "relevance": "SDLC-wide benchmark survey identifying code review as a major gap, directly motivating this survey's contribution."
     },
     {
       "title": "Fine-tuning and prompt engineering for large language models-based code review automation",
       "authors": ["Chanathip Pornprasit", "Chakkrit Tantithamthavorn"],
       "year": 2024,
       "doi": "10.1016/j.infsof.2024.107523",
       "relevance": "Study on fine-tuning and prompting strategies for LLM-based code review, relevant to understanding LLM evaluation methodology."
     }
   ]
 }