ai-research-survey

scan.json (25272B)

---

 {
   "paper": {
     "title": "Benchmarking and Studying the LLM-based Code Review",
     "authors": [
       "Zhengran Zeng",
       "Ruikai Shi",
       "Keke Han",
       "Yixin Li",
       "Kaicheng Sun",
       "Yidong Wang",
       "Zhuohao Yu",
       "Rui Xie",
       "Wei Ye",
       "Shikun Zhang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.01494"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWR-Bench, a 1000-PR benchmark with full project context, reveals that current ACR tools achieve at most 19.38% F1, with low precision (high false positives) as the primary limitation. ACR tools detect functional errors better than evolutionary/stylistic issues. A Multi-Review aggregation strategy improves F1 by up to 43.67% by synthesizing multiple independent review passes. Reasoning-enhanced LLMs perform better at code review than standard LLMs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository link is provided (reference [32]: anonymous.4open.science/status/swrbench-1D0E) for code and benchmark."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "SWR-Bench is released via the same anonymous link. The benchmark includes 1000 PRs with metadata, change-points, and codebase checkpoints."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, dependency versions) are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymous repo may contain them, but the paper itself does not describe how to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III and IV are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., PR-Review outperforms others, reasoning LLMs are better) based solely on comparing point estimates with no statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context, e.g., 'increasing issue detection F1 score by up to 43.67%' and absolute F1 values are provided (from 15.25% to 21.91%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark size of 1000 PRs (500 Change, 500 Clean) is stated but no justification is given for why this number is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five ACR approaches are compared: LLM-Review, SWR-Agent, CR-Agent, Hybrid-Review, and PR-Review (Section IV-B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent tools (PR-Agent/PR-Review, CR-Agent from EMNLP 2024) and SOTA LLMs (GPT-o3, Gemini-2.5-Pro, Claude-4-Opus, DeepSeek-R1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The Multi-Review experiment varies the number of aggregated reports (n=0,1,3,5,10) and compares Self-Agg vs Multi-Agg strategies (Figure 7), effectively ablating the aggregation component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results report Precision, Recall, F1, average count, and average severity for both overall and functional change-points."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 validates the LLM-based evaluation method with human experts: 3 human annotators and 2 LLMs independently evaluated 100 code review reports, achieving ~90% agreement on the Hit metric (Figure 5)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWR-Bench is used solely for evaluation, not for tuning any of the evaluated systems. All ACR tools used their default configurations."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides per-change-type performance breakdown across all 11 change types (E.1.1 through F.6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: high false positive rates, low precision, Hybrid-Review's poor performance due to unprocessed static analysis, and agent interaction overhead causing error propagation in CR-Agent."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: Hybrid-Review performs worst (4.48% F1), CR-Agent's multi-agent approach underperforms simpler methods, Claude-4-Opus/Sonnet underperform Claude-3.7-Sonnet, and Qwen-2.5-R1-7B fails due to output formatting issues."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: ~90% agreement validated in Figure 5a, 43.67% F1 improvement shown in Figure 7, and ACR underperformance demonstrated in Tables III-IV."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claim that Multi-Review 'improves' performance is supported by controlled experiments varying n with other factors held constant. The claim about reasoning enhancement is supported by comparing matched model pairs (Qwen-2.5-Chat vs Qwen-2.5-R1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'LLM-based Code Review' generally, but the benchmark uses only 12 Python projects aligned with SWE-Bench. This language-specific limitation is not explicitly acknowledged as a scope boundary."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section (Section V) is brief and does not discuss alternative explanations for observed results. For example, Claude-4's underperformance vs Claude-3.7 is noted as speculative without analysis of confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures change-point detection (hit/not-hit) and frames results in terms of issue detection F1. The claims match the measurement granularity — they do not claim to measure broader 'code review quality' beyond issue detection."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions with snapshot dates are provided: 'o3-0416', 'o4-mini-0416', 'GPT-4o-1120', 'Gemini-2.5-Pro-0325', 'Gemini-2.5-Flash-0427', 'Claude-3.7-Sonnet-0219', 'Claude-4-Sonnet-0514', 'Claude-4-Opus-0514', 'DeepSeek-R1-0120', 'DeepSeek-V3-0324'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language (e.g., 'instructing it to identify instances corresponding to the 11 predefined change-point types') but actual prompt text is not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0.2 for the two baselines they built. ACR tools used official default configurations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "SWR-Agent's scaffold is described (adapted from SWE-Agent, uses tools to explore codebase and execute snippets). CR-Agent's multi-agent architecture is described (two agents for format vs functional defects with inter-agent debate). PR-Review's prompt engineering approach is described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents a detailed 4-step pipeline: source collection and filtering (21,350 PRs), LLM-based verification with majority voting, SZZ filtering and resampling, and manual verification by 5 annotators."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' addresses internal, external, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are somewhat specific: 'potential bugs in our implementation', choosing 'representative tools based on an extensive literature review', selecting '12 popular, well-maintained open-source GitHub projects', and human validation confirming evaluation method reliability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Missing: Python-only limitation, restriction to open-source projects, exclusion of non-LLM and proprietary tools (acknowledged in tool selection but not as a scope boundary on conclusions)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full SWR-Bench dataset including PR source data, ground truth change-points, and codebase checkpoints is available via the anonymous repository link."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes data collection in detail: GitHub API crawling of 12 projects, PR metadata collected (titles, descriptions, commits, diffs, comments, statuses), with specific filtering criteria."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Human annotators are described only as 'five experienced graduate students majoring in computer science' with no details on recruitment, selection criteria, or potential biases. Project selection is described as 'aligned with SWE-Bench' but without explaining why these 12 specifically."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts: 21,350 PRs after initial filtering → 3,500 after LLM verification and quality filtering → 1,000 Change + 1,000 Clean manually verified → final 500+500. Figure 2 shows the pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University and Northwestern Polytechnical University. No company affiliations with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the evaluated models, despite using real GitHub PRs that may appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub PRs in SWR-Bench appeared in the training data of evaluated LLMs, despite the projects being popular open-source repos."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWR-Bench uses PRs from well-known GitHub projects (aligned with SWE-Bench). While the benchmark is new, the underlying PR data existed publicly before model training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The human annotators were used for benchmark construction and evaluation method validation, not as study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 'a full SWR-Bench evaluation (evaluating all 1000 PRs) costs approximately $1.57 using Gemini-2.5-Flash.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for the full experimental evaluation across all models and tools."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results appear to be single-run. The Venn diagram analysis (Figure 6b) shows variability across 5 runs of the same model but this is used to motivate Multi-Review, not to report result stability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for main results in Tables III and IV is not stated. Only the Multi-Review experiment explicitly varies run count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Tools use default configurations but no justification for not exploring alternatives."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'all evaluated ACR tools used their official default configurations' and sets temperature to 0.2 for baselines, avoiding cherry-picking of configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across tools and models with no statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built LLM-Review and SWR-Agent as baselines and constructed SWR-Bench itself. They do not acknowledge the bias of evaluating their own constructions."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different tools have vastly different compute profiles (e.g., Multi-Review with n=10 uses 10x the API calls) but performance is not normalized by compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 validates the evaluation method against human judgment (~90% agreement). The paper discusses what the benchmark measures (change-point detection) and why existing benchmarks are inadequate."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Table III compares tools with different scaffolds (single-turn vs agent vs multi-agent) using different LLMs, confounding scaffold and model effects. Table IV isolates models by fixing PR-Review as the scaffold, but Table III's conclusions about tools don't control for this."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub PRs in SWR-Bench predate the training data of evaluated models, which could mean models have seen the PR discussions and solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing full PR metadata (including review comments in the construction phase) could leak information about what change-points exist."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PRs come from 12 projects aligned with SWE-Bench. No discussion of whether models trained on SWE-Bench data have exposure to these same repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied beyond the SZZ algorithm for filtering missed changes."
       }
     }
   },
   "claims": [
     {
       "claim": "The objective LLM-as-Judge evaluation method achieves ~90% agreement with human expert judgment on the Hit metric.",
       "evidence": "Figure 5a shows Hit-Agreement between 89.2% and 94.9% across all pairwise combinations of 3 human annotators and 2 LLMs (RQ1).",
       "supported": "strong"
     },
     {
       "claim": "Current ACR tools achieve at most 19.38% F1 on SWR-Bench, limited primarily by high false positive rates.",
       "evidence": "Table III shows PR-Review with Gemini-2.5-Pro achieves the highest F1 of 19.38%, with most tools having precision below 10%.",
       "supported": "strong"
     },
     {
       "claim": "ACR tools are more adept at detecting functional errors than evolutionary changes.",
       "evidence": "Table V shows functional change types (F.1-F.6) achieve F1 scores of 19.60-27.65%, while evolutionary types (E.1.1-E.3.2) achieve 6.05-16.45%.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning-enhanced LLMs perform better at code review than standard LLMs.",
       "evidence": "Table IV shows Qwen-2.5-R1-14B achieves 15.95% F1 vs Qwen-2.5-14B at 9.01%. Gemini-2.5-Pro (reasoning) achieves 19.38% F1. However, the comparison is confounded by other model differences.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-Review aggregation strategy increases F1 scores by up to 43.67%.",
       "evidence": "Figure 7 shows Gemini-2.5-Flash Self-Agg (n=10) achieves 21.91% F1, up from baseline 15.25% — a 43.67% relative increase.",
       "supported": "strong"
     },
     {
       "claim": "Claude-4-Opus and Claude-4-Sonnet underperform Claude-3.7-Sonnet on code review.",
       "evidence": "Table IV: Claude-3.7-Sonnet achieves 18.23% F1 vs Claude-4-Opus at 16.99% and Claude-4-Sonnet at 16.61%.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests for any comparisons",
       "detail": "All comparative claims (tool rankings, reasoning enhancement benefits, model comparisons) are based on single-run point estimates with no significance tests, confidence intervals, or variance measures. The Venn diagram analysis (Figure 6b) demonstrates substantial run-to-run variability, yet main results report single runs."
     },
     {
       "flag": "Contamination risk from SWE-Bench-aligned projects",
       "detail": "SWR-Bench uses PRs from 12 projects aligned with SWE-Bench. Models trained on data including these repositories may have seen the PR discussions, code changes, and reviewer comments. This is not discussed."
     },
     {
       "flag": "Python-only scope not acknowledged as limitation",
       "detail": "All 12 projects are Python. The paper titles itself as benchmarking 'LLM-based Code Review' without bounding conclusions to Python."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "Authors built SWR-Bench, the evaluation method, and two of the five baselines (LLM-Review, SWR-Agent). Their benchmark design choices directly affect which tools look better or worse."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Foundational benchmark for LLM-based software engineering that SWR-Bench's project selection is aligned with."
     },
     {
       "title": "CodeAgent: Autonomous Communicative Agents for Code Review",
       "authors": ["Xunzhu Tang", "Kisub Kim"],
       "year": 2024,
       "relevance": "Multi-agent ACR system evaluated as a baseline, representing agent-based approaches to code review."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "doi": "10.1145/3540250.3549081",
       "relevance": "CodeReviewer dataset and model — one of the main prior benchmarks that SWR-Bench aims to supersede."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez"],
       "year": 2024,
       "relevance": "Agent framework adapted to create the SWR-Agent baseline for code review evaluation."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "authors": ["Junwei Liu", "Kaixin Wang"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Survey of LLM-based agents in software engineering, providing context for ACR tools."
     },
     {
       "title": "Combining Large Language Models with Static Analyzers for Code Review Generation",
       "authors": ["Imen Jaoua", "Omar Ben Sghaier"],
       "year": 2025,
       "arxiv_id": "2502.06633",
       "relevance": "Hybrid-Review approach combining LLMs with static analysis, evaluated as a baseline."
     },
     {
       "title": "BitSAI-CR: Automated Code Review via LLM in Practice",
       "authors": ["Tao Sun", "Jiaqi Xu"],
       "year": 2025,
       "arxiv_id": "2501.15134",
       "relevance": "Industrial ACR tool deployment study, representing proprietary approaches excluded from benchmark evaluation."
     },
     {
       "title": "Why do Multi-Agent LLM Systems Fail?",
       "authors": ["Mehmet Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of multi-agent system failures, cited to explain CR-Agent's underperformance."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2025,
       "relevance": "Contamination-aware code benchmark; paper notes LLM rankings on SWR-Bench diverge from LiveCodeBench."
     },
     {
       "title": "Deep Assessment of Code Review Generation Approaches: Beyond Lexical Similarity",
       "authors": ["Yanlin Jiang", "Hui Liu"],
       "year": 2025,
       "arxiv_id": "2501.05176",
       "relevance": "Critiques text-similarity metrics for code review evaluation, motivating SWR-Bench's objective evaluation method."
     }
   ]
 }