ai-research-survey

scan.json (20779B)

---

 {
   "paper": {
     "title": "CRScore++: Reinforcement Learning with Verifiable Tool and AI Feedback for Code Review",
     "authors": ["Manav Nitin Kapadnis", "Atharva Naik", "Carolyn Rosé"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.00296"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper. The paper references external tools (Ruff, PyScent) but does not release its own training/evaluation code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available CodeReviewer dataset (Apache 2.0 license, linked in Appendix B.3) and references public tools. The base dataset is publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 specifies GPU type (NVIDIA A100 80GB), model versions (Qwen2.5-Coder-3B/7B-Instruct), flash attention, batch size, max sequence length, training epochs, and references HuggingFace alignment-handbook recipes. Sufficient to recreate environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described in text but there are no executable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-4 report only point estimates. No confidence intervals, error bars, or ± notation appear anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., '56% relative improvement') but provides no statistical significance tests. Comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context (e.g., '56% relative improvement in comprehensiveness for 3B models' in Section 6, and absolute changes from zero-shot shown in parentheses in Tables 1-2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 33 samples per language were chosen for human evaluation, or why 5,000 instances for DPO. The limitations section acknowledges the human eval sample is small but provides no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any results. Single-run numbers only with no indication of variability across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes multiple baselines: Zero Shot, CR dataset (direct training on CodeReviewer), and Tool Guided, compared against Stage 1 (SFT) and Stage 2 (DPO)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are all internal configurations (zero-shot, direct fine-tuning, tool-guided). No comparison against other recent code review generation methods such as Jaoua et al. (2025) despite discussing them in related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The staged approach (Zero Shot → Tool Guided → Stage 1 SFT → Stage 2 DPO) serves as an ablation, showing incremental contributions of each component. Table 3 separately evaluates tool utilization."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: comprehensiveness, conciseness, and relevance (Tables 1-2, 4). Table 3 adds accuracy and coverage for tool utilization."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 describes a human evaluation study with 4 experienced software developers on 100 code changes (33 per language), rating on 5-point Likert scales. Results in Table 4."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses CodeReviewer test sets for evaluation, separate from the training data (20,888 SFT samples, 5,000 DPO samples). Cross-language evaluation uses Java and JavaScript test sets not seen during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size (3B vs 7B), training stage, and programming language (Python, Java, JavaScript) across Tables 1-4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the comprehensiveness-conciseness trade-off (Section 5.1), where DPO models sacrifice brevity. Also notes that models trained on CodeReviewer ground truth perform worse than zero-shot."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Models trained directly on CodeReviewer data perform worse than zero-shot (Table 1). DPO reduces conciseness scores. Table 3 shows DPO provides no accuracy improvements over SFT for tool utilization."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about improving student models through SFT+RL and cross-language generalization are supported by Tables 1-2 and 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about CRScore++ improving performance are supported by controlled ablation: same base model with progressive addition of components (zero-shot → tool-guided → SFT → DPO). This is adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Review' generally, but the paper only tests on Qwen2.5-Coder models with GPT-4o-mini as teacher, on CodeReviewer dataset, for 3 languages. The limitations section acknowledges single model family but the title/abstract do not bound claims to this setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the cross-language generalization could be due to the base model's pre-existing multilingual capabilities rather than CRScore++ training. No confounds are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4o Mini' and 'Qwen2.5-Coder-Instruct' but does not provide API snapshot dates or specific version identifiers for GPT-4o-mini. No date stamp like 'gpt-4o-mini-2024-07-18'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendices A.2, A.3, B.1, and B.2 for the teacher model, SFT training, LLM-as-a-judge evaluation, and CoT evaluation prompts respectively."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 and Section 3 report learning rate (2e-5), batch size (2/8), epochs (2), β=0.1 for DPO, weight decay 0.01, max sequence length 1024, fp16, seed 42, eval steps 100, AdamW optimizer."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a standard SFT+DPO training pipeline with static analysis tool integration, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 documents dataset statistics: 20,888 Python examples for SFT, 5,000 subset for DPO, 20 candidates per sample scored by GPT-4o-mini, pairs with Δ≥2 selected. Static analysis tool pipeline described in Section 3.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' provides a dedicated, substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: small human eval sample (33 per language), GPT-4o-mini evaluation bias, limitation to Qwen model family, exclusion of LCoT models, limited static analysis toolkit coverage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states: limited to 3 programming languages, Qwen model family only, no LCoT models tested, static analysis toolkit doesn't cover test coverage or architectural consistency."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated SFT training data, DPO preference pairs, and evaluation outputs are not released. Only the base CodeReviewer dataset is public."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 describes data collection: CodeReviewer dataset selection, annotation with Ruff and PyScent, GPT-4o-mini generation of reviews and topics, DPO pair creation with score differential ≥2."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation uses '4 experienced software developers' but does not describe how they were recruited, their background, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: CodeReviewer → static analysis annotation → GPT-4o-mini review generation (20,888 samples) → DPO subset (5,000) → 20 candidates per sample → scoring → pair selection (Δ≥2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section mentioning grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Carnegie Mellon University Language Technologies Institute are clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini as teacher/judge and Qwen2.5-Coder as student but does not state training data cutoff dates for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The CodeReviewer dataset is publicly available. No discussion of whether GPT-4o-mini or Qwen2.5-Coder may have seen CodeReviewer test data during pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeReviewer was published in 2022 and is publicly available. Both GPT-4o-mini and Qwen2.5-Coder were trained after 2022, so contamination is plausible. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the human evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. The Ethics Statement mentions informed consent and fair compensation but not ethics review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Annotators are described only as '4 experienced software developers' with no further demographic information (years of experience, etc.)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for selecting the 4 annotators are described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with treatment/control assignment. All annotators rated the same reviews."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether annotators knew which model configuration produced which review."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether all 4 annotators completed all annotations or if there was any dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency per review is reported. The paper mentions API calls for evaluation but not the cost of generating a single review at inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4 states: ~450 GPU hours on A100 hardware total, SFT ~8 hours, DPO ~6 hours, ~40,000 GPT-4o-mini API calls for data preparation, ~2,600 API calls for evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "CRScore++ achieves 56% relative improvement in comprehensiveness for 3B models and 42% relative improvement in relevance compared to zero-shot baselines on Python.",
       "evidence": "Table 1: Qwen 3B zero-shot comprehensiveness 0.43 → Stage 2 0.67 (+0.24); relevance 0.45 → 0.64 (+0.19). Section 6.",
       "supported": "strong"
     },
     {
       "claim": "Models trained exclusively on Python achieve nearly equivalent performance when reviewing Java and JavaScript code.",
       "evidence": "Table 2: Stage 2 3B Python comprehensiveness 0.67, Java 0.64, JavaScript 0.62. Gaps are small. Human evaluation (Table 4) shows similar patterns.",
       "supported": "moderate"
     },
     {
       "claim": "Models trained directly on CodeReviewer ground truth perform worse than zero-shot approaches.",
       "evidence": "Table 1: Qwen 3B CR dataset comprehensiveness 0.10 vs zero-shot 0.43; relevance 0.20 vs 0.45.",
       "supported": "strong"
     },
     {
       "claim": "DPO provides no accuracy improvements over SFT for tool utilization, indicating tool understanding is established during initial distillation.",
       "evidence": "Table 3: SFT models show higher or comparable accuracy scores to DPO across all languages. Section 5.3.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CRScore++ combines verifiable static analysis signals (linters, code smell detectors) with LLM-based subjective feedback for RL training of code review models. The two-stage approach (SFT + DPO) improves comprehensiveness and relevance of generated reviews at the cost of conciseness. Models trained only on Python generalize well to Java and JavaScript without language-specific fine-tuning. Notably, models trained on scraped CodeReviewer ground truth perform worse than zero-shot baselines, motivating the use of teacher-generated demonstrations.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on point estimate differences without any significance testing, error bars, or variance reporting. With LLM-as-a-judge evaluation, the variability of the judge itself is not quantified."
     },
     {
       "flag": "LLM-as-a-judge circularity",
       "detail": "GPT-4o-mini is used both as the teacher model for generating training data AND as the judge for evaluation. This creates a potential circularity where the evaluation metric favors outputs that match the teacher's style."
     },
     {
       "flag": "No external baselines",
       "detail": "Despite discussing Jaoua et al. (2025) in related work, no comparison is made against this or any other published code review generation method. All baselines are internal configurations."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CodeReviewer dataset (2022) is publicly available and likely in the training data of both GPT-4o-mini and Qwen2.5-Coder. This is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "CRScore: Grounding Automated Evaluation of Code Review Comments in Code Claims and Smells",
       "authors": ["Atharva Naik", "Marcus Alenius", "Daniel Fried", "Carolyn Rose"],
       "year": 2024,
       "relevance": "Predecessor metric for evaluating code review quality that this paper extends into a training framework."
     },
     {
       "title": "Combining Large Language Models with Static Analyzers for Code Review Generation",
       "authors": ["Imen Jaoua", "Oussama Ben Sghaier", "Houari Sahraoui"],
       "year": 2025,
       "arxiv_id": "2502.06633",
       "relevance": "Closely related work on hybrid LLM+static analysis approaches for code review generation."
     },
     {
       "title": "Automating Code Review Activities by Large-Scale Pre-Training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "relevance": "Source of the CodeReviewer dataset and baseline model used in this paper's experiments."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2024,
       "relevance": "Core alignment method (DPO) used in the paper's preference optimization stage."
     },
     {
       "title": "Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering",
       "authors": ["Ruiqi Wang", "Jiyu Guo", "Cuiyun Gao"],
       "year": 2025,
       "arxiv_id": "2502.06193",
       "relevance": "Validates LLM-as-a-judge for SE evaluation tasks, methodology used in this paper."
     },
     {
       "title": "RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback",
       "authors": ["Harrison Lee", "Samrat Phatale", "Hassan Mansoor"],
       "year": 2024,
       "relevance": "Foundation work on RLAIF that this paper extends with verifiable tool signals."
     },
     {
       "title": "CodeReviewQA: The Code Review Comprehension Assessment for Large Language Models",
       "authors": ["Hong Yi Lin", "Chunhua Liu", "Haoyu Gao"],
       "year": 2025,
       "arxiv_id": "2503.16167",
       "relevance": "Benchmark for evaluating LLM code review comprehension abilities."
     },
     {
       "title": "ReTool: Reinforcement Learning for Strategic Tool Use in LLMs",
       "authors": ["Jiazhan Feng", "Shijue Huang", "Xingwei Qu"],
       "year": 2025,
       "relevance": "Motivates using RL over tool feedback to enhance reasoning and tool utilization in LLMs."
     },
     {
       "title": "A Critical Evaluation of AI Feedback for Aligning Large Language Models",
       "authors": ["Archit Sharma", "Sedrick Scott Keh", "Eric Mitchell"],
       "year": 2024,
       "relevance": "Informs the design decision to use the same teacher model for both SFT and PO stages."
     }
   ]
 }