ai-research-survey

scan.json (27288B)

---

 {
   "paper": {
     "title": "CodeCriticBench: A Holistic Code Critique Benchmark for Large Language Models",
     "authors": [
       "Alexander Zhang",
       "Marcus Dong",
       "Jiaheng Liu",
       "Wei Zhang",
       "Yejie Wang",
       "Jian Yang",
       "Ge Zhang",
       "Tianyu Liu",
       "Zhongyuan Peng",
       "Yingshui Tan",
       "Yuanxing Zhang",
       "Zhexu Wang",
       "Weixun Wang",
       "Yancheng He",
       "Ken Deng",
       "Wangchunshu Zhou",
       "Wenhao Huang",
       "Zhaoxiang Zhang"
     ],
     "year": 2025,
     "arxiv_id": "2502.16614"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub link is provided in the paper header: https://github.com/multimodal-art-projection/CodeCriticBench. An anonymous data link is also given in Appendix A.1 (https://anonymous.4open.science/r/CodeCriticBench-D657/)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released via the anonymous link in Appendix A.1 and the GitHub repository. The paper states 'We have released our data on an anonymous website.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section listing library versions is provided in the paper. Model links are given in Appendix C (Tables 7, 8) but no execution environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or 'Reproducing Results' section is included in the paper. The GitHub repository is linked but the paper itself contains no specific instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3, 4, 5, and throughout the appendix are reported as single point estimates (e.g., ACC or MSE values) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'performance generally improves with more parameters', 'o1-like models achieving milestone results') but no statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context, e.g., 'Qwen2.5-Coder-3B-Instruct shows gains of 7.5% in ACC and 1.31 in MSE' (Section 4.3, CoT evaluation), and improvements from critiques (Figure 11, scores from 5.72 to 6.91). Results are reported with enough context to understand the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 4,300 samples and subsets like CodeCritic_400 (400 randomly sampled instances), but no justification or power analysis is provided for why these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures across experimental runs are reported. All results appear to be single-run numbers with no indication of result stability across multiple trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 38 models including both open-source and closed-source models of varying sizes, and compares against prior critique benchmarks in Table 2 (CriticBench, CriticEval, JudgeBench, RealCritic)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 38 evaluated models include state-of-the-art systems at the time of writing: DeepSeek-R1, OpenAI o1-Preview, Claude3.5-Sonnet, GPT-4o, Gemini2.0-Flash-Thinking, and Qwen2.5 series models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-style analyses: CoT vs. no-CoT evaluation (Table 5), effect of critique quality by model size (Figure 11), basic vs. advanced evaluation comparison (Figure 9), and performance across difficulty levels (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: ACC (Accuracy) for basic evaluation, MSE (Mean Squared Error) for advanced evaluation, and Pass@1 Accuracy for bug identification (Section 3.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is included. Section 4.3 describes ranking 8 models based on human evaluations (Figure 9), comparing human rankings against basic and advanced critique rankings. Additionally, 20 volunteers assessed correctness for the Code QA subset (Section 3.3)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code generation data is collected 'restricted to test sets' from CodeForces, MBPP, and LiveCodeBench (Section 3.2). The entire benchmark serves as a held-out evaluation set for the 38 models being evaluated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: by difficulty level (Easy/Medium/Hard in Figure 5), by task type (Code Gen vs. Code QA in Table 3), by application scenario (11 scenarios in Figure 4), by error type (23 categories in Table 6), and by model size group."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Error analysis is provided: specific error types where models struggle are discussed (e.g., 'most models, except Doubao-Coder-Preview and Claude3.5-Sonnet, struggle with identifying Performance Issue'), and hard difficulty cases show accuracy drops to ~30% (Section 4.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are reported: 'DeepSeek-R1 and OpenAI o1-Preview underperform on Easy to Medium data, likely due to overthinking' (Section 4.3). Some models perform near random guessing on hard problems (accuracy ~30%). Some error categories show near-zero detection rates (Table 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the benchmark includes two code tasks with different difficulties, basic and advanced evaluations with fine-grained checklists, and 'extensive experimental results' — all supported by the paper's Tables 1-3, Sections 3.1-3.4, and Section 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'This improvement is due to the model having access to more detailed context' (CoT evaluation), 'We suggest this is due to insufficient code-related optimization' (error studies), 'likely due to overthinking' (o1 models). These causal attributions are not supported by controlled experiments isolating the claimed mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Holistic Code Critique Benchmark' but the Limitations section acknowledges 'our evaluation is confined to single-file scenarios' and 'presently focused solely on code.' The benchmark covers only algorithmic problems and StackOverflow-style QA, not all code critique scenarios. The broad title oversells the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Alternative explanations for the results are not substantively discussed. For instance, the scaling law finding and CoT improvements are not examined for confounds. The paper presents one interpretation without considering alternatives (e.g., could CoT improvements be due to formatting rather than reasoning?)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'Claude3.5-Sonnet', 'GPT 4o', 'DeepSeek-v3', 'Gemini2.0-Flash-Thinking'. No API version strings, snapshot dates, or specific model identifiers (e.g., 'gpt-4o-2024-05-13') are provided. Tables 7-8 link to model pages but do not specify exact versions used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix B: Figure 18 (basic evaluation prompt), Figure 19 (advanced evaluation prompt), Figure 20 (error type lists), Figure 21 (bug insertion prompt), Figures 22-28 (additional prompts including CoT variants). The prompts use template variables ($Question, $Answer, $Checklists) but the template structure is fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, sampling settings) are reported for any of the 38 models evaluated. These settings can significantly affect LLM output quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is a direct prompt-response setup where models receive prompts and generate critiques."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the data collection pipeline: for Code Gen, data from CodeForces/MBPP/LiveCodeBench restricted to test sets, DeepSeek-v3 rewriting, bug insertion with sandbox verification and manual review. For Code QA, rule-based filtering of StackOverflow content, Qwen2.5-72B question generation, manual and LLM-assisted review. Section 3.3 documents the difficulty labeling and score calibration process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion (page 13), discussing single-file scope and code-only domain limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section contains only two generic points: evaluation limited to single-file scenarios, and benchmark focused solely on code. No specific threats to validity are discussed, such as potential biases from using LLMs to construct the dataset, inter-annotator agreement among the 20 volunteers, or the representativeness of StackOverflow-sourced data."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'our evaluation is confined to single-file scenarios' and 'CodeCriticBench is presently focused solely on code.' These are specific scope boundaries about what was not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The raw dataset is available via both the GitHub repository (https://github.com/multimodal-art-projection/CodeCriticBench) and an anonymous data link (Appendix A.1), enabling independent verification of the benchmark data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data collection in detail: sources (CodeForces, MBPP, LiveCodeBench for Code Gen; StackOverflow for Code QA), how problems were rewritten by DeepSeek-v3, how bugs were inserted and verified, and how QA pairs were generated and filtered."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '20 volunteers with coding experience' for correctness assessment (Section 3.3) and 'human experts' for iterative discussions, but does not describe how these volunteers or experts were recruited, their backgrounds beyond 'coding experience', or whether the recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: Figure 2 illustrates the data collection process, Section 3.2 describes extraction and filtering steps, Section 3.3 documents difficulty labeling (12 models, 80%/60% thresholds), and score annotation (20% manual annotation → 3 LLMs → linear regression calibration). Final counts are provided (4,300 samples: 1,517 Easy, 1,084 Medium, 1,699 Hard)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper. Authors are affiliated with multiple institutions and companies (Alibaba, Kuaishou, OPPO, M-A-P) but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header: NJU, M-A-P, Alibaba, BUAA, CASIA, BUPT, Kuaishou, OPPO. Authors affiliated with companies whose models are evaluated (e.g., Alibaba → Qwen models) are identifiable."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. However, several authors are from Alibaba (which produces Qwen models, heavily featured in the evaluation), making funder independence questionable even if funding were disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations are present in the paper. Several authors are employed by companies (Alibaba, Kuaishou, OPPO) that produce or use the evaluated models."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 38 pre-trained models on the benchmark but does not state any training data cutoff dates for the models. This is relevant because CodeForces and MBPP problems may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether benchmark problems appeared in model training data is provided. The paper uses MBPP (published 2021) and CodeForces problems, which could be in the training data of many tested models. LiveCodeBench is referenced for contamination-free evaluation but this property is not verified for the full CodeCriticBench."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP and CodeForces problems have been publicly available for years and are likely in training data of many evaluated models. The paper acknowledges LiveCodeBench is 'contamination free' in the reference but does not address contamination risk for its own benchmark, which also includes MBPP and CodeForces data. DeepSeek-v3 rewriting may partially mitigate this but the extent is not analyzed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human annotation (20 volunteers for Code QA correctness and 400 instances for ranking comparison) is part of benchmark construction and a small validation study, not a formal human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The 20 volunteers assessed code correctness and 8 models were ranked — this is benchmark construction work rather than a human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The volunteer annotation and human evaluation ranking are part of benchmark construction, not a human subjects study. Demographics are not structurally required."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study was conducted. The volunteers participated in annotation tasks as part of benchmark construction."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants requiring randomization. The human involvement is limited to annotation tasks."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants requiring blinding. Annotation tasks do not structurally require blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study was conducted, so attrition reporting is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API expenses, tokens consumed, or wall-clock time are reported for running 38 models on the 4,300-sample benchmark. The evaluation likely incurred substantial API costs for closed-source models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, total API spend, or hardware specifications are mentioned for running the benchmark evaluation or dataset construction."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeCriticBench is the first holistic code critique benchmark that covers both code generation and code QA tasks with basic and advanced evaluations.",
       "evidence": "Table 2 compares CodeCriticBench against CriticBench, CriticEval, JudgeBench, and RealCritic, showing it is the only benchmark with both Code Gen (3,200 samples), Code QA (1,100 samples), basic and advanced evaluation settings.",
       "supported": "moderate"
     },
     {
       "claim": "Performance on CodeCriticBench generally improves with model scale, supporting scaling laws.",
       "evidence": "Figure 3 shows ACC trending upward with parameter count across model families. Table 3 shows o1-like models (largest effective compute) achieving the highest ACC scores (72-75%).",
       "supported": "moderate"
     },
     {
       "claim": "Advanced critique evaluation rankings align more closely with human evaluation than basic critique evaluation.",
       "evidence": "Figure 9 shows that model rankings from the advanced setting closely match human evaluation rankings across 8 models on CodeCritic_400, while basic evaluation rankings diverge.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought evaluation improves both ACC and MSE compared to direct evaluation.",
       "evidence": "Table 5 shows CoT improves ACC by 7.5-15.5% and MSE by 1.31-3.13 across four Qwen2.5-Coder models on CodeCritic_400.",
       "supported": "moderate"
     },
     {
       "claim": "o1-like models (DeepSeek-R1, OpenAI o1-Preview) underperform on Easy/Medium data but excel on Hard data due to their architecture.",
       "evidence": "Figure 5 shows DeepSeek-R1 at 92.21% on Easy (below others at 93-98%) but 51.97% on Hard (vs. ~27-34% for most models). The paper hypothesizes this is 'partly due to o1-like architecture' but provides no mechanistic evidence.",
       "supported": "weak"
     },
     {
       "claim": "Model-generated critiques improve answer quality, and critique quality increases with model size.",
       "evidence": "Figure 11 shows fine-grained scores improving from 5.72 (original) to 6.76-6.91 after applying critiques from Qwen2.5-Coder models of increasing size (1.5B to 14B), evaluated by GPT-4o on CodeCritic_400.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeCriticBench introduces a 4,300-sample benchmark spanning code generation and code QA with basic (correctness) and advanced (fine-grained scoring) evaluation modes. Evaluation of 38 LLMs shows performance scales with model size, with o1-like reasoning models (DeepSeek-R1, OpenAI o1-Preview) achieving the highest accuracy (72-75% ACC) especially on hard problems. The advanced fine-grained evaluation correlates better with human judgments than binary correctness evaluation, and chain-of-thought prompting improves evaluation accuracy by 7.5-15.5% over direct prompting.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across 38 models are single-run point estimates with no error bars, confidence intervals, standard deviations, or repeated trials. For LLM evaluations where outputs can vary with sampling, this is a significant omission that makes it impossible to assess whether observed differences are meaningful."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 38 models. These settings substantially affect LLM output quality and can change the results entirely."
     },
     {
       "flag": "Potential conflicts of interest",
       "detail": "Authors from Alibaba (which produces Qwen/Qwen2.5-Coder models that are heavily featured and among the best-performing open-source models in the evaluation) and other companies do not disclose any conflicts of interest. Qwen models are used in both dataset construction (Qwen2.5-72B for QA generation, Qwen2.5-Coder for responses) and evaluation."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MBPP (published 2021) and CodeForces problems are included in the benchmark. Many of the evaluated models were trained well after these datasets became public, creating a serious contamination risk that is not discussed or analyzed."
     },
     {
       "flag": "Circular dataset construction",
       "detail": "LLMs (DeepSeek-v3, Qwen2.5-Coder, GPT-4o, Claude3.5-Sonnet) were used to construct the dataset (rewrite problems, generate QA pairs, generate evaluation checklists, assign scores), and then the same or similar models are evaluated on the dataset. This creates potential circularity where models may perform better on data they helped create."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "No specific API versions or snapshot dates are provided for any of the closed-source models (Claude3.5-Sonnet, GPT-4o, DeepSeek-v3, etc.), making results unreproducible as model behavior changes across versions."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational code generation benchmark used in the CodeCriticBench comparison."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "relevance": "Source of contamination-free code problems used in CodeCriticBench's code generation subset."
     },
     {
       "title": "CriticBench: Benchmarking LLMs for Critique-Correct Reasoning",
       "authors": ["Zicheng Lin"],
       "year": 2024,
       "relevance": "Prior critique benchmark that CodeCriticBench directly compares against and aims to improve upon."
     },
     {
       "title": "CriticEval: Evaluating Large-Scale Language Model as Critic",
       "authors": ["Tian Lan"],
       "year": 2024,
       "relevance": "Large-scale critique evaluation benchmark compared against in Table 2."
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges",
       "authors": ["Sijun Tan"],
       "year": 2024,
       "relevance": "Benchmark for LLM-as-judge evaluation, directly compared in Table 2."
     },
     {
       "title": "RealCritic: Towards Effectiveness-Driven Evaluation of Language Model Critiques",
       "authors": ["Zhengyang Tang"],
       "year": 2025,
       "arxiv_id": "2501.14492",
       "relevance": "Recent critique benchmark that CodeCriticBench positions itself against."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Foundational work on LLM self-critique and iterative refinement, motivating critique benchmark development."
     },
     {
       "title": "LLM Critics Help Catch LLM Bugs",
       "authors": ["Nat McAleese"],
       "year": 2024,
       "arxiv_id": "2407.00215",
       "relevance": "Demonstrates LLM critique capacity for finding bugs, directly related to CodeCriticBench's bug detection evaluation."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for the Qwen2.5-Coder model family heavily featured in CodeCriticBench evaluations and dataset construction."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Technical report for DeepSeek-Coder models evaluated in the benchmark."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP, a code generation benchmark used as a data source for CodeCriticBench."
     },
     {
       "title": "FullStack Bench: Evaluating LLMs as Full Stack Coder",
       "authors": ["Siyao Liu"],
       "year": 2024,
       "arxiv_id": "2412.00535",
       "relevance": "Full-stack code evaluation benchmark with category definitions adopted for CodeCriticBench's Code QA classification."
     }
   ]
 }