ai-research-survey

scan.json (25679B)

---

 {
   "paper": {
     "title": "CodeJudgeBench: Benchmarking LLM-as-a-Judge for Coding Tasks",
     "authors": [
       "Hongchao Jiang",
       "Yiming Chen",
       "Yushi Cao",
       "Hung-yi Lee",
       "Robby T. Tan"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.10535"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/hongcha0/CodeJudgeBench (Section 1, after the abstract)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "A HuggingFace dataset is provided: https://huggingface.co/datasets/mattymchen/codejudgebench (Section 1, after the abstract)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment setup instructions, requirements.txt, Dockerfile, or dependency specifications are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code and data links are given, no README, reproduction guide, or scripts for replicating the main experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., accuracy percentages in Tables 3, 4, 5, 7) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., 'thinking models significantly outperform non-thinking models') but uses no statistical significance tests. Comparisons are based solely on comparing raw accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (e.g., Cohen's d, odds ratios) are reported. While raw accuracy differences are visible in tables, there is no contextual framing of effect magnitudes beyond comparing numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 5,352 pairs (Table 1) but no justification is given for why this size is sufficient for the claims being made. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Each model appears to have been evaluated once per configuration with single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 26 LLM-as-a-Judge models including both specialized judge models (AceCodeRM, Prometheus, Self-Taught, Skywork-Critic) and general-purpose models, providing baseline comparisons (Table 2, Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include very recent models: Claude 4, Gemini 2.5, Qwen3, DeepSeek R1 (2025), RM-R1 (2025), and other current state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 provides ablation-style analysis: point-wise vs. pair-wise evaluation (Table 4), candidate pre-processing strategies (Table 5: raw response vs. full code vs. no comments), and response ordering effects (Section 5.2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only accuracy as its evaluation metric throughout all experiments. No additional metrics (e.g., F1, AUC, Cohen's kappa for agreement) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system outputs is included. All evaluation is automated via unit test pass/fail verification. Given that the paper studies LLM judgment quality, human agreement with LLM judges would have been informative."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no explicit separation of development and test splits. All 5,352 pairs are used for evaluation with no held-out set mentioned. The difficulty categorization (easy/medium/hard) is determined post-hoc based on model performance."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (CodeGen, CodeRepair, TestGen), by difficulty level (easy, medium, hard), by response source model, and by response ordering (position A vs. B). See Tables 3, 5, and Figures 4-5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses failure modes including position bias and model-specific variance. The paper shows models that perform poorly and discusses why (e.g., RM-R1 recency bias, non-thinking models approaching random baseline)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: fine-tuning thinking models for LLM-as-a-Judge does not improve performance (Section 5.1), point-wise evaluation significantly underperforms pair-wise (Section 5.3, Table 4), removing comments hurts performance (Table 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that thinking models outperform non-thinking models (supported by Table 3), small thinking models outperform large judge-tuned models (Qwen3-8B vs. Prometheus-14B/Self-Taught-70B in Table 3), position sensitivity exists (supported by Figure 4), and pair-wise outperforms point-wise (supported by Table 4). All claims are supported by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims such as 'thinking models allocate more tokens for code analysis, which enhances their ability to understand and accurately judge code responses' (Section 5.1) and attributes fine-tuning failures to 'insufficient code-related training data.' These causal explanations are speculative without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title claims to benchmark 'LLM-as-a-Judge for Coding Tasks' broadly, but results are limited to competitive programming problems from LiveCodeBench (LeetCode, AtCoder, CodeForces). Real-world software engineering tasks, different programming languages beyond Python, and non-algorithmic coding are not tested. The scope is not explicitly bounded to competitive programming."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. For example, the paper does not consider whether thinking models' advantage comes from longer outputs rather than reasoning quality, whether position bias results from training data artifacts, or whether the benchmark's difficulty is an artifact of the response generation process."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Most models lack specific version identifiers. The paper uses 'Claude-3.7-Sonnet', 'Claude-4-Sonnet', 'Gemini-2.5-Pro', 'Gemini-2.5-Flash' without API version snapshots or dates. For example, 'Claude 3.7' with an access date of 2025-5-15 and 'Claude 4' accessed 2025-5-25 are in references, but no API snapshot identifiers are provided. Marketing names without snapshot dates do not count per the schema."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 states 'we use the pair-wise prompt from [39]' and 'we follow the prompts and sampling parameters provided in their official implementations' for judge-tuned LLMs, but the actual prompt text is not provided in the paper or appendix. The reader would need to chase multiple external references to reconstruct prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the evaluated models. Section 4.3 mentions following 'sampling parameters provided in their official implementations' for judge-tuned models but does not state what these are. General LLM settings are not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates LLMs as judges via direct prompting without multi-step workflows, tool use, or agent loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.2-3.4 describe the data construction pipeline in detail: response collection from specific models, verification via unit tests (pass all = good, fail any = bad), and pairing strategy (random selection of one good and one bad). Filtering criteria (discard problems with all-correct or all-incorrect responses) are stated. Table 1 provides counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion briefly mentions 'Future work will focus on expanding CodeJudgeBench' but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no discussion of potential issues such as benchmark construction bias, the narrow scope of competitive programming problems, or the limitation of binary correctness as ground truth."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge that results are limited to competitive programming, to Python, to execution-free judging, or that the benchmark may not represent real-world coding evaluation scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace (https://huggingface.co/datasets/mattymchen/codejudgebench), allowing independent verification of the benchmark data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.5 describe the data collection process: problems sourced from LiveCodeBench-v6 (1,055 problems from May 2023 to April 2025), responses generated by specific LLMs (Claude-3.7-Sonnet, Gemini-2.5-Flash, Gemini-2.5-Pro, etc.), verification via unit tests."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is a standard benchmark (LiveCodeBench) with LLM-generated responses."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Sections 3.2-3.4 and Figure 3: response collection from LLMs, verification via unit tests, and pairing of good/bad responses. Table 1 provides counts at each stage per source model and task."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: ASUS Intelligent Cloud Services (AICS) and National Taiwan University. However, ASUS is a technology company, and no conflict-of-interest statement addresses this."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The primary affiliation is ASUS Intelligent Cloud Services, a commercial entity that could have interest in LLM evaluation outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 26 LLMs as judges on benchmark data but does not state training data cutoff dates for any of the models. LiveCodeBench-v6 uses problems from May 2023 to April 2025, but it is unclear which models may have been trained on earlier portions of this data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states they use LiveCodeBench which 'mitigates data contamination by continually collecting new problems from platforms such as LeetCode, AtCoder, and CodeForces.' This acknowledges the contamination concern and relies on LiveCodeBench's temporal design, though no direct analysis of overlap is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly chooses LiveCodeBench to mitigate contamination risk (Section 3.1): it uses problems published between May 2023 and April 2025, which are continually updated to stay ahead of model training cutoffs. This is a reasonable mitigation strategy, though individual model cutoffs are not verified."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite evaluating 26 models across 5,352+ data points with multiple configurations. The cost of running the benchmark is not discussed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware specifications are stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Thinking models significantly outperform non-thinking models on coding judgment tasks.",
       "evidence": "Table 3 shows thinking models (Claude-4-Sonnet 79.93%, Gemini-2.5-Pro 82.12%, QwQ 76.40%) substantially outperform non-thinking models (Claude-3.5-Sonnet 59.12%, Gemini-2.0-Flash 54.80%) on average accuracy across all tasks.",
       "supported": "strong"
     },
     {
       "claim": "Small thinking models (e.g., Qwen3-8B) can outperform specially trained LLM-as-a-Judge models up to 70B in size.",
       "evidence": "Table 3 shows Qwen3-8B (65.60% avg) outperforms Prometheus-14B (57.04%), Self-Taught-70B (56.36%), and Skywork-Critic-70B (55.98%).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning thinking models specifically for LLM-as-a-Judge tasks does not yield improved performance.",
       "evidence": "Table 3 shows RM-R1-32B (62.60%) underperforms general-purpose Qwen3-32B (72.42%) and QwQ (76.40%), despite RM-R1 being specifically trained for judge tasks (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "Response ordering significantly impacts LLM-as-a-Judge accuracy, with discrepancies reaching up to 14%.",
       "evidence": "Figure 4 shows accuracy differences between correct-at-position-A vs. correct-at-position-B configurations, with some models showing substantial gaps (e.g., RM-R1-32B, Claude-3.7) across all three tasks.",
       "supported": "strong"
     },
     {
       "claim": "Pair-wise evaluation outperforms point-wise evaluation for coding judgment tasks.",
       "evidence": "Table 4 shows point-wise evaluation produces ~50% ties for most models on CodeGen, significantly reducing effective accuracy compared to pair-wise results in Table 3 (Section 5.3).",
       "supported": "strong"
     },
     {
       "claim": "Providing the full raw model response (including reasoning/comments) leads to better judge performance than code-only input.",
       "evidence": "Table 5 shows 'raw response' (RR) averaging 71.43% vs. 'full code' (FC) at 70.31% vs. 'no comments' (NC) at 69.27% overall. The difference is modest but consistent.",
       "supported": "moderate"
     },
     {
       "claim": "CodeJudgeBench is more challenging than existing benchmarks like JudgeBench and RM-Bench.",
       "evidence": "Figure 1 shows frontier models achieve 93-98% on JudgeBench and 78-87% on RM-Bench code splits, but only 66-82% on CodeJudgeBench CodeGen task.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeJudgeBench evaluates 26 LLM-as-a-Judge models across code generation, code repair, and unit test generation using 5,352 curated pairs from LiveCodeBench. Thinking models (e.g., Claude-4-Sonnet at 79.93%, Gemini-2.5-Pro at 82.12%) dramatically outperform non-thinking models (most below 60% accuracy), and even small 8B thinking models surpass large 70B judge-specialized models. All models exhibit significant position bias in pairwise evaluation, with accuracy swings up to 14% when response order is swapped. Pair-wise evaluation substantially outperforms point-wise evaluation due to frequent ties in scalar scoring.",
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All comparisons are based on single-run point estimates with no confidence intervals, significance tests, or variance reporting. Claims of 'significant' differences are based solely on comparing raw accuracy numbers, making it impossible to distinguish genuine performance differences from noise."
     },
     {
       "flag": "Single metric (accuracy only)",
       "detail": "The entire evaluation uses only accuracy. No additional metrics (e.g., agreement with human judges, Cohen's kappa, calibration measures) are reported, limiting understanding of judge quality beyond binary correctness."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity discussion, which is a significant omission for a benchmark paper. There is no acknowledgment that competitive programming problems may not represent real-world coding tasks."
     },
     {
       "flag": "Overly broad claims from narrow scope",
       "detail": "The paper claims to benchmark 'LLM-as-a-Judge for Coding Tasks' broadly, but all problems come from competitive programming (LiveCodeBench). Real-world software engineering, multi-file projects, different programming languages, and non-algorithmic coding are not represented."
     },
     {
       "flag": "Prompts and hyperparameters not provided",
       "detail": "The actual prompts used for evaluation are not included in the paper, and no temperature/sampling parameters are reported. Readers cannot fully reproduce the experiments without chasing multiple external references."
     },
     {
       "flag": "Corporate affiliation without conflict disclosure",
       "detail": "The primary affiliation is ASUS Intelligent Cloud Services (a commercial entity), yet no funding or competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-judge paradigm demonstrating high correlation between GPT-4 judgments and human evaluations."
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges",
       "authors": ["Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery"],
       "year": 2025,
       "relevance": "Prior LLM-as-a-Judge benchmark that CodeJudgeBench directly compares against and claims to surpass in difficulty."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2025,
       "relevance": "Source of coding problems used to construct CodeJudgeBench; uses temporal approach to mitigate data contamination."
     },
     {
       "title": "RM-Bench: Benchmarking Reward Models of Language Models with Subtlety and Style",
       "authors": ["Yantao Liu", "Zijun Yao", "Rui Min"],
       "year": 2025,
       "relevance": "Reward model benchmark that includes a coding split; CodeJudgeBench shows models score much lower on its tasks."
     },
     {
       "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models",
       "authors": ["Seungone Kim", "Juyoung Suk", "Shayne Longpre"],
       "year": 2024,
       "relevance": "Open-source LLM judge evaluated in CodeJudgeBench; demonstrates that specialized judge training does not transfer well to coding tasks."
     },
     {
       "title": "Self-Taught Evaluators",
       "authors": ["Tianlu Wang", "Ilia Kulikov", "Olga Golovneva"],
       "year": 2024,
       "arxiv_id": "2408.02666",
       "relevance": "LLM-as-a-Judge trained iteratively on synthetic data; evaluated in CodeJudgeBench where it underperforms smaller thinking models."
     },
     {
       "title": "AceCoder: Acing Coder RL via Automated Test-Case Synthesis",
       "authors": ["Huaye Zeng", "Dongfu Jiang", "Haozhe Wang"],
       "year": 2025,
       "relevance": "Develops AceCodeRM, a specialized code judge model evaluated in CodeJudgeBench that underperforms general thinking models."
     },
     {
       "title": "RM-R1: Reward Modeling as Reasoning",
       "authors": ["Xiusi Chen", "Gaotang Li", "Ziqi Wang"],
       "year": 2025,
       "arxiv_id": "2505.02387",
       "relevance": "Thinking LLM-as-a-Judge model using chain-of-rubrics; evaluated extensively in CodeJudgeBench showing mixed results."
     },
     {
       "title": "From Code to Courtroom: LLMs as the New Software Judges",
       "authors": ["Junda He", "Jieke Shi", "Terry Yue Zhuo"],
       "year": 2025,
       "relevance": "Survey of LLM-as-a-Judge for coding tasks, providing taxonomy of judging criteria including code functionality and code quality."
     },
     {
       "title": "A Survey on LLM-as-a-Judge",
       "authors": ["Jiawei Gu", "Xuhui Jiang", "Zhichao Shi"],
       "year": 2024,
       "arxiv_id": "2411.15594",
       "relevance": "Comprehensive survey of the LLM-as-a-Judge paradigm that contextualizes the need for domain-specific benchmarks like CodeJudgeBench."
     },
     {
       "title": "CodeMonkeys: Scaling Test-Time Compute for Software Engineering",
       "authors": ["Ryan Ehrlich", "Bradley Brown", "Jordan Juravsky"],
       "year": 2025,
       "relevance": "Demonstrates inference-time scaling for code, related to CodeJudgeBench's Best-of-N evaluation using judge models."
     },
     {
       "title": "Scoring Verifiers: Evaluating Synthetic Verification for Code and Reasoning",
       "authors": ["Aleksander Ficek", "Somshubra Majumdar"],
       "year": 2025,
       "relevance": "Code verification benchmark using ranking evaluation; one of the coding judge benchmarks that CodeJudgeBench extends."
     }
   ]
 }