ai-research-survey

scan.json (24169B)

---

 {
   "paper": {
     "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
     "authors": ["Weixi Tong", "Tianyi Zhang"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.02184"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and datasets are available on GitHub https://github.com/VichyTong/CodeJudge' in both the abstract and Section 5 (Conclusion)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (HumanEval-X, CoNaLa, APPS, BigCodeBench) and states code and datasets are available at their GitHub repository. The datasets themselves are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'eight A100-80GB GPUs' for local models and the GPT-3.5-Turbo-1106 API, but does not provide a requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the code is released on GitHub, the paper itself does not include step-by-step reproduction instructions, a 'Reproducing Results' section, or specific commands to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across three runs for open-source models in the appendix tables (Tables 11-15), e.g., '0.515±0.04'. However, GPT-3.5 experiments were only run once (temperature 0), so no variance is reported for those."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CODEJUDGE 'significantly outperformed existing methods' but does not report any statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on comparing correlation coefficient numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'CODEJUDGE achieves significantly higher correlations (12.1%-41.8%) than existing methods' and provides full numeric baselines in Tables 3-6 with absolute values to compute differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 100 competition-level tasks were sampled from APPS, or why the specific dataset sizes were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for open-source model experiments across three runs in the appendix tables (e.g., Table 11: '0.625±0.00'). However, GPT-3.5 experiments were only run once due to cost, so no variance is available for those results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against nine existing methods: six token-based (BLEU, ROUGE-L, METEOR, chrF, CodeBLEU, RUBY), one embedding-based (CodeBERTScore), one LLM-based (ICE-Score), and a VANILLA baseline (Section 4.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ICE-Score (Zhuo, 2024) is the state-of-the-art LLM-based code evaluation method. CodeBERTScore (Zhou et al., 2023) and BigCodeBench (Zhuo et al., 2024) are also recent. The baselines represent the field well."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates multiple components: with vs. without reference code, different prompt designs (CoT, few-shot, combinations thereof in Appendix A / Table 7), and compares the two CODEJUDGE methods (Analyze-then-Summarize vs. Taxonomy-Guided Fault Localization)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Kendall's tau coefficient, Spearman's rank correlation, and accuracy as evaluation metrics (Section 4.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include human evaluation of CODEJUDGE's outputs. It uses existing test case results and human-annotated usefulness scores from CoNaLa as ground truth, but no humans independently evaluate the quality of CODEJUDGE's judgments beyond automated metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The penalty score settings for the fault localization method were tuned on a small validation set of 32 tasks (20% of HumanEval, Appendix B), with results reported on the remaining tasks and separate datasets (CoNaLa, APPS, BigCodeBench)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Table 5), by dataset (Tables 3-4), and by LLM evaluator (Table 6). The appendix provides per-language results for all models (Tables 11-15)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 includes a failure case analysis of 600 cases where CODEJUDGE made wrong predictions, identifying three failure patterns: wrong analysis of code logic (52.83%), wrong identification of task requirements (26.42%), and requirements of error handling (20.75%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CoT and few-shot learning methods do not improve CODEJUDGE's performance (Table 7, Appendix A), that providing reference code hurts performance on CoNaLa, and that all methods suffer significant performance drops on APPS and BigCodeBench."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CODEJUDGE 'significantly outperformed existing methods in most settings' and 'achieved better results even when using a much smaller model, Llama-3-8B-Instruct.' Both are supported by Tables 3 and 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that 'slow thinking' prompting improves evaluation quality. This is supported by controlled comparisons: CODEJUDGE vs. VANILLA (which uses the same LLMs without the structured reasoning pipeline), constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across five programming languages, four datasets, and four LLMs. Claims are generally scoped to the tested settings. The paper notes performance drops on challenging benchmarks and acknowledges language-specific variation (Table 5)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the improvement over ICE-Score might be due to longer prompts giving the model more tokens to reason over (rather than the specific 'slow thinking' structure), or whether the taxonomy design biases results toward certain error types."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-3.5-Turbo-1106' (Section 4.4), 'CodeLlama-Instruct (34B)', 'Llama-3-Instruct (8B)', and 'Llama-3-Instruct (70B)'. The GPT version includes a snapshot date. Open-source model sizes are specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for all methods are provided in the appendix: VANILLA prompts (Tables 16-17), CODEJUDGE Analyze-then-Summarize prompt (Table 18), and Fault Localization prompt (Table 19). These include the actual text used with placeholders clearly marked."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 states: 'we set the temperature to 0 and top_p to 1' for GPT-3.5, and 'temperature to 0.4 and top_p to 0.9' for open-source models. Penalty weights for fault localization are specified in Appendix B."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-step scaffolding (analysis then summarization) is described in detail in Sections 3.1 and 3.2, with a full workflow diagram in Figure 2. The taxonomy of inconsistencies is provided in Table 2, and the scoring formula is given in Appendix B."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes data sources: official dataset versions are used, code generation for HumanEval-X and APPS uses MultiPL-E, BigCodeBench uses official pre-generated samples. For APPS, 100 competition-level tasks were randomly sampled. CoNaLa human annotations are from Evtikhiev et al. (2023)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and discusses that LLMs can generate incorrect judgments on complex cases like APPS, and that the framework's quality depends on the backbone model."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is brief (one paragraph) and only mentions that 'LLMs can generate incorrect judgments or fail to completely follow system prompts when evaluating challenging and complex cases such as the APPS benchmark.' This is more of a known limitation than a threat to validity. No specific threats like construct validity of the correlation metrics, external validity across unseen languages/tasks, or internal validity concerns are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show or claim. It does not bound its findings to specific languages, task types, or model families, despite testing only on specific sets of each. The title and framing ('Evaluating Code Generation with Large Language Models') are broader than the tested settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (HumanEval-X, CoNaLa, APPS, BigCodeBench) and states that code and datasets are released at https://github.com/VichyTong/CodeJudge, enabling verification of raw results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes each dataset's origin: HumanEval-X from CodeGeeX (164 tasks, 5 languages), CoNaLa from StackOverflow (472 tasks), APPS from code competitions (100 randomly sampled competition-level tasks), and BigCodeBench (1,140 tasks). Appendix C provides additional detail on code generation procedures."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study. The study uses existing benchmarks and model-generated code."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from datasets to evaluation is documented: datasets are sourced from official releases, code is generated using specified tools (MultiPL-E, official BigCodeBench samples), evaluation is performed using scipy correlation metrics, and postprocessing is described in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Weixi Tong at Huazhong University of Science and Technology, Tianyi Zhang at Purdue University. These are academic institutions without obvious conflicts."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funder cannot be assessed. The absence of a funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the four LLMs used (GPT-3.5-Turbo-1106, CodeLlama-34B, Llama-3-8B, Llama-3-70B). This matters because the LLM evaluators may have seen the benchmark tasks during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the LLM evaluators (GPT-3.5, Llama-3, CodeLlama) may have seen HumanEval, CoNaLa, or APPS problems during training. HumanEval was published in 2021 and is widely known — models trained after 2021 almost certainly encountered it."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, CoNaLa in 2018, and APPS in 2021 — all well before the training cutoffs of GPT-3.5 and Llama-3. The LLM evaluator could have memorized these problems and their solutions, which would inflate its ability to judge code correctness. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates automated methods using existing benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 (Appendix D) reports average execution times per evaluation for all four models (e.g., GPT-3.5-Turbo: 2.36s for A.S., 1.14s for F.L.). The paper notes costs were high enough that GPT-3.5 experiments were only run once."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using eight A100-80GB GPUs for local models but does not state total GPU hours, API spend, or overall computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CODEJUDGE achieves significantly higher correlations (12.1%-41.8%) than existing methods in most settings when using GPT-3.5-Turbo.",
       "evidence": "Table 3 shows CODEJUDGE achieves 0.612 Kendall's tau on HumanEval-X vs. 0.475 for ICE-Score (the best existing LLM-based method), and similar advantages on other datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Even with a smaller model (Llama-3-8B-Instruct), CODEJUDGE outperforms ICE-Score which uses GPT-3.5-Turbo.",
       "evidence": "Table 6 shows Llama-3-8B CODEJUDGE achieves 0.523 tau on CoNaLa and 0.480 on HumanEval-X, vs. ICE-Score with GPT-3.5 at 0.253/0.475 on the same datasets.",
       "supported": "moderate"
     },
     {
       "claim": "CODEJUDGE without reference code outperforms all existing methods that use reference code in most settings.",
       "evidence": "Table 3: CODEJUDGE w/o REF achieves 0.502 tau on HumanEval-X vs. 0.372 for CodeBERTScoreF3, the best reference-based method. On APPS and BigCodeBench, the advantage is smaller or absent.",
       "supported": "moderate"
     },
     {
       "claim": "CODEJUDGE achieves 80.56% average accuracy on binary code correctness prediction.",
       "evidence": "Table 4 shows average accuracies of 80.56% on HumanEval-X, 68.33% on APPS, and 74.56% on BigCodeBench using GPT-3.5-Turbo.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought and few-shot prompting do not improve CODEJUDGE's performance.",
       "evidence": "Table 7 (Appendix A) shows that all CoT and few-shot variants achieve lower accuracy than the base CODEJUDGE method (81.63% base vs. 77.65% CoT, 78.22% CoT+few-shot, etc.).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeJudge is an LLM-based code evaluation framework that uses 'slow thinking' prompting strategies to assess code correctness without test cases. It achieves significantly higher statistical correlations with ground truth than nine existing methods across four datasets and five programming languages. The framework works with multiple LLMs and can operate without reference code while still outperforming reference-dependent methods. Performance degrades substantially on competition-level problems (APPS), and failure analysis reveals that wrong analysis of code logic accounts for 52.83% of errors.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims CODEJUDGE 'significantly outperformed existing methods' but provides no statistical significance tests. Differences are based solely on comparing correlation coefficients without testing whether differences are statistically significant."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "HumanEval (2021), CoNaLa (2018), and APPS (2021) were all published before the training cutoffs of the LLM evaluators used. The LLMs may have memorized these problems and their solutions, artificially inflating their ability to judge code correctness. This is never discussed."
     },
     {
       "flag": "Single run for main results",
       "detail": "The main GPT-3.5-Turbo results (Tables 3-5) are based on a single run due to cost, with temperature 0. While this reduces randomness, it means no variance can be reported for the primary model, and any API non-determinism at temperature 0 is not accounted for."
     },
     {
       "flag": "Validation set used for hyperparameter tuning from same benchmark",
       "detail": "The penalty weights for fault localization were tuned on 20% of HumanEval (32 tasks, Appendix B), but HumanEval-X results are still reported. While separate tasks may have been used, this creates a risk of indirect data leakage within the same benchmark distribution."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and pass@k, foundational benchmark and metric for LLM code generation evaluation."
     },
     {
       "title": "ICE-score: Instructing large language models to evaluate code",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "relevance": "Primary baseline; the state-of-the-art LLM-based code evaluation method that CODEJUDGE improves upon."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Introduces EvalPlus for augmenting test cases in code generation evaluation, addressing test suite insufficiency."
     },
     {
       "title": "Out of the BLEU: How should we assess quality of the code generation models?",
       "authors": ["Mikhail Evtikhiev", "Egor Bogomolov"],
       "year": 2023,
       "relevance": "Shows statistically significant disagreement between human judges and token-based metrics for code evaluation."
     },
     {
       "title": "CodeBERTScore: Evaluating code generation with pretrained models of code",
       "authors": ["Shuyan Zhou", "Uri Alon"],
       "year": 2023,
       "relevance": "Embedding-based code evaluation metric using CodeBERT, serving as a baseline in this study."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-bench and chatbot arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "relevance": "Foundational work on using LLMs as evaluators, directly motivating the LLM-as-code-judge approach."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Recent code generation benchmark with complex multi-API tasks, used as an evaluation dataset."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Competition-level code generation benchmark used to test CODEJUDGE on challenging problems."
     },
     {
       "title": "G-eval: NLG evaluation using GPT-4 with better human alignment",
       "authors": ["Yang Liu"],
       "year": 2023,
       "relevance": "LLM-based evaluation framework for text generation that inspired LLM-as-judge approaches for code."
     },
     {
       "title": "ChatEval: Towards better LLM-based evaluators through multi-agent debate",
       "authors": ["Chi-Min Chan", "Weize Chen"],
       "year": 2024,
       "relevance": "Multi-agent LLM evaluation approach relevant to understanding LLM-as-judge capabilities."
     },
     {
       "title": "CodeGeeX: A pre-trained model for code generation with multilingual benchmarking on HumanEval-X",
       "authors": ["Qinkai Zheng", "Xiao Xia"],
       "year": 2023,
       "relevance": "Introduces HumanEval-X multilingual benchmark, the primary evaluation dataset used in this study."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Chain-of-thought prompting technique tested as a prompt design variant for CODEJUDGE."
     }
   ]
 }