ai-research-survey

scan.json (26527B)

---

 {
   "paper": {
     "title": "CodeMMLU: A Multi-Task Benchmark for Assessing Code Understanding & Reasoning Capabilities of CodeLLMs",
     "authors": [
       "Dung Nguyen Manh",
       "Thang Phan Chau",
       "Nam Le Hai",
       "Thong T. Doan",
       "Nam V. Nguyen",
       "Quang Pham",
       "Nghi D. Q. Bui"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2410.01999"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'CodeMMLU is publicly available at: CodeMMLU' and the benchmark is distributed under the MIT license (Appendix A.3). Appendix C provides HuggingFace model IDs for all evaluated models."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly released under the MIT license (Appendix A.3). The abstract and Section 1 state it is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup with library versions is provided in the paper or appendix."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts to replicate experiments are described in the paper. The paper describes the evaluation setup (Section 4.1) but does not provide executable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 3, 7, 9, 10, 11 report only point estimates (accuracy percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4o outperformed all models', 'DeepSeek R1 performs significantly worse than its base model') based solely on comparing accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides percentage improvements with baseline context throughout, e.g., 'DeepSeek-Coder-33b surpasses its base model by approximately 29%' (Section 4.2), and Table 4 shows absolute and relative performance changes for each answer position. The Pearson correlation r=0.61 is reported for knowledge vs. real-world task alignment (Section 4.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of questions per category (e.g., why 163 code completion, 76 code repair) or for the sample of 100 instances used in manual verification (Section 3.3). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation (STD) is reported in Table 8 for selection bias experiments across answer orderings. However, across-run variance for the main evaluation results is not reported, though MCQ benchmarks with deterministic answer extraction are typically single-run by design."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares CodeMMLU performance against results on established benchmarks including HumanEval, MBPP, MMLU, and GSM8k (Table 7). Multiple model families serve as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes contemporary state-of-the-art models: GPT-4o, GPT o3-mini, Claude 3.7 Sonnet, DeepSeek R1/V3, Llama 3.3-70B, Phi-4, and QwenCoder 2.5 (Table 3, Figure 1). These are current as of early 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides ablation-like analyses: comparing zero-shot vs few-shot vs CoT prompting strategies (Figure 3, Appendix B.2), examining the effect of answer order permutations (Tables 4, 8), and comparing base vs instruction-tuned models within families."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy as the primary metric but supplements it with Pearson correlation (r=0.61 for knowledge vs. real-world task correlation), perplexity scores (Table 5), n-gram accuracy for contamination analysis (Table 6), and standard deviation for selection bias analysis (Table 8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the benchmark quality or model outputs is performed. The manual verification of 100 instances per subject (Section 3.3) is for dataset quality assurance during construction, not evaluation of model outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper that evaluates pre-existing models on a new test set. There is no model training or tuning involved, so the train/test split concept does not apply in the traditional sense."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: Tables 9-11 show results for each of 9 task categories (API & Frameworks, PL Syntax, Software Principles, DBMS & SQL, Others, Code Completion, Fill in the Blank, Code Repair, Execution Prediction) across all models and prompt settings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Appendix B.2 discuss failure cases in detail, including CoT failures with qualitative examples (Figures 11, 12, 14). The paper shows specific examples where reasoning models fail due to over-reasoning."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: CoT prompting consistently hurts performance (Section 4.2, Figure 3), reasoning models like DeepSeek R1 underperform their base models (Table 3), and the discrepancy between HumanEval and MCQ format performance (Section 4.2, Table 4, Figure 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the benchmark has ~20,000 questions (Table 2 confirms 19,912), spans diverse domains and programming languages (Table 2 confirms 52 topics, 10+ languages), and that state-of-the-art models struggle (Table 3 shows GPT-4o at 56.40% overall). All claims are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the additional complexity introduced by step-by-step reasoning does not align well with knowledge-seeking tasks' (Section 4.2) and that instruction tuning 'substantially' improves performance. These are inferred from observational comparisons without controlled experiments isolating the causal mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to assess 'Code Understanding & Reasoning Capabilities' broadly, but the benchmark is limited to multiple-choice format, which is a narrow proxy for understanding. The paper does not sufficiently bound its claims to the MCQ evaluation format. The conclusion states CodeMMLU evaluates 'a wide range of software knowledge and real-world programming tasks' without qualifying the MCQ limitation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for key findings: the CoT performance decline might be due to 'overreasoning' rather than lack of capability (Section 4.2), MCQ bias could explain some performance variation rather than true understanding differences (Section 4.2, Tables 4, 8), and data leakage could explain open-ended benchmark performance (Section A.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C, Table 12 provides specific model identifiers including API version dates for proprietary models (e.g., 'GPT-4o-2024-05-13', 'Claude-3.5-sonnet-20241022') and HuggingFace model IDs for open-source models (e.g., 'meta-llama/Meta-Llama-3.1-70B-Instruct')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix C.1 for all evaluation configurations: zero-shot, few-shot, and Chain-of-Thought prompts for each task type (knowledge MCQ, code completion, fill-in-the-blank, code repair, defect detection). Data creation prompts for distractor generation are also provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the 43 models evaluated. Section 4.1 describes the answer extraction method but not the generation parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct prompting of LLMs with MCQ questions and extracting answers via regex."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and Appendix A.1 document the data preprocessing pipeline in detail: rule-based filtering (removing HTML, images, links), LLM-based filtering (three criteria with threshold of 4, removing ~25.6% of raw data), MinHash LSH deduplication (256 permutations, 0.8 threshold), and execution-based filtering for fundamental tasks. Figure 2 provides a visual overview."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The paper acknowledges the difficulty of eliminating data leakage (Appendix A.2) but does not have a structured discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the representativeness of MCQ format for measuring understanding, the quality of LLM-generated distractors, or the sensitivity of results to prompt format."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the MCQ evaluation format or acknowledge that MCQ performance may not generalize to actual code understanding in real-world settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly released under the MIT license (Appendix A.3), which includes the questions, answer choices, and ground truth labels, allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: knowledge-based questions from W3Schools, GeeksforGeeks, and Common Crawl (Section 3.1); fundamental tasks derived from HumanEval, QuixBugs, LeetCode, and IBM CodeNet (Section 3.2). Specific Common Crawl portions (CC-MAIN-2021-41 to CC-MAIN-2024-46) are identified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are publicly available benchmark datasets and web resources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a visual overview of the data creation pipeline. Section 3.3 and Appendix A.1 describe each filtering step with quantitative results (e.g., removing ~25.6% via LLM filtering). The process from raw collection through filtering to final benchmark is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. The authors are affiliated with FPT Software AI Center (a commercial entity) but funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: FPT Software AI Center, Hanoi University of Science and Technology, and Independent Researcher. These are prominently displayed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided. Since the authors are primarily from FPT Software AI Center (a commercial software company), and the paper evaluates models including those from companies FPT may partner with or compete with, the funding independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 43 models evaluated. This is relevant because the knowledge-based questions are sourced from publicly available web content that could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 directly addresses data contamination. The paper uses perplexity analysis (Table 5) and n-gram accuracy analysis (Table 6) across multiple model families (Mistral, DeepSeek, Llama) to assess potential data leakage, showing CodeMMLU has higher perplexity (less leakage) than comparable benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 addresses contamination risk using the methodology from Xu et al. (2024). Tables 5-6 compare CodeMMLU's contamination metrics (perplexity and 5-gram accuracy) against CodeScope and CodeApex, showing significantly lower contamination risk for CodeMMLU. The paper also notes that MCQ format transformation and synthetic distractors mitigate leakage risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported for the evaluation of 43 models across ~20,000 questions in multiple prompt configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, hardware used, or compute budget are stated for either benchmark construction or model evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Even state-of-the-art models struggle with CodeMMLU, with GPT-4o achieving only 56.40% overall accuracy and the best model (GPT o3-mini) reaching 62.36%.",
       "evidence": "Table 3 shows GPT-4o at 56.40% and GPT o3-mini at 62.36% CodeMMLU accuracy. Figure 1 provides a radar chart visualization across all task categories.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought prompting consistently hurts performance on CodeMMLU compared to zero-shot and few-shot settings.",
       "evidence": "Figure 3 shows CoT underperforming zero-shot and few-shot for GPT-4o across all 9 task categories. Section 4.2 and Appendix B.2 provide detailed analysis showing this pattern across multiple models.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek R1 performs significantly worse than its base model DeepSeek V3 on CodeMMLU (43.91% vs 49.08%), despite being designed for reasoning tasks.",
       "evidence": "Table 3 shows DeepSeek R1 at 43.91% vs DeepSeek V3 at 49.08%. Section 4.2 attributes this to 'overreasoning'.",
       "supported": "strong"
     },
     {
       "claim": "There is a strong correlation (r=0.61) between software knowledge performance and real-world coding task performance.",
       "evidence": "Section 4.2 reports a Pearson correlation of r=0.61 computed from 43 LLMs across 15 families. Figure 5 visualizes the correlation.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs exhibit significant selection bias in MCQ format, with performance varying dramatically based on answer position.",
       "evidence": "Table 4 shows extreme fluctuations (e.g., CodeLlama-7B-Python: 0% for position A, 90.24% for position B). Table 8 and Figure 9 provide STD analysis showing high-quality models are more resilient.",
       "supported": "strong"
     },
     {
       "claim": "Performance on open-ended code generation (HumanEval) does not reliably predict performance on MCQ code completion for the same questions.",
       "evidence": "Section 4.2 and Figure 7 show low correlation coefficients (r=0.10-0.13) between HumanEval pass rates and CodeMMLU code completion accuracy for the same questions.",
       "supported": "strong"
     },
     {
       "claim": "CodeMMLU is the first MCQ benchmark for software and coding-related tasks.",
       "evidence": "Section 1 claims this explicitly. Table 1 compares against existing benchmarks (APPS, MBPP, HumanEval, CRUXEval, LiveCodeBench, CodeApex), showing only CodeApex has MCQ format but with a more limited scope (2,056 vs 19,912 questions).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeMMLU is a 19,912-question multiple-choice benchmark for evaluating LLM code understanding across 52 topics and 10+ programming languages, published at ICLR 2025. Evaluation of 43 LLMs reveals that even top models achieve modest accuracy (GPT o3-mini at 62.36%), Chain-of-Thought prompting consistently degrades performance, and reasoning models like DeepSeek R1 underperform their base models due to overreasoning. A notable discrepancy exists between open-ended code generation performance (HumanEval) and MCQ code comprehension for the same questions, suggesting generation benchmarks may overestimate code understanding.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "An ICLR 2025 paper with no dedicated limitations section is unusual. The paper does not discuss threats to validity or bound its claims about 'code understanding' to what the MCQ format can actually measure."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 43 models evaluated. These settings significantly affect output quality and could explain some performance variation."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on raw accuracy differences without confidence intervals or significance tests. Given that MCQ evaluation on a fixed test set is deterministic for a given model configuration, single-run comparisons may be acceptable, but the paper still claims 'significant' differences without statistical backing."
     },
     {
       "flag": "Overclaimed scope",
       "detail": "The paper claims to assess 'code understanding & reasoning capabilities' but only measures MCQ performance. Multiple-choice format is a narrow proxy for actual understanding; a model could exploit answer distributions or elimination strategies rather than genuinely understanding code."
     },
     {
       "flag": "No funding disclosure",
       "detail": "Authors are primarily affiliated with FPT Software AI Center (a commercial entity) but no funding or conflict of interest statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational code generation benchmark that CodeMMLU converts to MCQ format and directly compares against."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "Introduces the MMLU benchmark that directly inspired CodeMMLU's design philosophy of MCQ-based LLM evaluation."
     },
     {
       "title": "CRUXEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "A closely related code reasoning benchmark that evaluates LLMs' ability to predict code execution outcomes."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "A contamination-aware code benchmark that uses temporal filtering, addressing the same data leakage concerns as CodeMMLU."
     },
     {
       "title": "Large language models are not robust multiple choice selectors",
       "authors": ["Chujie Zheng", "Hao Zhou"],
       "year": 2024,
       "relevance": "Directly relevant to CodeMMLU's analysis of MCQ selection bias in LLMs; the paper builds on this finding."
     },
     {
       "title": "On leakage of code generation evaluation datasets",
       "authors": ["Alexandre Matton", "Tom Sherborne"],
       "year": 2024,
       "arxiv_id": "2407.07565",
       "relevance": "Addresses benchmark contamination in code evaluation, a key methodological concern that CodeMMLU attempts to mitigate."
     },
     {
       "title": "Benchmarking benchmark leakage in large language models",
       "authors": ["Ruijie Xu", "Zengzhi Wang"],
       "year": 2024,
       "arxiv_id": "2404.18824",
       "relevance": "Provides the contamination detection methodology (perplexity and n-gram accuracy) that CodeMMLU uses to assess data leakage."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2024,
       "relevance": "Questions reliability of automated code evaluation, motivating CodeMMLU's approach of MCQ format over execution-based testing."
     },
     {
       "title": "To CoT or not to CoT? Chain-of-thought helps mainly on math and symbolic reasoning",
       "authors": ["Zayne Sprague", "Fangcong Yin"],
       "year": 2024,
       "arxiv_id": "2409.12183",
       "relevance": "Supports CodeMMLU's finding that CoT prompting does not improve performance on non-mathematical tasks."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational CoT prompting paper; CodeMMLU provides evidence against its effectiveness in code understanding domains."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP, a widely-used code generation benchmark that CodeMMLU compares against for ranking consistency."
     },
     {
       "title": "Starcoder 2 and the stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "One of the open-source code LLM families evaluated on CodeMMLU, relevant to the survey's coverage of code generation models."
     }
   ]
 }