ai-research-survey

scan.json (21638B)

---

 {
   "paper": {
     "title": "CodeMMLU: A Multi-Task Benchmark for Assessing Code Understanding & Reasoning Capabilities of CodeLLMs",
     "authors": ["Dung Nguyen Manh", "Thang Phan Chau", "Nam Le Hai", "Thong T. Doan", "Nam V. Nguyen", "Quang Pham", "Nghi D. Q. Bui"],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2410.01999"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'CodeMMLU is publicly available at: CodeMMLU' with a GitHub link referenced, and mentions MIT license distribution."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "CodeMMLU benchmark dataset (~20,000 questions) is publicly released under MIT license as stated in Appendix A.3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup with library versions is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts to replicate experiments are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (accuracy percentages) throughout Tables 3, 7, 8 with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4o outperformed all models') based solely on comparing accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context, e.g., 'DeepSeek-Coder-33b surpasses its base model by approximately 29%', and percentage improvements/drops for CoT and answer permutation experiments with deltas shown in Table 4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why ~20,000 questions is the right size, nor is there power analysis for the number of models evaluated."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported for selection bias experiments in Table 8 (STD column). However, no variance across multiple runs of the same model is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against existing benchmarks (HumanEval, MBPP, MMLU, GSM8k) in Table 7 and evaluates 43 LLMs across 15 families."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models like GPT-4o, GPT o3-mini, Claude 3.7 Sonnet, DeepSeek R1, Phi4, and Qwen2.5 — all recent at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper examines the impact of different prompting strategies (zero-shot, few-shot, CoT) in Figure 3 and Appendix B.2, and studies answer permutation effects as a form of ablation on the MCQ format."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary and essentially only metric used is accuracy (percentage of correct answers). No additional metrics like F1 or per-difficulty breakdowns with separate scoring are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual verification was performed: 'we randomly selected 100 instances from each subject area for manual verification against the three criteria' (Section 3.3). This evaluates the benchmark quality, though not system outputs directly."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no explicit discussion of a held-out test set separate from the reported evaluation set. The entire CodeMMLU appears to be both the benchmark and the reported test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-category breakdowns are provided in Tables 3 and 9-11, with results split across syntactic knowledge, semantic knowledge, and fundamental tasks, plus per-subject results in Figure 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed qualitatively in Appendix B.2 with specific examples (Figures 11, 12, 14) showing where CoT reasoning fails, and Section 4.2 discusses where models struggle (execution prediction)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CoT prompting consistently impairs performance (Section 4.2, Figure 3), and that reasoning models like DeepSeek R1 underperform compared to non-reasoning counterparts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'even state-of-the-art models struggle with CodeMMLU' are supported by Table 3 showing no model exceeding ~62% accuracy. Claims about benchmark scale (~20,000 questions) and diversity are verified in Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'CoT consistently impair performance' and 'the additional complexity introduced by step-by-step reasoning does not align well with knowledge-seeking tasks' without controlled experiments isolating the causal mechanism. The CoT analysis is observational across models."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Assessing Code Understanding & Reasoning Capabilities of CodeLLMs' is broad but the benchmark is MCQ-only, primarily English, and draws from specific sources (W3Schools, Common Crawl, HumanEval, LeetCode). The paper does not explicitly bound its generalization claims to these formats and sources."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: CoT performance drop may be due to 'overreasoning' (Section 4.2), HumanEval-MCQ discrepancy could indicate data leakage in open-ended benchmarks vs. actual understanding. The data leakage analysis in Appendix A.2 considers whether results reflect memorization."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names (e.g., 'GPT-4o', 'Claude3.7 Sonnet', 'DeepSeek R1') without API version strings or snapshot dates. No specific model IDs like 'gpt-4o-2024-05-13' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper references prompt configurations in Appendix C.1 for the LLM-based filter and shows example prompts/configurations for zero-shot, few-shot, and CoT settings."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the model evaluations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on MCQ questions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is well-documented in Section 3.3 and Appendix A.1, including rule-based filtering, LLM-based filtering with threshold of 4, MinHash LSH deduplication with 256 permutations and 0.8 similarity threshold, and execution-based filtering. The pipeline is shown in Figure 2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The data contamination analysis in Appendix A.2 acknowledges leakage concerns but does not constitute a threats-to-validity section."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what populations/settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly released under MIT license, allowing independent verification of the questions and answers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail in Section 3: knowledge-based questions from W3Schools, Common Crawl, GeeksForGeeks; fundamental tasks from HumanEval, QuixBugs, LeetCode, IBM CodeNet. Specific sources and processes are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — this is a benchmark evaluation paper using public data sources and automated evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Figure 2 and Sections 3.1-3.3, including crawling, filtering stages (rule-based, LLM-based, execution-based), deduplication, and distractor synthesis. Appendix A.1 provides additional detail including the ~25.6% removal rate from filtering."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: FPT Software AI Center, Hanoi University of Science and Technology, and Independent Researcher."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. Authors are from FPT Software AI Center, which has a potential interest in code LLM evaluation tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models, despite this being critical for assessing whether models may have seen the benchmark data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 directly addresses data leakage using perplexity analysis and n-gram accuracy (Tables 5, 6) following the methodology of Xu et al. (2024), showing CodeMMLU has higher perplexity and lower n-gram accuracy than other benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2 acknowledges that 'creating a fully leakage-free benchmark is virtually impossible' and describes mitigation strategies: transforming seed data into MCQ format, synthetic distractors, and quantitative contamination analysis via perplexity and n-gram accuracy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time for running the benchmark across 43 models is reported, despite the paper's claim that CodeMMLU is 'cost-effective'."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend for the evaluation campaign is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Even state-of-the-art models struggle with CodeMMLU, with the best model (GPT o3-mini) achieving only 62.36% accuracy.",
       "evidence": "Table 3 shows GPT o3-mini at 62.36% and GPT-4o at 56.40% overall accuracy across CodeMMLU tasks.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought prompting consistently impairs performance on CodeMMLU compared to zero-shot and few-shot settings.",
       "evidence": "Figure 3 shows GPT-4o performance declining with CoT across most task categories. Section 4.2 and Appendix B.2 provide analysis with specific examples of reasoning failures.",
       "supported": "moderate"
     },
     {
       "claim": "There is a moderate correlation (Pearson r=0.61) between performance on knowledge-based tasks and real-world coding challenges.",
       "evidence": "Section 4.2 reports Pearson r=0.61 derived from accuracy of 43 LLMs across 15 families, shown in Figure 5.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs exhibit significant selection bias sensitivity in MCQ format, with performance fluctuating dramatically based on answer position.",
       "evidence": "Table 4 shows DeepSeek-Coder-34B with standard deviation of 36.66 across answer positions. Table 8 shows STD values ranging from 2.81 (Claude3 Opus) to 38.07 (CodeLlama-7B-Instruct).",
       "supported": "strong"
     },
     {
       "claim": "Performance in open-ended code generation (HumanEval) does not reliably predict performance in MCQ code completion tasks.",
       "evidence": "Figure 7 shows weak correlation coefficients (0.097-0.134) between HumanEval and CodeMMLU code completion for several models. Table 4 shows large discrepancies.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning models like DeepSeek R1 tend to overreason, generating the longest responses while underperforming compared to non-reasoning counterparts.",
       "evidence": "Table 3 shows DeepSeek R1 (43.91%) underperforming DeepSeek V3 (49.08%). Figure 6 shows R1 generates ~1750 tokens on average vs ~200 for GPT-4o.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeMMLU is a ~20,000 question MCQ benchmark for code understanding spanning 52 topics and 10+ programming languages. Evaluation of 43 LLMs reveals that even the best models achieve only ~62% accuracy, with significant performance gaps between knowledge tasks and real-world coding challenges. Chain-of-Thought prompting generally hurts performance on code knowledge tasks, and models show substantial selection bias sensitivity to answer ordering in MCQ format. The weak correlation between HumanEval and MCQ performance suggests open-ended benchmarks may overestimate code understanding.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity section, which is unusual for an ICLR paper proposing a new benchmark."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 43+ models evaluated, making reproduction difficult and potentially affecting results."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on raw accuracy numbers without significance tests, despite small performance differences between some models."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "All models referenced by marketing names only (GPT-4o, Claude 3.7 Sonnet, etc.) without API versions or snapshot dates, making exact reproduction impossible."
     },
     {
       "flag": "Cost-effectiveness claimed without evidence",
       "detail": "The paper claims CodeMMLU is 'cost-effective' but provides no actual cost data for running the benchmark."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational code generation evaluation that CodeMMLU extends to MCQ format."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark that inspired CodeMMLU's multi-task MCQ design for code domains."
     },
     {
       "title": "CRUXEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Roziere"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark evaluating LLMs' ability to understand program execution."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code benchmark using temporal splits, relevant to benchmark methodology."
     },
     {
       "title": "Large language models are not robust multiple choice selectors",
       "authors": ["Chujie Zheng"],
       "year": 2024,
       "relevance": "Studies MCQ selection bias in LLMs, directly related to CodeMMLU's findings on answer order sensitivity."
     },
     {
       "title": "Benchmarking benchmark leakage in large language models",
       "authors": ["Ruijie Xu"],
       "year": 2024,
       "arxiv_id": "2404.18824",
       "relevance": "Provides the data contamination detection methodology (perplexity, n-gram accuracy) used by CodeMMLU."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code generation benchmark for LLMs with complex evaluation criteria."
     },
     {
       "title": "On leakage of code generation evaluation datasets",
       "authors": ["Alexandre Matton"],
       "year": 2024,
       "arxiv_id": "2407.07565",
       "relevance": "Directly addresses data leakage risks in code benchmarks, motivating CodeMMLU's contamination mitigation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2024,
       "relevance": "Rigorous evaluation methodology for LLM code generation (HumanEval+), relevant to benchmark design quality."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational CoT prompting paper whose effectiveness CodeMMLU challenges for code knowledge tasks."
     }
   ]
 }