ai-research-survey

scan.json (24742B)

---

 {
   "paper": {
     "title": "CodeArena: A Collective Evaluation Platform for LLM Code Generation",
     "authors": [
       "Mingzhe Du",
       "Anh Tuan Luu",
       "Bin Ji",
       "Xiaobao Wu",
       "Dong Huang",
       "Terry Yue Zhuo",
       "Qian Liu",
       "See-Kiong Ng"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.01295"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a website (https://codearena.online), a PyPI package (codearena), and the platform is described as built on the open-source DMOJ framework. The platform itself is accessible and a Python library is published at https://pypi.org/project/codearena/."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'all solutions and test cases are publicly accessible' (Section 1, Ethics Statement). A trial account is provided (Account: Test / Password: Haveatry!) for browsing data (Appendix D). The platform stores and makes accessible all submitted solutions and test cases."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions '8 NVIDIA A100 GPUs' and 'bfloat16' format for open-source models, but no requirements.txt, Dockerfile, or detailed dependency specifications are provided for reproducing the platform or experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the platform architecture and API endpoints but does not provide instructions for replicating the experimental evaluation results (leaderboard numbers)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates for Dynamic Points and Pass scores. No confidence intervals or error bars are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about model rankings (e.g., 'most closed-source LLMs adhere to the scaling law, significantly outperforming their open-source counterparts') but provides no statistical significance tests to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw Dynamic Point scores and Pass percentages are reported in Table 1, but no effect sizes (e.g., Cohen's d, percentage improvement with baseline context) are calculated or discussed."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 19 models on problems imported from APPS and Mercury benchmarks but does not justify the number of problems used or discuss whether the problem set is sufficiently large for reliable rankings."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each LLM is allowed only a single attempt per problem. No variance, standard deviation, or spread measures across multiple runs are reported. Single-run results only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare CodeArena's evaluation methodology against alternative evaluation approaches (e.g., standard Pass@1 on static benchmarks vs. Dynamic Points). The leaderboard shows model comparisons but no baseline evaluation framework is compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baseline evaluation methodology is compared. LiveCodeBench (Jain et al., 2024), which also addresses contamination through temporal updates, is discussed in related work but not directly compared against as a baseline approach."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The Dynamic Point metric combines Challenge Score and Efficiency Score but no ablation is performed to show the individual contribution of each component or to validate that the collective scoring mechanism works better than simpler alternatives."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both Dynamic Points (DP) and Pass rate as metrics in Table 1. The DP itself combines Challenge Score and Efficiency Score."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark/platform paper evaluating code generation correctness and efficiency through automated test case execution. Human evaluation of system outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a platform/benchmark paper, not a machine learning training paper. There is no training/validation/test split to evaluate; the platform evaluates submissions against test cases."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides per-difficulty-level acceptance rate breakdowns. Table 1 provides individual model results. Figure 4 shows per-model Dynamic Point changes over time checkpoints."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. The paper does not analyze specific problems where models failed, types of errors in generated code, or patterns in failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows that acceptance rate does not differentiate well across difficulty levels, which is a negative finding about existing difficulty labeling. Figure 4 shows open-source models declining over time, which is a meaningful negative observation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the collective evaluation mechanism 'dynamically recalibrates individual model scores... mitigating score biases caused by widespread benchmark leakage.' However, no empirical evidence is provided that the Dynamic Point mechanism actually mitigates contamination bias — it is argued theoretically via the formula but not validated experimentally (e.g., by comparing rankings on contaminated vs. clean problems)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the dynamic scoring mechanism 'effectively mitigates the influence of data contamination' (Section 3.1). This is a causal claim but no controlled experiment demonstrates that contaminated problems produce different rankings under the DP metric vs. standard scoring. The evidence is only the mathematical formula, not an empirical validation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Collective Evaluation Platform for LLM Code Generation' broadly, but the evaluation uses only Python problems from APPS and Mercury. The platform supports multiple languages (Python, C, C++, Go, Haskell) but evaluation is shown only for Python. This overgeneralization is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper observes that open-source models decline in DP scores over time while closed-source models remain stable (Figure 4), suggesting contamination. However, alternative explanations (e.g., closed-source models being generally more capable, new problems being harder) are not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and Appendix A list models like 'GPT-4o', 'Claude-3-5-sonnet', 'Gemini-1.5-flash' without specifying exact API versions or snapshot dates. For example, 'GPT-4o' without a date-versioned identifier (e.g., gpt-4o-2024-05-13) is insufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides both the system prompt and inference prompt template used for all LLMs. While the inference prompt has placeholders (Example Problem Description, Example Solution, Problem), these are structural placeholders for the benchmark problems themselves, and the actual prompt text/structure is fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C states temperature of 0.7 for open-source models but does not report temperature or other inference settings for closed-source model API calls. For closed-source models, only the API links are provided (Appendix A), with no inference parameters specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct one-shot prompting of LLMs for code generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the problem collection workflow (importing from APPS and Mercury, collecting from CodeForces and LeetCode weekly contests), test case generation process (using GPT-4o to generate test case generators), and filtering (removing problems with ambiguous outputs). The data pipeline is reasonably documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section exists after the Conclusion, discussing reliance on external data sources and automated test case quality."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions only two generic concerns: reliance on external data sources and automated test case quality. These are specific to the platform but not to the experimental evaluation results. No threats to the validity of the leaderboard rankings or the Dynamic Point mechanism are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. For example, it does not acknowledge that results are shown only for Python, that the Dynamic Point mechanism's contamination-mitigation effect is not empirically validated, or that the leaderboard reflects single-run performance."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The platform makes all solutions and test cases publicly accessible. A trial account is provided for browsing the data repository. The raw submission data is available through the platform APIs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the data collection procedure: problems imported from APPS and Mercury benchmarks, plus regular collection from CodeForces and LeetCode weekly contests. Test cases are generated using GPT-4o."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for the evaluation. The 'Code Generators' are LLMs pre-registered by the researchers, not recruited human users."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the pipeline: problem collection → test case generation via GPT-4o → filtering out ambiguous problems → submission through APIs → sandbox execution → dynamic scoring. Each step is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section listing grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: NTU, NUS, HKU, Monash, and ByteDance. One author (Qian Liu) is from ByteDance, which is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. One author is affiliated with ByteDance, which has commercial interest in LLM evaluation, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 19 LLMs on code generation benchmarks including APPS and Mercury, but does not state the training data cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper's central motivation is benchmark contamination, it does not analyze whether the specific problems used (from APPS, Mercury, LeetCode, CodeForces) appear in the training data of the evaluated models. The Dynamic Point mechanism is proposed as a mitigation but no empirical analysis of actual contamination is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges contamination as a problem and proposes Dynamic Points as a theoretical mitigation, but does not empirically verify whether APPS or Mercury benchmark problems were in the training data of evaluated models. APPS (2021) and HumanEval are well-known contaminated benchmarks for post-2022 models, but this is not specifically addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The evaluation involves only LLM code generators."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper calls APIs for multiple closed-source LLMs and runs open-source models on 8 A100 GPUs but reports no inference costs, API costs, tokens consumed, or wall-clock time for the evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions '8 NVIDIA A100 GPUs' for open-source model inference but does not quantify total GPU hours, API spend for closed-source models, or overall computational budget."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeArena's Dynamic Point mechanism mitigates score biases caused by benchmark leakage by dynamically recalibrating individual model scores based on holistic performance of all participating models.",
       "evidence": "Section 3.1 presents the Challenge Score formula (Eq. 1) which divides the basic problem score by the number of models solving it. The theoretical argument is that leaked problems solved by many models contribute less to the ranking. No empirical validation of contamination mitigation is provided.",
       "supported": "weak"
     },
     {
       "claim": "Most closed-source LLMs adhere to the scaling law, significantly outperforming their open-source counterparts.",
       "evidence": "Table 1 shows closed-source models generally rank higher, with DeepSeek-Coder at 249.28 DP vs. the best open-source model DeepSeek-Coder-V2-Lite at 223.67 DP. However, no statistical test supports the claim of 'significant' outperformance, and DeepSeek-Coder-V2-Lite (open-source, small) ranks 5th, challenging the scaling claim.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source LLMs exhibit a clear downward trend in Dynamic Point scores over time, while closed-source LLMs maintain stable scores.",
       "evidence": "Figure 4 shows DP changes over checkpoints from July to November 2024. Open-source models (especially DeepSeek-V2-Lite) decline while closed-source models remain stable or slightly improve. However, this is shown as a trend line without statistical analysis or explanation of the mechanism.",
       "supported": "moderate"
     },
     {
       "claim": "Acceptance rate does not show significant variation across difficulty levels inherited from LeetCode.",
       "evidence": "Figure 5 shows acceptance rate distribution across Easy, Medium, and Hard difficulty levels. The distributions appear overlapping, suggesting LeetCode's human-defined difficulty does not predict LLM difficulty well. However, no statistical test for the claimed lack of variation is provided.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeArena introduces an online evaluation platform for LLM code generation with three main features: a Dynamic Point scoring mechanism that theoretically mitigates benchmark contamination by weighting problems inversely to their acceptance rate, publicly accessible solution and test case data, and automation-friendly APIs. Evaluation of 19 LLMs shows closed-source models generally outperform open-source ones, and temporal analysis reveals declining Dynamic Point scores for open-source models as new problems are added. The Dynamic Point mechanism is argued theoretically but not empirically validated against actual contamination.",
   "red_flags": [
     {
       "flag": "Central claim not empirically validated",
       "detail": "The paper's main contribution — that Dynamic Points mitigate benchmark contamination — is supported only by the mathematical formula, not by any controlled experiment comparing rankings on contaminated vs. clean problems, or showing that known-contaminated models are penalized."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Model rankings and comparative claims (e.g., 'significantly outperforming') are made without any statistical tests, confidence intervals, or error bars. Each model gets only one attempt per problem."
     },
     {
       "flag": "Missing model version specificity",
       "detail": "Closed-source models are referenced by marketing names (GPT-4o, Claude-3-5-sonnet) with only website links, not API version identifiers. Model behavior changes across versions, making results unreproducible."
     },
     {
       "flag": "Self-citation pattern",
       "detail": "The first author (Mingzhe Du) is also the first author of the Mercury benchmark, which is heavily featured as a key data source and related work. Two other co-authors (Dong Huang, Terry Yue Zhuo) are authors of EffiBench and BigCodeBench respectively, also used as comparisons. While not inherently problematic, this overlap means the evaluation ecosystem is not independently validated."
     },
     {
       "flag": "Incomplete hyperparameter reporting",
       "detail": "Temperature is reported for open-source models (0.7) but not for closed-source API calls. This asymmetry means the evaluation conditions are not comparable across model types."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational LLM code generation benchmark that CodeArena builds upon and aims to improve."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Directly comparable contamination-free LLM code evaluation approach using temporal problem updates, a key related work to CodeArena."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code generation benchmark evaluating LLMs on complex programming tasks with diverse function calls."
     },
     {
       "title": "Mercury: An Efficiency Benchmark for LLM Code Synthesis",
       "authors": ["Mingzhe Du", "Anh Tuan Luu"],
       "year": 2024,
       "arxiv_id": "2402.07844",
       "relevance": "Efficiency-centric code generation benchmark that is integrated into CodeArena and shares authors with this paper."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
       "authors": ["Qihao Zhu", "Daya Guo"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Open-source code generation model that achieves top performance in CodeArena, relevant to evaluating open vs. closed-source model capabilities."
     },
     {
       "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "Security-focused evaluation suite for LLM-generated code, relevant to understanding the broader LLM code evaluation landscape."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Major code generation benchmark integrated into CodeArena's problem set, important for understanding benchmark contamination risks."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Evaluates rigorousness of LLM code evaluation methodology, relevant to understanding evaluation quality concerns."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code generation model and training data, relevant to understanding code generation capabilities and data contamination."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code generation model family evaluated in CodeArena."
     }
   ]
 }