ai-research-survey

scan.json (27784B)

---

 {
   "paper": {
     "title": "CODEELO: Benchmarking Competition-level Code Generation of LLMs with Human-comparable Elo Ratings",
     "authors": [
       "Shanghaoran Quan",
       "Jiaxi Yang",
       "Bowen Yu",
       "Bo Zheng",
       "Dayiheng Liu",
       "An Yang",
       "Xuancheng Ren",
       "Bofei Gao",
       "Yibo Miao",
       "Yunlong Feng",
       "Zekun Wang",
       "Jian Yang",
       "Zeyu Cui",
       "Yang Fan",
       "Yichang Zhang",
       "Binyuan Hui",
       "Junyang Lin"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2501.01257"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states in Section 8 (Ethical Statement): 'we will conduct a comprehensive risk assessment and seek permission from the CodeForces platform before open-sourcing the entire submission and evaluation scaffold, and we have not included it in this version of the paper.' The submission/evaluation code is not released. The dataset is released at https://hf.co/datasets/Qwen/CodeElo and the benchmark website exists at https://CodeElo-bench.github.io, but the evaluation scaffold code is withheld."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly available at https://hf.co/datasets/Qwen/CodeElo, as stated in the paper header. The benchmark problems with metadata (contest divisions, difficulty ratings, algorithm tags) are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B mentions decoding hyperparameters and that vLLM was used for open-source model inference, but no requirements.txt, Docker setup, library versions, or detailed environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide step-by-step reproduction instructions. The evaluation scaffold code is withheld (Section 8), and the paper recommends that 'others independently reproduce our proposed method to conduct evaluations' without providing scripts or a README."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports standard deviation values in parentheses after overall Elo ratings (e.g., 'o1-mini 1578 (89.2)'—though these are percentile ranks, not std devs). Section 5.3 discusses rating variance and presents violin plots (Figure 3) showing the distribution of Elo ratings across contests, reporting 'standard deviation between 300 and 500' per contest and 'around 50' for overall averages across 54 contests."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'o1-mini and QwQ-32B-Preview stand out significantly') but no statistical significance tests are performed. Differences between models are assessed by comparing Elo ratings and pass rates directly."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute Elo ratings and pass rates with baseline context. For example, 'o1-mini ... achieving Elo ratings of 1578' compared to human percentiles, and detailed pass rates across difficulty levels (Table 3). The magnitude of differences is clear from the numerical results."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 54 contests with 387 problems, but no justification is given for why this number is sufficient. Section 5.3 mentions that 'increasing the number of tested contests can be beneficial' and that 54 contests yield acceptable variance, but no formal power analysis or sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 reports variance across contests via violin plots (Figure 3), noting 'most models exhibit a standard deviation between 300 and 500' per contest and 'around 50' for overall averages. However, individual results per model appear to be from single runs per contest (8 attempts per problem), not multiple independent runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 30 open-source and 3 proprietary LLMs, providing comparisons across model families and sizes. Table 1 compares CODEELO against six prior benchmarks (APPS, CodeContests, TACO, xCodeEval, USACO, LiveCodeBench) on feature dimensions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary models as of late 2024: o1-mini, Claude-3.5-Sonnet-2024-10-22, ChatGPT-4o-latest-2024-11-20, QwQ-32B-Preview, Qwen2.5 series, DeepSeek-V2.5, and Mistral-Large-Instruct-2411. The benchmark comparisons include LiveCodeBench (2024) and USACO (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "There is no ablation study of the benchmark's components. The paper does not systematically vary aspects of the evaluation method (e.g., number of attempts, prompt design) to understand their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Elo ratings (overall and per-division), pass rates at different difficulty levels (Easy/Medium/Hard), and pass@n for n=1,2,4,8, as shown in Table 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is performed. The evaluation is entirely automated through the CodeForces platform judging system. Human-comparable Elo ratings are computed from platform data, but no humans evaluate the quality of model-generated code beyond pass/fail."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses contest problems from May 4, 2024 to November 4, 2024, and explicitly states they 'only tested on the recently held contests to avoid data contamination' (Section 3.1). The test cases are hidden on the CodeForces platform and unavailable to models."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: by contest division (Div. 1+2, 2, 3, 4), by problem difficulty (Easy/Medium/Hard), by algorithm tag (16 categories in Table 4), and by programming language (C++ vs Python in Figure 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses algorithm categories where models struggle: 'they struggle with dp (DP), dfs and similar (DFS), and trees (Tr.), with many models failing to solve even a single problem under these algorithms.' Section 5.2 discusses the Python vs C++ performance gap as a failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that most models 'struggle even with the easiest problems, placing in the lowest 25 percent among all human participants' (Abstract). It also reports that models perform worse in Python than C++ despite defaulting to Python, which is a finding that contradicts assumptions in prior benchmarks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: o1-mini achieving Elo 1578 and QwQ-32B-Preview achieving 1261 are shown in Figure 1 and Table 3. The claim that 'other models struggle even with the easiest problems, placing in the lowest 25 percent' is supported by Table 3 and Table 6 percentile data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims, e.g., 'This suggests that increasing the length of the chain-of-thought (CoT) is a promising way to improve the models' reasoning ability' (Section 4.2). This causal inference is drawn from observing that o1-like reasoning models perform better, but this is confounded by many other differences between models (training data, architecture, scale). No controlled experiment isolates CoT length as the causal factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'Competition-level Code Generation of LLMs' generally, but the evaluation is limited to CodeForces problems only, using C++ as the primary language, with a specific time window (May-Nov 2024). The paper does not explicitly bound its findings to CodeForces-style competitive programming rather than competition-level coding in general."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its main findings. For example, the C++ vs Python performance gap could be due to training data composition rather than runtime efficiency, but this is not explored. The strong performance of o1-like models could be due to factors beyond CoT length."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Table 5) lists all 33 models with specific version identifiers and HuggingFace endpoints. Proprietary models include date-stamped versions: 'Claude-3-5-Sonnet-2024-10-22', 'ChatGPT-4o-latest-2024-11-20'. Open-source models have exact HuggingFace model paths. However, o1-mini lacks a specific snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompt is provided in Section 4.1: 'You are a coding expert. Given a competition-level coding problem, you need to write a C++ program to solve it. You may start by outlining your thought process. In the end, please provide the complete code in a code block enclosed with ``` ```.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports decoding hyperparameters: 'temperature=0.7, top_p=0.8, top_k=20, and repetition_penalty=1.1' for open-source models, maximum output tokens of 4,096 (32,768 for QwQ-32B-Preview). Proprietary models use 'API calls with default parameters.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation scaffold is described: models receive the problem, generate a solution, the code block is parsed, and an automatic submission bot submits it to CodeForces (Section 3.3.1). Up to 8 attempts per problem, no time penalty, but failed-attempt penalties are counted. This is a simple scaffold (prompt → generate → submit) without iterative feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection: problems scraped from CodeForces in raw HTML format, parsed into sections (problem description, input format, output format, examples, notes). Section 3.2 describes classification by division, difficulty rating, and algorithm tags. Section 4.1 explains that Div. 1 contests were discarded and the time window was May 4 - November 4, 2024."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 is dedicated to 'Limitations' and discusses two specific limitations: the 8-submission limit per problem and reliance on the CodeForces platform for judging."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses specific threats: (1) the 8-submission cap may underestimate model Elo ratings; (2) platform dependency means offline evaluation is not possible. These are specific to this study rather than generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound generalization to CodeForces-only, does not state that results may not transfer to other competition platforms (AtCoder, LeetCode contests), and does not clarify that the Elo ratings are specific to the tested time period and problem set."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly available at https://hf.co/datasets/Qwen/CodeElo, which includes the problems with metadata. The CodeForces platform provides the ground truth judging, which is independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection from CodeForces: all problems from rated contests, recently held contests selected for testing. Section 4.1 specifies the time window (May 4 - November 4, 2024), totaling 54 contests and 387 problems. Table 2 provides statistics per division."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study. The benchmark evaluates LLMs, and human Elo ratings are taken from publicly available CodeForces platform data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: problems scraped from CodeForces in HTML → parsed into sections → classified by division/difficulty/tags → models prompted → solutions submitted via bot → results collected → Elo ratings calculated. Section 4.1 explains discarding Div. 1 contests. The flow from collection to final analysis is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgment of financial support is mentioned in the paper. All authors are from Alibaba Group (Qwen Team), but there is no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as 'Qwen Team, Alibaba Group' with institutional email addresses. The affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from Alibaba Group, which develops the Qwen model family. Several Qwen models are evaluated in the benchmark. Alibaba has a financial interest in demonstrating Qwen models' capabilities. The funder (employer) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. The authors work at Alibaba, which develops and markets several of the evaluated models (Qwen2.5 series, QwQ-32B-Preview), but this conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 33 evaluated models. Section 3.1 mentions using 'recently held contests' to avoid contamination, but the actual training cutoffs are not reported."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states: 'we only tested on the recently held contests to avoid data contamination.' The temporal selection (May-November 2024 contests) is explicitly chosen to reduce overlap with training data. However, no formal analysis of whether any model's training data could include these problems is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses contamination as a key design consideration. Section 3.1 explicitly states temporal selection to avoid it. Section 2 notes that LiveCodeBench 'avoids contamination by re-scraping new problems every month.' CODEELO's online update mechanism is designed to maintain contamination-free evaluation. The benchmark also supports ongoing updates with new contest problems."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The benchmark evaluates LLMs; human Elo ratings are from existing CodeForces platform data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, or tokens consumed are reported. The paper evaluates 33 models across 387 problems with up to 8 attempts each, but the total cost is not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated. The paper mentions using vLLM for open-source model inference but does not report GPU hours, hardware used, or total computational resources."
       }
     }
   },
   "claims": [
     {
       "claim": "o1-mini achieves the best Elo rating of 1578, surpassing nearly 90 percent of human participants on CodeForces.",
       "evidence": "Figure 1 and Table 3 show o1-mini at Elo 1578 with 89.2 percentile rank. Table 6 confirms the 90th percentile human rating is 1603, making 1578 approximately the 89th percentile.",
       "supported": "strong"
     },
     {
       "claim": "QwQ-32B-Preview stands out among open-source models with an Elo rating of 1261, placing around the 60th percentile.",
       "evidence": "Table 3 shows QwQ-32B-Preview at 1261 (63.6 percentile). Table 6 shows the 60th percentile is 1218, consistent with the claim.",
       "supported": "strong"
     },
     {
       "claim": "Most models struggle even with the easiest problems and fall within the lowest 10th percentile of Elo ratings among human participants.",
       "evidence": "Table 3 shows that all open-source models except QwQ-32B-Preview, and the non-o1 proprietary models, have Elo ratings below 710, which Table 6 places below the 25th percentile. Many models fall below 383 (10th percentile).",
       "supported": "strong"
     },
     {
       "claim": "Increasing the length of chain-of-thought (CoT) is a promising way to improve models' reasoning ability.",
       "evidence": "This claim is based on the observation that o1-mini and QwQ-32B-Preview (both o1-like reasoning models) significantly outperform other models. Section 4.2. However, this is a correlation, not a controlled experiment isolating CoT length.",
       "supported": "weak"
     },
     {
       "claim": "Models perform better in C++ than Python on competition-level coding problems.",
       "evidence": "Figure 2 shows Elo ratings for 5 models when constrained to C++ vs Python. All models achieve higher ratings with C++. Section 5.2 discusses this finding in detail.",
       "supported": "moderate"
     },
     {
       "claim": "CODEELO achieves zero false positives by submitting solutions directly to the CodeForces platform.",
       "evidence": "Section 3.3.1 explains that solutions are submitted to CodeForces and judged against the platform's hidden test cases and special judges. Since the platform's judgment is the ground truth for competitive programming, this achieves zero false positives by definition.",
       "supported": "strong"
     },
     {
       "claim": "About 30% of CodeForces problems require special judges.",
       "evidence": "Appendix F states: 'We conducted an empirical study and found that 30 out of 100 randomly selected problems required special judges.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CODEELO introduces a competition-level code generation benchmark using CodeForces problems with direct platform submission for zero false-positive judging and human-comparable Elo ratings. Testing 33 LLMs, the paper finds that o1-mini (Elo 1578, ~89th percentile) and QwQ-32B-Preview (Elo 1261, ~64th percentile) dramatically outperform all other models, most of which fall below the 20th percentile of human participants. An unexpected finding is that all tested models perform better when generating C++ rather than Python, despite defaulting to Python, contradicting prior benchmarks' exclusive use of Python evaluation.",
   "red_flags": [
     {
       "flag": "Conflict of interest: Alibaba evaluating Qwen models",
       "detail": "All authors are from Alibaba's Qwen Team. Multiple Qwen models are evaluated, and QwQ-32B-Preview is highlighted as the best open-source model. While the benchmark evaluates many non-Qwen models too, the undisclosed conflict of interest is a concern. No competing interests statement is included."
     },
     {
       "flag": "No statistical significance tests for comparative claims",
       "detail": "The paper makes numerous claims about models 'standing out significantly' and performing better than others, but no statistical tests are applied. Given the variance shown in Figure 3 (std dev 300-500 per contest), some of the smaller differences between models may not be statistically significant."
     },
     {
       "flag": "Evaluation scaffold code withheld",
       "detail": "The submission bot and evaluation scaffold are not released (Section 8), making independent verification of the evaluation procedure difficult. Researchers must independently implement the submission pipeline."
     },
     {
       "flag": "Causal claims from correlational evidence",
       "detail": "The claim that 'increasing the length of chain-of-thought is a promising way to improve models' reasoning ability' (Section 4.2) is based on observing that o1-like models perform better, but many confounding factors differ between these models."
     },
     {
       "flag": "No compute or cost reporting",
       "detail": "Running 33 models across 387 problems with 8 attempts each represents substantial compute, but no cost or compute budget is reported, making practical reproducibility assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark that CODEELO directly compares against; relevant to benchmark evaluation methodology."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Foundational work on competition-level code generation using CodeForces, providing context for CODEELO's benchmark design."
     },
     {
       "title": "Can Language Models Solve Olympiad Programming?",
       "authors": ["Quan Shi", "Michael Tang", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "arxiv_id": "2404.10952",
       "relevance": "USACO benchmark for competition-level programming evaluation, directly compared in Table 1."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational for LLM code generation evaluation."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Early competition-level coding benchmark directly compared against CODEELO."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "General code benchmark with complex instructions, contrasted with competition-level benchmarks."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "MBPP benchmark paper, foundational for LLM code generation evaluation methodology."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for one of the major model families evaluated in the benchmark."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Technical report for DeepSeek-Coder models, a major model family evaluated in the benchmark."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
       "authors": ["Qihao Zhu", "Daya Guo", "Zhihong Shao"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "State-of-the-art open-source code model evaluated as a competitive baseline."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri", "Abhinav Pandey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Technical report for Llama 3.1 models used as baselines in the benchmark evaluation."
     }
   ]
 }