ai-research-survey

scan.json (27156B)

---

 {
   "paper": {
     "title": "LeetCodeDataset: A Temporal Dataset for Robust Evaluation and Efficient Training of Code LLMs",
     "authors": [
       "Yunhui Xia",
       "Wei Shen",
       "Yan Wang",
       "Jason Klein Liu",
       "Huifeng Sun",
       "Siyue Wu",
       "Jian Hu",
       "Xiaolong Xu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.14655",
     "doi": "10.48550/arXiv.2504.14655"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LeetCodeDataset curates 2,869 Python LeetCode problems with temporal splits (pre/post July 2024) for contamination-free evaluation and SFT training. Reasoning models (DeepSeek-R1 at 65.2% pass@1) substantially outperform non-reasoning models (GPT-4o at 35.6%) on competitive programming tasks. SFT with only 2.6K model-generated solutions matches or exceeds performance of models trained on 110K samples on HumanEval and MBPP, though this advantage does not extend to harder benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository link provided in abstract: https://github.com/newfacade/LeetCodeDataset"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "HuggingFace dataset link provided in abstract: https://huggingface.co/datasets/newfacade/LeetCodeDataset"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided in the paper. Only model names and training hyperparameters are listed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper mentions an 'evaluation toolkit' but does not include instructions for reproducing the evaluation or SFT results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates (e.g., '65.23%', '79.9%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims reasoning models 'significantly outperform' non-reasoning ones but uses no statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are presented with baseline context enabling effect size assessment, e.g., '79.9% vs. 55.5% on HumanEval' for model-generated vs human-written training data (Section 4.2), and per-model pass rates across difficulty levels (Table 2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set contains 256 problems and SFT uses 2.6K samples. No justification is given for why these sizes are adequate for the claims made, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 evaluates six models (GPT-4o, Claude 3.7 Sonnet, DeepSeek-V3, DeepSeek-R1, Qwen2.5-Max, QwQ-Plus). Section 4 compares SFT against five other training datasets."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include GPT-4o-0806, Claude 3.7 Sonnet, DeepSeek-R1, and QwQ-Plus — all released in 2024-2025 and representing the state of the art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 compares model-generated vs. human-written responses for SFT (79.9% vs 55.5% on HumanEval), isolating the effect of response quality while holding the training set constant."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only pass@1 is reported throughout the paper. While it is evaluated on four benchmarks (HumanEval, MBPP, LiveCodeBench, LeetCodeDataset), only a single metric is used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via test case pass/fail. No human evaluation of solution quality, readability, or efficiency is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "A strict temporal split is used: problems released after July 1, 2024 form the 256-problem test set, while earlier problems are allocated for training (Section 2.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-difficulty breakdowns (Easy/Medium/Hard). Table 3 provides per-topic-tag breakdowns across 24 algorithm categories for all six models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis or specific failure examples are shown. The paper notes models struggle with certain tags (e.g., GPT-4o at 7.7% on Binary Search) but does not analyze why or show failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly reports that the 2.6K-trained SFT model 'underperformed on hard benchmarks,' noting that 'small-scale SFT primarily develops basic programming skills.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: (1) reasoning models outperform non-reasoning — Table 2 shows DeepSeek-R1 65.23% vs GPT-4o 35.55%. (2) SFT with 2.6K samples matches 110K — Table 4 shows 79.9% vs 77.4% on HumanEval."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The key causal claim (model-generated data improves SFT over human-written) uses controlled comparison: same base model (Qwen2.5-Coder-7B), same hyperparameters, same 2.6K training problems, differing only in response source (Section 4)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Robust Evaluation and Efficient Training of Code LLMs' but the dataset covers only Python problems from LeetCode. Section 6 acknowledges coverage gaps but the title and abstract do not bound to Python or competitive programming."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the key findings. For example, the model-generated vs human-written gap could be due to response formatting rather than content quality, and the reasoning model advantage could partly reflect longer inference compute, but neither is considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@1 on LeetCode problems and frames results in terms of coding performance on those specific tasks. Claims match the granularity of measurements without inflating to broader constructs like 'programming ability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o-0806 includes a specific snapshot date, but Claude 3.7 Sonnet, DeepSeek-V3, DeepSeek-R1, Qwen2.5-Max, and QwQ-Plus are all marketing names without snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix provides prompts for input generation (Figures 4-5) used in dataset construction, but the actual evaluation prompts used to test models are not provided. The paper states queries are 'consistent with LiveCodeBench's construction' without reproducing them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3: 'temperature=0.2 and top_p=0.95' for evaluation. Section 4.1: 'learning rate of 1e-5, warmup ratio of 0.1, cosine learning rate scheduling, batch size of 32, 3 epochs.' Section 2.1: 'T=1.0' for data generation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on coding problems without tool use or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the full pipeline: metadata acquisition via GraphQL API (3,505 problems → 3,115 Python), canonical solution verification, entry point identification via pattern matching, LLM-based input generation, sandboxed test case generation (final 2,869 problems)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' presents three specific limitations: false positive risks, complexity analysis gap, and coverage gaps for multi-entry-point problems."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this dataset: (1) imbalanced test case distribution increasing false positive risk, (2) inability to assess time/space complexity, (3) exclusion of multi-function problems."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states that multi-entry-point problems are excluded, complexity analysis is out of scope, and Section 2.1 notes the dataset covers only Python submissions on LeetCode."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is available on HuggingFace (https://huggingface.co/datasets/newfacade/LeetCodeDataset) for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 provides a detailed description of data collection: GraphQL API for metadata, GitHub repositories for canonical solutions verified on LeetCode, LLM-generated test inputs, sandboxed execution for test case outputs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is sourced from the LeetCode platform, a standard public programming problem repository."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the full pipeline with counts at each stage: 3,505 total problems → 3,115 Python-supported → 2,869 with generated outputs. Each transformation step is explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grant information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Authors list only personal email addresses (163.com, 126.com, gmail.com, ieee.org). No institutional or corporate affiliations are disclosed, which is unusual for an academic paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-4o-0806's release date (August 2024) is mentioned as a proxy. Actual training data cutoff dates are not stated for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 explicitly discusses contamination via temporal analysis: 'The minimal temporal overlap between GPT-4o-0806's release date and our test problem release window strongly suggests authentic model capability measurements.' Monthly accuracy curves (Figure 3) are analyzed for contamination signals."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The core design addresses contamination: temporal split at July 2024 ensures test problems post-date model training. Section 3 analyzes monthly accuracy curves for contamination artifacts and concludes fluctuations are due to problem difficulty, not contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for any of the six evaluated models or for the data generation pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or training time is reported for either the SFT experiments or the dataset construction pipeline."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple seeds or seed sensitivity analysis is mentioned. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs producing the evaluation results is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 lists fixed hyperparameters but provides no justification for their selection and no mention of any hyperparameter search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'consistent hyperparameters' were used but does not justify why these specific values were chosen or whether they were tuned."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluated models on it without discussing whether their benchmark design might favor certain model types or problem-solving approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Reasoning models (DeepSeek-R1, QwQ-Plus) use substantially more inference compute via chain-of-thought than non-reasoning models, but performance is compared without normalizing for or discussing this compute difference."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LeetCode competitive programming problems actually measure the coding or reasoning abilities the paper claims to evaluate. The validity of LeetCode as a proxy for real-world coding capability is assumed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated directly on coding problems."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the central concern addressed by the dataset design. Problems released after July 1, 2024 form the test set, ensuring they post-date model training data collection. Section 3 analyzes monthly accuracy for temporal patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak answer information through the problem description, constraints, or test case structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LeetCode problems share structural similarities with problems in model training data from other sources (e.g., similar problems on Codeforces, USACO, or GitHub)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splits serve as a concrete leakage prevention method. Additionally, Section 3 and Figure 3 analyze monthly pass@1 curves as a detection method — stable or rising accuracy post-cutoff would indicate contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Reasoning models significantly outperform non-reasoning counterparts on competitive coding tasks.",
       "evidence": "Table 2: DeepSeek-R1 achieves 65.23% pass@1 vs GPT-4o at 35.55% and DeepSeek-V3 at 35.55%. QwQ-Plus achieves 56.25%. The gap is especially large on Hard problems (DeepSeek-R1 41.86% vs GPT-4o 10.47%).",
       "supported": "moderate"
     },
     {
       "claim": "SFT with 2.6K model-generated LeetCode samples achieves performance comparable to models trained on 110K samples.",
       "evidence": "Table 4: LeetCodeDataset model-generated achieves 79.9% HumanEval and 77.5% MBPP, matching or exceeding Magicoder Evol-Instruct-110K (77.4% HumanEval, 74.1% MBPP). However, on harder benchmarks (LiveCodeBench 15.4% vs 16.9% for OpenThoughts; LeetCodeDataset 12.5% vs 16.4%), the advantage disappears.",
       "supported": "moderate"
     },
     {
       "claim": "Model-generated training data substantially outperforms human-written responses for code SFT.",
       "evidence": "Table 4: Same 2.6K LeetCode problems, model-generated responses achieve 79.9% vs 55.5% on HumanEval and 77.5% vs 53.4% on MBPP. Controlled comparison using same base model and hyperparameters.",
       "supported": "strong"
     },
     {
       "claim": "The temporal split enables contamination-free evaluation.",
       "evidence": "Section 3 and Figure 3: Monthly accuracy curves show no systematic decline for post-July-2024 problems. The minimal temporal overlap between GPT-4o-0806's August 2024 release and the test window is noted. However, this is argued indirectly rather than via direct contamination detection.",
       "supported": "moderate"
     },
     {
       "claim": "The dataset covers over 90% of all Python problems on LeetCode with 100+ test cases per problem.",
       "evidence": "Section 2.1: 2,869 out of 3,115 Python-supported problems (92%) were successfully processed. 'An average of over 100 inputs per problem' generated via LLM prompting.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or multiple runs",
       "detail": "All evaluation results (Tables 2-4) are single-run point estimates with no uncertainty quantification. Given the stochastic nature of LLM inference (temperature=0.2) and SFT training, results could vary meaningfully across runs."
     },
     {
       "flag": "Statistical significance claimed without tests",
       "detail": "The paper uses the word 'significantly' to describe the reasoning model advantage, but no statistical tests are performed. The difference could be within noise for some comparisons."
     },
     {
       "flag": "Missing author affiliations",
       "detail": "All eight authors list only personal email addresses (163.com, 126.com, gmail.com, ieee.org) with no institutional or corporate affiliations. This makes it impossible to assess potential conflicts of interest."
     },
     {
       "flag": "Selective framing of data efficiency claim",
       "detail": "The abstract's claim that 2.6K samples match 110K is true only for HumanEval and MBPP (easy benchmarks). On harder benchmarks (LiveCodeBench, LeetCodeDataset test set), the 2.6K model underperforms, which the paper acknowledges in Section 4.2 but the abstract omits."
     },
     {
       "flag": "Uncontrolled compute confound in reasoning model comparison",
       "detail": "Reasoning models (DeepSeek-R1, QwQ-Plus) use chain-of-thought with substantially more inference compute than non-reasoning models. The performance comparison does not account for this difference, making it unclear how much of the advantage comes from reasoning vs. raw compute."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational benchmark for code generation evaluation used as a baseline in this paper."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation evaluation, used as a baseline in this paper."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Key prior work on contamination-free code evaluation using temporal updates; this paper explicitly extends and competes with LiveCodeBench."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Introduces CodeContests dataset for competitive programming evaluation, a key prior benchmark in this space."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Introduces APPS benchmark for code generation from competitive programming, a foundational dataset this work improves upon."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Key reasoning model evaluated in this paper, demonstrating the reasoning model advantage in competitive coding."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces EvalPlus for more rigorous code generation evaluation, directly relevant to benchmark quality and false positive concerns."
     },
     {
       "title": "Magicoder: Empowering code generation with OSS-Instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2024,
       "arxiv_id": "2312.02120",
       "relevance": "Provides two SFT training datasets (Evol-Instruct-110K, OSS-Instruct-75K) used as baselines in the data efficiency comparison."
     },
     {
       "title": "Qwen2.5-Coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "relevance": "Describes the base model (Qwen2.5-Coder-7B) used for SFT experiments and the model (Qwen2.5-Coder-32B-Instruct) used for training data generation."
     },
     {
       "title": "GPT-4o system card",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2410.21276",
       "relevance": "Technical report for GPT-4o, one of the key models evaluated in this benchmark study."
     },
     {
       "title": "CODEELO: Benchmarking competition-level code generation of LLMs with human-comparable Elo ratings",
       "authors": ["Shanghaoran Quan", "Jiaxi Yang", "Bowen Yu"],
       "year": 2025,
       "arxiv_id": "2501.01257",
       "relevance": "Contemporary code benchmark using Elo ratings from CodeForces submissions, complementary approach to LeetCode-based evaluation."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Extends HumanEval/MBPP to 18 programming languages, relevant to multilingual code generation evaluation."
     }
   ]
 }