ai-research-survey

scan-v5.json (22753B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "LeetCodeDataset: A Temporal Dataset for Robust Evaluation and Efficient Training of Code LLMs",
     "authors": [
       "Yunhui Xia",
       "Wei Shen",
       "Yan Wang",
       "Jason Klein Liu",
       "Huifeng Sun",
       "Siyue Wu",
       "Jian Hu",
       "Xiaolong Xu"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.14655",
     "doi": "10.48550/arXiv.2504.14655"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are substantiated: high-quality benchmark (validated through 100+ test cases and metadata), temporal splits enabling contamination-free eval (demonstrated in Figure 3), and SFT efficiency (Table 4 shows 2.6K-sample model matches 110K-sample baselines on HumanEval/MBPP).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes a causal claim: model-generated training data improves SFT performance vs. human-written (79.9% vs. 55.5% on HumanEval). Study design holds all hyperparameters constant and varies only data source, which is adequate for this comparison.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Scope is appropriately bounded to competitive-programming code generation (LeetCode problems). Paper acknowledges limitations on hard benchmarks (Section 4.2: 'small-scale SFT primarily develops basic programming skills'). No overgeneralization to all code or software tasks.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Paper reports that reasoning models outperform non-reasoning (DeepSeek-R1 65.23% vs. Claude 50.78%) but does not discuss whether advantage stems from architecture, training procedure, reasoning time, or data. Model-generated vs. human-written superiority (Table 4) lacks mechanism analysis.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Paper claims to assess 'reasoning abilities' (Abstract, Section 1) but measures pass@1 rate on test cases. Passing test cases is not the same as demonstrating reasoning—a model could brute-force solutions. The distinction between measurement (pass@1) and claim (reasoning ability) is not acknowledged.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' is a dedicated subsection discussing false positive risks, complexity analysis gaps, and coverage gaps (multiple entry points).",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are listed: 'our dataset lacks extremely complex input patterns and suffers from an imbalanced test case distribution' (false positive risk); 'Determining time/space complexity...exceeds our current scope' (specific capability gap); 'We haven't included...problems with multiple solution entry points' (concrete scope limitation).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: Python-only (Section 2.1: '3,115 supported Python submissions'), single-function entry points (Section 2.1: 'our implementation focuses exclusively on single-function starter code scenarios'), temporal split (pre/post July 2024).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or statement appears in the paper. No acknowledgments section provided.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author names and email addresses are listed, but no institutional affiliations are provided. No disclosure of potential conflicts with LeetCode platform.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures (patents, equity, consulting) are declared.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "Critical terms lack formal definition: 'reasoning models' vs 'non-reasoning models' (used by example, not defined—what makes DeepSeek-R1 'reasoning'?), 'contamination' (used contextually without definition), 'pass@1' (used throughout without introduction).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Intended contributions are explicit: (1) LeetCodeDataset—a new benchmark with 2,869 problems, 100+ test cases per problem; (2) contamination-free evaluation via temporal splits; (3) efficient SFT through curated training data.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 engages with code-generation benchmarks (HumanEval, MBPP, LiveCodeBench, APPS, CodeContests) and fine-tuning datasets (Magicoder, CodeAlpaca, Open-R1). Paper positions LeetCodeDataset as improving on LiveCodeBench (broader coverage, richer metadata) and providing training splits.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "Paper claims benchmark measures 'reasoning abilities' and 'coding capability' but provides no explicit argument for construct validity. No analysis of whether pass@1 rate on test cases actually reflects reasoning or coding skill (vs. memorization, pattern matching, etc.).",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports difficulty distribution: Easy 23.91%, Medium 52.21%, Hard 23.88%. Table 2 confirms discrimination across difficulty tiers—pass rates decline from Easy (94.44% max) to Hard (41.86% max), validating difficulty stratification.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 reveals ceiling effect on Easy (DeepSeek-R1 94.44%, leaving 5.6pp margin). Hard problems avoid floor effects (even best model 41.86%). Data demonstrates discrimination exists, though not explicitly discussed as ceiling/floor analysis.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "Paper provides no human baseline. Cannot validate whether Easy is actually easy for humans or whether Hard poses meaningful human-level difficulty. Only machine performance is reported.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Paper uses pass@1 (binary: code passes all test cases or not) without justification. No explanation for why pass@1 over pass@k, partial credit, test coverage, or other metrics.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is explicitly designed: temporal split (pre/post July 2024) prevents training data leakage. Figure 3 analyzes monthly accuracy by problem release month to detect post-release performance decline (signature of contamination). Approach mirrors LiveCodeBench's strategy.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of temporal robustness or planned updates. Unlike LiveCodeBench ('live updates'), this benchmark provides static splits. No mention of how the dataset will remain challenging as models improve or when new problems should be added.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section 6 lists limitations (false positives, complexity gap, coverage) but does not discuss failure modes of the benchmark itself (e.g., Can solutions be gamed? Are test cases brittle to prompt engineering? Does the benchmark systematically underweight certain coding skills?). Limitations are about coverage, not robustness.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Paper states 'The dataset and evaluation framework are available on Hugging Face and Github.' Mention of 'evaluation framework' indicates baseline evaluation code is provided, enabling reproduction of reported results.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Data collection methodology is documented: Section 2.1 describes metadata acquisition (slug, problem_id, difficulty, description, tags), canonical solution verification, test case generation, and preprocessing. Source (LeetCode GraphQL API) and collection process are detailed.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "Paper states dataset is 'available on Hugging Face and Github' but does not specify licensing terms, usage restrictions, or whether derivatives are permitted. No explicit license (MIT, Apache, CC, etc.) is mentioned.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Intended uses are explicit: (1) 'benchmarking' code models to evaluate reasoning and coding capability; (2) supervised fine-tuning via training split; (3) support for RL training via test cases as verifiers. Context is competitive-programming code generation.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Reasoning models significantly outperform non-reasoning counterparts on code generation.",
       "evidence": "Table 2: DeepSeek-R1 (reasoning, pass@1=65.23%) and QwQ-Plus (reasoning, 56.25%) exceed non-reasoning models (GPT-4o 35.55%, DeepSeek-V3 35.55%, Claude 50.78%). Table 3 shows consistency across topic tags.",
       "supported": "strong"
     },
     {
       "claim": "SFT with 2.6K curated model-generated samples achieves performance comparable to 110K-sample datasets.",
       "evidence": "Table 4: LeetCodeDataset 2.6K samples achieves 79.9% HumanEval, 77.5% MBPP, exceeding Magicoder 111.1K (77.4% HumanEval, 74.1% MBPP).",
       "supported": "moderate"
     },
     {
       "claim": "Model-generated training data outperforms human-written responses for code generation SFT.",
       "evidence": "Table 4: LeetCodeDataset with model-generated responses: 79.9% HumanEval, 77.5% MBPP vs. human responses: 55.5% HumanEval, 53.4% MBPP (same sample size, 2.6K).",
       "supported": "strong"
     },
     {
       "claim": "Temporal problem splits prevent contamination and enable authentic capability measurement.",
       "evidence": "Figure 3 shows minimal monthly accuracy decline post-release for GPT-4o-0806 (overlap Aug 2024 vs test set post-July 2024), interpreted as evidence of clean evaluation. Applies LiveCodeBench methodology.",
       "supported": "moderate"
     },
     {
       "claim": "100+ test cases per problem minimize false positives in solution verification.",
       "evidence": "Section 2.1 describes multi-stage test case generation (LLM one-shot, LLM complex inputs, multiple samples). Section 6 acknowledges residual false-positive risk despite effort.",
       "supported": "moderate"
     },
     {
       "claim": "Difficulty tiers (Easy/Medium/Hard) effectively discriminate model capability.",
       "evidence": "Table 2 shows clear stratification: Easy 81–94% pass rates, Medium 26–69%, Hard 10–42% across models. Performance correlates with difficulty classification.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-creation",
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "LeetCodeDataset curates 2,869 Python LeetCode problems (90% of platform) with 100+ verified test cases and temporal splits (pre/post July 2024) to enable contamination-free evaluation. Reasoning models (DeepSeek-R1: 65.23%, QwQ-Plus: 56.25%) substantially outperform non-reasoning models (Claude: 50.78%, GPT-4o: 35.55%) on the 256-problem test set, confirming reasoning capability advantage in competition-level code generation. Supervised fine-tuning with 2.6K model-generated samples achieves 79.9% HumanEval and 77.5% MBPP—performance matching or exceeding much larger datasets (75–111K samples)—demonstrating exceptional data efficiency through high-quality curation.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "Cannot validate whether Easy, Medium, Hard tiers match human skill levels. Benchmark is calibrated only to machine performance."
     },
     {
       "flag": "Scoring rubric not justified",
       "detail": "Pass@1 (binary correctness) chosen without justification. No comparison to pass@k, partial credit, test coverage, or complexity-aware metrics."
     },
     {
       "flag": "Proxy outcome mismatch",
       "detail": "Paper claims to assess 'reasoning abilities' but measures test-case pass rate. Passing tests does not necessarily demonstrate reasoning; solutions could succeed through pattern matching."
     },
     {
       "flag": "Ceiling effects on Easy problems",
       "detail": "DeepSeek-R1 achieves 94.44% on Easy tier, leaving only 5.6pp discrimination margin. May not adequately differentiate high-performing models."
     },
     {
       "flag": "No conflict of interest disclosure",
       "detail": "No funding statement or author affiliation disclosure. Unclear if any authors have ties to LeetCode platform."
     },
     {
       "flag": "Key terms undefined",
       "detail": "'Reasoning models' vs 'non-reasoning' not formally defined. 'Contamination' and 'pass@1' used throughout without introduction."
     },
     {
       "flag": "No plan for temporal updates",
       "detail": "Unlike LiveCodeBench (live updates), this is a static dataset. No mechanism described for keeping benchmark fresh as models improve."
     },
     {
       "flag": "Limited hard benchmark performance",
       "detail": "Section 4.2 acknowledges SFT model underperforms on hard benchmarks (12.5% LiveCodeBench). Questions practical utility for rigorous evaluation."
     },
     {
       "flag": "No failure mode analysis",
       "detail": "Paper lists coverage gaps but does not discuss what the benchmark systematically fails to measure or how solutions might be gamed."
     },
     {
       "flag": "License terms not specified",
       "detail": "Dataset availability on HuggingFace/GitHub is mentioned, but no explicit license (MIT, Apache, CC) is stated."
     }
   ],
   "cited_papers": [
     {
       "title": "LiveCodeBench: Holistic and contamination-free evaluation of large language models for code",
       "relevance": "Directly related prior work on temporal contamination avoidance in code benchmarking; paper cites LiveCodeBench's approach as motivation for temporal splits."
     },
     {
       "title": "HumanEval: Hand-written evaluation set for evaluating code generation models",
       "relevance": "Foundational code-generation benchmark; LeetCodeDataset evaluated on HumanEval (79.9% for SFT model) for calibration."
     },
     {
       "title": "MBPP: A scalable benchmark for evaluating neural code generation models",
       "relevance": "Standard Python code benchmark; used to validate SFT training efficiency (77.5% with 2.6K samples)."
     },
     {
       "title": "APPS: Learning to code by solving problems",
       "relevance": "Competitive programming dataset; compared as existing baseline for code-generation SFT datasets."
     },
     {
       "title": "CodeContests: A large-scale competition-based code generation dataset",
       "relevance": "Related competitive programming benchmark; positioned as precursor to LeetCodeDataset improvements."
     },
     {
       "title": "Magicoder: Empowering code generation with OSS-instruct",
       "relevance": "SFT baseline dataset (75K samples, 73.8% HumanEval); LeetCodeDataset shows 2.6K samples > 75K samples in data efficiency."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "relevance": "Evaluated as top-performing reasoning model (65.23% pass@1); demonstrates reasoning architecture advantage in code generation."
     },
     {
       "title": "Direct Preference Optimization (DPO): Direct preference optimization with a zero-reference objective",
       "relevance": "Mentioned as an alternative training method (alongside SFT, RL) that LeetCodeDataset could support."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Practitioners can immediately use LeetCodeDataset for model training (SFT) and evaluation. Public release on HuggingFace/GitHub enables direct adoption."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that reasoning models outperform non-reasoning models aligns with expected model capabilities. Data efficiency result (2.6K vs 110K samples) is moderately surprising but presented without mechanistic insight."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper does not raise AI safety, alignment, or risk concerns. Focus is purely on code-generation capability and data efficiency."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial findings, conflicting methodologies, or debate angles. Paper presents a benchmark contribution straightforwardly."
     },
     "demo_ability": {
       "score": 3,
       "justification": "LeetCodeDataset is publicly available on HuggingFace and GitHub with evaluation framework. Users can immediately download and run models on the benchmark."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Models evaluated include OpenAI (GPT-4o), Anthropic (Claude), and DeepSeek (well-known in AI), raising credibility. Authors appear to be from academic/research settings, not major labs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44436031",
         "title": "Show HN: Arch-Router – 1.5B model for LLM routing by preferences, not benchmarks",
         "points": 66,
         "comments": 15,
         "url": "https://news.ycombinator.com/item?id=44436031",
         "created_at": "2025-07-01T17:13:11Z"
       },
       {
         "hn_id": "27075013",
         "title": "MarioNette: Self-Supervised Sprite Learning",
         "points": 47,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=27075013",
         "created_at": "2021-05-07T12:09:34Z"
       },
       {
         "hn_id": "44771836",
         "title": "Arch-Router: Aligning LLM Routing with Human Preferences",
         "points": 9,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=44771836",
         "created_at": "2025-08-02T21:42:16Z"
       },
       {
         "hn_id": "44597819",
         "title": "Show HN: 1.5B LLM routing model that aligns to preferences, not leaderboards",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44597819",
         "created_at": "2025-07-17T20:29:12Z"
       },
       {
         "hn_id": "45324991",
         "title": "Show HN: Model-literals, model-aliases, and preference-aligned routing for LLMs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45324991",
         "created_at": "2025-09-21T17:45:54Z"
       },
       {
         "hn_id": "44650696",
         "title": "Show HN: RouteGPT – model routing on ChatGPT aligned to user preferences",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44650696",
         "created_at": "2025-07-22T17:52:01Z"
       },
       {
         "hn_id": "43191958",
         "title": "Volume estimates for unions of convex sets, and the Kakeya set conjecture in d=3",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43191958",
         "created_at": "2025-02-27T06:47:24Z"
       },
       {
         "hn_id": "45340869",
         "title": "Wan-Animate: Unified Character Animation, Replacement with Holistic Replication",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45340869",
         "created_at": "2025-09-22T23:23:18Z"
       },
       {
         "hn_id": "43206059",
         "title": "A Solution of the Kakeya Conjecture",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43206059",
         "created_at": "2025-02-28T14:34:55Z"
       },
       {
         "hn_id": "44774539",
         "title": "Show HN: Arch-Router – Aligning LLM Routing with Human Preferences",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44774539",
         "created_at": "2025-08-03T06:32:04Z"
       }
     ],
     "top_points": 66,
     "total_points": 137,
     "total_comments": 19
   }
 }