ai-research-survey

scan-v4.json (19704B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Xuanming Zhang",
       "Yunfei Zhao",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2410.22821",
     "doi": "10.48550/arXiv.2410.22821"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: leak rate reduction to 2.18% (Table 3), gpt-4 Pass@1 of 20.74% (Table 4 shows 20.73%), domain-specific findings for gpt-4 and StarCoder 2 (Tables 6-7).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate support: 'We attribute the improvements to the domain knowledge contained in contexts' (Section 3.3) and 'The potential reason for comfort and strange domains is that the pre-training data mix of LLMs is different' (Section 3.4). Neither claim is tested or controlled for confounds.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is generally careful to bound claims to EvoCodeBench-2403 and Python. The Limitations section explicitly states it is 'a monolingual (i.e., Python) benchmark' and notes the small size. Claims are typically prefixed with 'on EvoCodeBench-2403'.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations. The large performance drop vs. prior benchmarks is attributed solely to data leakage, without considering that the benchmark may simply be harder due to repo-level complexity, different task distribution, or other factors.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'EvoCodeBench reveals the actual abilities of these LLMs in real-world repositories' (abstract) but measures only Pass@k on a 275-sample Python benchmark. The gap between 'Pass@k on EvoCodeBench' and 'actual abilities in real-world repositories' is not acknowledged.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Discussion' contains a substantive 'Limitations' subsection discussing two main limitations: Python-only and small benchmark size.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'EvoCodeBench is a monolingual (i.e., Python) benchmark and ignores other programming languages' and 'the size of EvoCodeBench is currently smaller than some existing benchmarks' due to the recent-repositories-only constraint. They also discuss LLM annotation failure modes (missing details, inaccurate domain labels).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states Python-only scope, 275 samples from 25 repos, first version only covering Oct 2023 - Mar 2024, and that domain distribution may be unbalanced. Future plans for expansion are noted.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists National Natural Science Foundation of China grants (62192731, 62152730, etc.), National Key R&D Program, and Major Program of Hubei Province.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Peking University, Bytedance, and Alibaba Group. These are prominently displayed on the first page.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSFC government grants and a government R&D program. The paper evaluates models from OpenAI, DeepSeek, BigCode, and Meta — not products of the funders. The Alibaba/Bytedance-affiliated authors' companies do not have models being evaluated.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Authors from Alibaba Group and Bytedance may have commercial interests related to code LLMs but this is not disclosed.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'data leakage' (test data in training data), 'domain-specific evaluation', 'DSI' (Domain-Specific Improvement with formula), 'comfort domains' (DSI > 10%), 'strange domains' (DSI < -10%), Pass@k and Recall@k with mathematical definitions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are numbered in the abstract and introduction: (1) evolving temporal benchmark, (2) domain taxonomy with labels, (3) domain-specific evaluation metrics including DSI.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides substantive comparison with related benchmarks (HumanEval, ClassEval, DevEval, CoderEval, LiveCodeBench, EvoEval), explicitly explaining how EvoCodeBench differs and fills gaps in existing work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper asserts that repo-level code generation is more realistic, and shows distribution alignment with 500 real repositories (Table 2), but does not argue why Pass@k measures the claimed capability ('actual coding ability') beyond inherited convention from prior work.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "No difficulty distribution is characterized; there are no easy/medium/hard tiers, no item-level difficulty analysis, and no systematic measure of what makes individual tasks harder or easier beyond the domain label.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No explicit ceiling/floor effect check is performed; without-context scores (~5-8% Pass@1) approach floor levels for some models, which is not discussed as a discriminability concern.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline performance is reported; the paper only includes human evaluation of annotation quality (requirements and domain labels), not human performance on the code generation tasks.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Pass@k and Recall@k are adopted from prior work without justifying why these metrics (vs. others like CodeBLEU, exact match, or partial credit) are appropriate; the choice of k ∈ {1, 3, 5, 10} and temperature 0.4 are stated but not justified.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Temporal splitting is the core anti-contamination design: repositories created October 2023–March 2024, after most evaluated models' training cutoffs; validated with CDD detection showing <3% leak rate vs 41.47% for HumanEval.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly plans 6-month update cycles, states that scores across versions are not comparable, and explains the automated collection pipeline that enables new versions to be constructed from latest repositories.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper acknowledges annotation errors (~3.3% requirements, ~1.5% domain labels) but does not discuss benchmark failure modes like test case inadequacy, gaming via requirement memorization, or the small sample sizes in minority domains making domain conclusions unreliable.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "All prompts, LLM completions, and benchmark code are released on GitHub (https://github.com/seketeam/EvoCodeBench) and HuggingFace, enabling reproduction of all reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C is a full datasheet following Gebru et al. (2021) format covering motivation, composition, collection process, preprocessing, uses, distribution, and maintenance — with specific details including timeframes, tools used, and collection criteria.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "CC-4.0 license for the dataset and BSD 3-Clause for code are explicitly stated; dataset is accessible via GitHub and HuggingFace with Croissant metadata.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The datasheet states the benchmark is designed for code generation evaluation and lists other possible tasks, but does not specify what conclusions should NOT be drawn (e.g., that domain findings based on n=1 or n=2 samples are unreliable).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "EvoCodeBench reduces data leakage from 41.47% (HumanEval/gpt-3.5) to under 2.18% across all evaluated LLMs.",
       "evidence": "Table 3: CDD detection results showing leak ratios of 0.73%–2.18% for EvoCodeBench-2403 vs 41.47% for HumanEval.",
       "supported": "strong"
     },
     {
       "claim": "The highest Pass@1 on EvoCodeBench-2403 is only 20.73% (gpt-4), far below 53.04% on DevEval, suggesting prior benchmarks were inflated by contamination.",
       "evidence": "Table 4 shows gpt-4 Local File Infilling Pass@1 = 20.73%; the 53.04% DevEval figure is cited from prior work [16].",
       "supported": "moderate"
     },
     {
       "claim": "Providing local file context improves Pass@1 by 100%+ compared to no-context generation across models.",
       "evidence": "Table 4: gpt-4 without context 7.27% vs Local File Infilling 20.73% (185% increase); similar gains for all models.",
       "supported": "strong"
     },
     {
       "claim": "StarCoder 2-15B performs comparably to gpt-4 in the Database domain despite being much smaller overall.",
       "evidence": "Table 6: Database Pass@1 — gpt-4 38.89%, StarCoder 2-15B 38.89%. Table 7: StarCoder 2-15B has comfort domain status in Database (DSI > 10%).",
       "supported": "moderate"
     },
     {
       "claim": "gpt-4 underperforms relative to other models in the Internet domain despite leading overall.",
       "evidence": "Table 6: Internet Pass@1 — gpt-4 20.00% vs gpt-3.5/DeepSeek Coder 26.67%. Table 7: Internet is a strange domain for gpt-4 (DSI < -10%).",
       "supported": "moderate"
     },
     {
       "claim": "Auto-generated requirements by gpt-4 are comparable to human-written ones in 96.7% of cases.",
       "evidence": "Table 8: Human evaluators rated 30 Win, 236 Tie, 9 Lose for gpt-4 requirements; Cohen's Kappa = 0.9 among evaluators.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "EvoCodeBench is a repo-level Python code generation benchmark that addresses data contamination via temporal splitting (repositories created Oct 2023–Mar 2024), reducing measured leakage from 41.47% to under 3%. LLM performance on this benchmark is dramatically lower than on established benchmarks (gpt-4 tops at 20.73% Pass@1 vs 53.04% on DevEval), suggesting prior evaluations were inflated by contamination. Domain-specific analysis reveals meaningful performance heterogeneity: model overall rankings do not hold within specific domains (e.g., StarCoder 2-15B matches gpt-4 in Database), and an automated 4-stage pipeline enables periodic benchmark refresh every 6 months to maintain temporal freshness.",
   "red_flags": [
     {
       "flag": "Trivially small domain samples",
       "detail": "Several domains have extremely small sample counts: Security n=1, Utilities n=2, Communications n=8, Text Processing n=12. Domain-specific conclusions for these categories are statistically meaningless and should not be reported."
     },
     {
       "flag": "Circular LLM annotation validation",
       "detail": "Requirements and domain labels are generated by gpt-4, then quality is partially assessed via human evaluation — but the 96.7% agreement rate is measured by humans comparing gpt-4 outputs to other human-written ones, not an independent quality audit of whether the gpt-4 requirements accurately capture function semantics."
     },
     {
       "flag": "No human performance baseline",
       "detail": "The paper establishes no human baseline for the code generation tasks, making it impossible to assess whether benchmarked scores represent meaningful capability gaps or near-human performance."
     },
     {
       "flag": "Leakage vs difficulty confound",
       "detail": "The claim that lower EvoCodeBench scores prove leakage in prior benchmarks does not control for task difficulty differences; temporal repositories and different selection criteria may produce inherently harder tasks independent of contamination."
     },
     {
       "flag": "Alibaba author conflict",
       "detail": "Three authors are from Alibaba Group, which has commercial interests in code LLM development; no competing interests statement is present despite this affiliation."
     },
     {
       "flag": "No ceiling/floor effect analysis",
       "detail": "Without-context Pass@1 scores (~5-8%) approach floor levels and are not analyzed for whether the no-context condition meaningfully discriminates among models."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Primary baseline benchmark that EvoCodeBench is designed to improve upon; used to demonstrate data leakage rates."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Closest prior work for repo-level code generation; EvoCodeBench directly compares against it for leakage, dependency statistics, and performance."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-Trained Models",
       "relevance": "Repo-level benchmark with known contamination issues that EvoCodeBench is designed to address."
     },
     {
       "title": "ClassEval: Evaluating Large Language Models in Class-level Code Generation",
       "relevance": "Domain-annotated benchmark compared to EvoCodeBench in Table 2; cited for cost of manual annotation (500 person-hours)."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Contemporary contamination-resistant benchmark using competitive programming; EvoCodeBench distinguishes itself as repo-level vs snippet-level."
     },
     {
       "title": "EvoEval: Evolving Coding Benchmarks via LLM",
       "relevance": "Alternative contamination-resistant approach (LLM mutation of HumanEval); compared to EvoCodeBench's temporal filtering approach."
     },
     {
       "title": "Generalization or Memorization: Data Contamination and Trustworthy Evaluation for Large Language Models (CDD)",
       "relevance": "The leakage detection method (CDD) used to validate EvoCodeBench's contamination resistance claims."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "relevance": "One of the primary evaluated code LLMs; DeepSeek Coder-33B achieves the best dependency recall on EvoCodeBench."
     },
     {
       "title": "Datasheets for Datasets",
       "relevance": "Framework used for EvoCodeBench's Appendix C dataset documentation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners can use EvoCodeBench to evaluate code LLMs for domain-specific selection, and the domain taxonomy provides actionable guidance."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that gpt-4 achieves only 20.73% Pass@1 (vs 53.04% on DevEval) suggests prior benchmarks may be leaked, though this concern was already growing in the community."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised by this benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit criticism that existing benchmarks (especially HumanEval with 41.47% leakage) are unreliable, but presented diplomatically."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released on GitHub and HuggingFace; researchers can download and evaluate their own models on the benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Peking University and Alibaba, published at NeurIPS; evaluates well-known models (gpt-4, DeepSeek Coder, StarCoder 2)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42172392",
         "title": "Epipolar-Free 3D Gaussian Splatting for Generalizable Novel View Synthesis",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42172392"
       },
       {
         "hn_id": "42007858",
         "title": "Universality of the π²/6 Pathway in Avoiding Model Collapse [pdf]",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42007858"
       }
     ],
     "top_points": 2,
     "total_points": 3,
     "total_comments": 0
   }
 }