ai-research-survey

scan-v5.json (19202B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Xuanming Zhang",
       "Yunfei Zhao",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2410.22821",
     "doi": "10.48550/arXiv.2410.22821"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Key abstract claims — leak rate reduction (41.47%→2.18%), gpt-4 Pass@1 of 20.74%, gpt-4 Internet domain weakness, StarCoder 2-15B Database strength — are directly supported by Tables 3, 4, and 6/7 respectively.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'LLMs benefit from more code contexts' and attributes 104–152% improvements to 'domain knowledge contained in contexts,' but this is observational (comparing prompt settings); no controls rule out alternative explanations such as reduced generation length or structural constraint effects.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly claims EvoCodeBench 'reveals the actual abilities of LLMs in real-world repositories,' a broad generalization from 275 Python-only samples from 25 repositories; the gap between test-passing and real-world development ability is not bounded.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not distinguish whether lower scores vs. prior benchmarks reflect contamination removal versus inherently harder repo-level tasks; no alternative explanations for context improvement or domain variation are considered.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Pass@k (test case execution) is equated with 'actual abilities in real-world repositories' without acknowledging the gap between passing provided test cases and broader software development competence.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 contains a dedicated 'Limitations' paragraph explicitly discussing the monolingual (Python only) scope and small dataset size relative to some prior benchmarks.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discussion is generic (language coverage and size) and omits specific threats such as test case quality variability, LLM-annotation bias, or the near-zero sample count in Security (1) and Utilities (2) domains undermining domain-specific claims.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to Python, repositories created Oct 2023–Mar 2024, and notes that Pass@k across different benchmark versions are not comparable.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgements section: National Natural Science Foundation of China (multiple grants), National Key R&D Program, and Major Program of Hubei Province.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the title page: Peking University, Bytedance, and Alibaba Group.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Primary funders are Chinese government research grants (NSFC, national R&D programs) with no financial stake in benchmark outcomes; the paper evaluates competitors' models (OpenAI, DeepSeek, Meta, BigCode).",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, no disclosure of patents, equity, or consulting relationships; Alibaba-affiliated co-authors' potential interests in competing code models are not addressed.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'repo-level code generation' (Section 2.2), 'data leakage' (Section 1), 'Domain-Specific Improvement/DSI' (Equation 3), 'comfort/strange domains' (threshold T=10%), Pass@k (Equation 1), and Recall@k (Equation 2).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states three contributions: evolving data to prevent leakage, a 10-domain taxonomy with labels, and domain-specific evaluation metrics (DSI, comfort/strange domains).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 systematically compares EvoCodeBench with HumanEval, MBPP, ClassEval, CoderEval, DevEval, LiveCodeBench, and EvoEval, explaining differences in scope, approach, and methodology; Table 2 provides a structured comparison.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues construct validity by demonstrating that EvoCodeBench-2403 aligns with 500 real-world repositories on code distribution and dependency distribution (Table 2), supporting the claim that it approximates real-world coding conditions.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "No difficulty tiers (easy/medium/hard) are defined or measured; the paper reports aggregate Pass@k but does not characterize the distribution of item difficulty across the 275 samples.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No ceiling effect exists (top Pass@1 ≈20%), but floor effects are not addressed: in the Text Processing domain (12 samples), 7 of 8 models score 0% Pass@1 — a measurement validity concern not discussed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline for the coding task is provided; human evaluation is conducted only for annotation quality (requirements and domain labels), not for benchmark programming tasks themselves.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@k (functional correctness via test execution) is established from prior work with formula provided (Equation 1); Recall@k is introduced with clear formula (Equation 2) and justified as measuring dependency utilization in repo-level generation.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is core to the design: repositories created after most LLMs' training cutoffs (Oct 2023–Mar 2024); CDD detection validates leak rates of 0.73%–2.18% across all evaluated models (Table 3).",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Temporal robustness is explicitly designed via the 'evolving' mechanism (updates every ~6 months), and the paper notes that Pass@k across versions are not comparable, with new versions planned.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss failure modes of the benchmark itself, such as test case quality variability, gaming via function name similarity (used in RAG baseline), or consistent biases introduced by LLM-generated requirements.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "All prompts, LLM completions, and code are released on GitHub and HuggingFace, with Croissant metadata provided, enabling full reproduction of reported results.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "A comprehensive datasheet (Appendix C, following Datasheets for Datasets v8 [9]) covers motivation, composition, collection process, preprocessing, uses, distribution, and maintenance with detailed answers to all standard questions.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "Dataset is under CC-4.0 license, code under BSD 3-Clause; access via GitHub and HuggingFace is specified with long-term maintenance commitment from the SEKE team at Peking University.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Intended use (evaluating LLMs in repo-level code generation) is clearly stated; alternative uses are mentioned (code completion, test generation, summarization); however, explicit guidance on what should NOT be concluded is absent.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "EvoCodeBench reduces data leakage from 41.47% (HumanEval/gpt-3.5) to under 2.18% across all tested models.",
       "evidence": "CDD detection results in Table 3 show leak rates of 0.73%–2.18% for EvoCodeBench-2403 vs. 41.47% for HumanEval with gpt-3.5.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4's highest Pass@1 on EvoCodeBench-2403 is only 20.73%, far below its 53.04% on the prior repo-level benchmark DevEval.",
       "evidence": "Table 4 shows gpt-4 Pass@1 of 20.73% (Local File Infilling); the DevEval comparison is stated directly in Section 3.3.",
       "supported": "strong"
     },
     {
       "claim": "LLMs benefit substantially from code context: adding local file context improves gpt-4 Pass@1 by 104–152%.",
       "evidence": "Table 4 shows gpt-4 at 7.27% (Without Context) vs. 17.45% (Completion) and 20.73% (Infilling); percentages stated in Section 3.3 but causal mechanism is assumed not tested.",
       "supported": "moderate"
     },
     {
       "claim": "Domain-specific ranking diverges from overall ranking: gpt-4 underperforms in the Internet domain despite leading overall.",
       "evidence": "Table 6 shows gpt-4 Pass@1 of 20.00% in Internet vs. 26.67% for gpt-3.5 and DeepSeek Coder; Table 7 confirms gpt-4 DSI of -28.59% in Internet.",
       "supported": "strong"
     },
     {
       "claim": "StarCoder 2-15B unexpectedly performs as well as GPT-4 in the Database domain, outperforming larger 33B models.",
       "evidence": "Table 6 shows StarCoder 2-15B at 38.89% in Database, equal to gpt-4, gpt-3.5, and DeepSeek Coder 33B; Table 7 confirms positive DSI for StarCoder 2-7B vs. negative for DeepSeek Coder 33B.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4-generated annotations are comparable to human-written ones: 96.7% requirement quality and 98.5% domain label agreement.",
       "evidence": "Human evaluation in Table 8 with Cohen's Kappa of 0.9 among evaluators; gpt-4 wins/ties on (30+236)/275=96.7% of requirements and (3+268)/275=98.5% of domain labels.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "EvoCodeBench-2403 achieves data leak rates under 2.2% by restricting to repositories created after major LLMs' training cutoffs (Oct 2023–Mar 2024), compared to 41.47% for HumanEval. All tested models score dramatically lower than on prior benchmarks (gpt-4 Pass@1: 20.73% vs. 53.04% on DevEval), suggesting significant contamination in existing evaluations. Domain-specific evaluation reveals that overall ranking does not predict domain-level ranking: gpt-4 underperforms in Internet while StarCoder 2-15B matches gpt-4 in Database despite being smaller. The benchmark's evolving design — planned updates every ~6 months from new repositories — is the primary mechanism for sustained contamination resistance.",
   "red_flags": [
     {
       "flag": "Severely skewed domain distribution",
       "detail": "Scientific Engineering has 120 samples while Security has 1 and Utilities has 2; domains with <10 samples are excluded from domain analysis, quietly undercutting the '10 domain coverage' contribution."
     },
     {
       "flag": "LLM evaluates its own annotation quality",
       "detail": "GPT-4 generates the natural language requirements and domain labels, then GPT-4 is one of the 8 evaluated models; requirements phrased in GPT-4's style may structurally advantage GPT-4 without the authors acknowledging this bias."
     },
     {
       "flag": "No human baseline on benchmark coding tasks",
       "detail": "Without a human ceiling, the practical significance of 20% Pass@1 cannot be calibrated — it is unclear whether this is difficult or trivially easy for an expert developer."
     },
     {
       "flag": "Floor effects in Text Processing not addressed",
       "detail": "7 of 8 models score 0% Pass@1 in Text Processing (12 samples); this is a measurement validity concern for that domain that is not acknowledged or analyzed."
     },
     {
       "flag": "Causal attribution without controls",
       "detail": "Performance improvement from code context is attributed to 'domain knowledge in contexts,' but confounds like reduced generation length, structural constraints, or hint about function existence are not ruled out."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational snippet-level benchmark that EvoCodeBench explicitly addresses for data leakage; provides the 41.47% contamination baseline for comparison."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Primary prior-art repo-level benchmark; EvoCodeBench is directly compared to DevEval on methodology and Pass@k results."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-Trained Models",
       "relevance": "Another repo-level benchmark compared in Table 2; establishes the dependency-aware evaluation paradigm EvoCodeBench extends."
     },
     {
       "title": "Evaluating Large Language Models in Class-Level Code Generation (ClassEval)",
       "relevance": "Benchmark with manually designed domain labels; EvoCodeBench addresses limitations of its narrow domain coverage and potential future leakage."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Contemporary contamination-free benchmark for snippet-level competitive programming; contrasted with EvoCodeBench's repo-level approach."
     },
     {
       "title": "EvoEval: Evolving Coding Benchmarks via LLM",
       "relevance": "Related work on evolving benchmarks using LLM mutation; the paper differentiates EvoCodeBench's temporal split approach from EvoEval's mutation approach."
     },
     {
       "title": "Generalization or Memorization: Data Contamination and Trustworthy Evaluation for Large Language Models (CDD)",
       "relevance": "Detection method used to empirically validate EvoCodeBench's contamination resistance claims in Table 3."
     },
     {
       "title": "Datasheets for Datasets",
       "relevance": "Framework followed for EvoCodeBench's comprehensive dataset documentation in Appendix C."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Practitioners can directly use EvoCodeBench to select the best model for their specific programming domain, with immediately available data on GitHub and HuggingFace."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that overall ranking predicts domain-specific performance — a 15B model outperforms GPT-4 in Database, and GPT-4 is worst-in-class for Internet despite leading overall."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper focuses exclusively on evaluation methodology."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implies that prior benchmark results (including HumanEval with 41.47% leakage) are unreliable, but frames this constructively rather than as a direct attack on prior work."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Benchmark is immediately runnable from GitHub with full code, prompts, and model completions released for community reuse."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Peking University and Alibaba DAMO Academy are respectable institutions but not marquee AI labs; the paper evaluates well-known models (GPT-4, DeepSeek Coder) which adds indirect recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42172392",
         "title": "Epipolar-Free 3D Gaussian Splatting for Generalizable Novel View Synthesis",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42172392"
       },
       {
         "hn_id": "42007858",
         "title": "Universality of the π²/6 Pathway in Avoiding Model Collapse [pdf]",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42007858"
       }
     ],
     "top_points": 2,
     "total_points": 3,
     "total_comments": 0
   }
 }