ai-research-survey

scan.json (30663B)

---

 {
   "paper": {
     "title": "EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Xuanming Zhang",
       "Yunfei Zhao",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems (NeurIPS 2024, Datasets and Benchmarks Track)",
     "arxiv_id": "2410.22821",
     "doi": "10.48550/arXiv.2410.22821"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EvoCodeBench-2403 reduces data leakage from 41.47% (HumanEval) to under 3% by using repositories created after October 2023. The highest Pass@1 (gpt-4) on EvoCodeBench-2403 is only 20.73%, far below scores on prior benchmarks, suggesting those may be leaked. LLMs show distinct comfort and strange domains: gpt-4 excels in most domains but underperforms in Internet, while StarCoder 2-15B unexpectedly matches gpt-4 in Database. Providing code context improves gpt-4's Pass@1 by up to 152%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub (https://github.com/seketeam/EvoCodeBench) and HuggingFace (https://huggingface.co/datasets/LJ0815/EvoCodeBench) links are provided in the abstract footnote and Appendix A."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released on GitHub and HuggingFace under CC-4.0 license. All prompts and LLMs' completions are also released (Appendix A)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 mentions '4 NVIDIA A100-40GB GPUs' and Appendix E.1 describes models, but no requirements.txt, Dockerfile, or library versions are provided. Not enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases code, all prompts (Figures 5-8, Appendix E.2), LLM completions, and benchmark data on GitHub. Experimental settings are thoroughly documented in Section 3 and Appendix E."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 4, 5, 6, and 7 report only point estimates for Pass@k, Recall@k, and DSI. No confidence intervals or error bars are provided anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'gpt-4 performs best', 'StarCoder 2-15B unexpectedly performs well') based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: 'Pass@1 of gpt-4 is improved by 104% and 152%' (Section 3.3). Table 3 shows leakage rate drops from 41.47% to 2.18%. DSI values in Table 7 show relative improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 275 samples from 25 repositories with no justification for why this size is adequate. The paper acknowledges it is 'currently smaller than some existing benchmarks' in the Limitations section but provides no power analysis or statistical justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. While they sample 20 programs per requirement for k>1, only the unbiased Pass@k point estimates are reported without any measure of variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight LLMs are compared against each other (Table 4), and the benchmark's leakage rate is compared against HumanEval (Table 3). Performance is compared across three experimental settings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models — gpt-4-turbo-1106, DeepSeek Coder, StarCoder 2, CodeLLaMa — were all contemporary at the time of writing (late 2023 / early 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the benchmark's collection pipeline components. The three experimental settings (without context, completion, infilling) are separate evaluation conditions, not ablations of the benchmark design."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@k (functional correctness), Recall@k (dependency recall), and Domain-Specific Improvement (DSI) as evaluation metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes a human evaluation where 10 developers (5 annotators + 5 evaluators) assessed auto-generated annotations. Cohen's Kappa of 0.9 is reported. Results in Table 8."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark is designed as a leakage-free test set. Repositories were created after October 2023, post-dating model training cutoffs. CDD verification (Table 3) confirms less than 3% potential leakage."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 breaks down Pass@1 by domain (7 domains). Table 7 shows DSI per domain per model. Table 1 shows domain distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 'Error Analyses' describes manual analysis of 50 error cases of gpt-4: 29 implementation logic errors, 20 missing context cases, and 1 vague requirement. Figure 2 shows a specific failure example."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that gpt-4 underperforms in the Internet domain (Table 6/7), that all LLMs perform poorly without context (7.27% max Pass@1), and that the overall Pass@1 is very low compared to prior benchmarks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: leak rate reduction to 2.18% (Table 3), gpt-4 Pass@1 of 20.74% (Table 4 shows 20.73%), domain-specific findings for gpt-4 and StarCoder 2 (Tables 6-7)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate support: 'We attribute the improvements to the domain knowledge contained in contexts' (Section 3.3) and 'The potential reason for comfort and strange domains is that the pre-training data mix of LLMs is different' (Section 3.4). Neither claim is tested or controlled for confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is generally careful to bound claims to EvoCodeBench-2403 and Python. The Limitations section explicitly states it is 'a monolingual (i.e., Python) benchmark' and notes the small size. Claims are typically prefixed with 'on EvoCodeBench-2403'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations. The large performance drop vs. prior benchmarks is attributed solely to data leakage, without considering that the benchmark may simply be harder due to repo-level complexity, different task distribution, or other factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'EvoCodeBench reveals the actual abilities of these LLMs in real-world repositories' (abstract) but measures only Pass@k on a 275-sample Python benchmark. The gap between 'Pass@k on EvoCodeBench' and 'actual abilities in real-world repositories' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 specifies gpt-4-turbo-1106, gpt-3.5-turbo-1106. Appendix E.1 provides further version details. Open-source models are specified with parameter sizes (DeepSeek Coder-33B/6.7B, StarCoder 2-15B/7B, CodeLLaMa-13B/7B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 3-8 and Appendix E.2 for all settings (without context, completion, infilling, RAG, annotation generation). The fill values come from the released benchmark data, so prompts are fully reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports temperature=0.4, top-p=0.95, max generation length=500, greedy search for k=1, nucleus sampling with 20 samples for k>1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are evaluated directly via prompting without agent loops, tools, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 documents the 4-stage pipeline: repository selection criteria, execution-based filtering, automatic annotations via static analysis and LLM, and benchmark construction. Filtering criteria are specified at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Discussion' contains a substantive 'Limitations' subsection discussing two main limitations: Python-only and small benchmark size."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'EvoCodeBench is a monolingual (i.e., Python) benchmark and ignores other programming languages' and 'the size of EvoCodeBench is currently smaller than some existing benchmarks' due to the recent-repositories-only constraint. They also discuss LLM annotation failure modes (missing details, inaccurate domain labels)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states Python-only scope, 275 samples from 25 repos, first version only covering Oct 2023 - Mar 2024, and that domain distribution may be unbalanced. Future plans for expansion are noted."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is available on GitHub and HuggingFace. All prompts and LLMs' completions are released. Repository URLs are provided in Table 9."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes the 4-stage collection pipeline in detail: repo selection criteria (stars, licenses, recency, tests), function scraping, execution-based filtering, automatic annotations, and benchmark construction."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria are detailed in Section 2.3 (Stage I): open-source Python, permissive licenses, created within last 6 months, non-fork, non-malicious, 50+ stars, having unit tests. Human evaluators are described as having 3+ years Python experience (Section 4)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full 4-stage pipeline is documented in Section 2.3 with filtering criteria at each stage. Table 9 shows per-repository statistics (25 repos → 275 samples). The pipeline from raw repos to final benchmark is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists National Natural Science Foundation of China grants (62192731, 62152730, etc.), National Key R&D Program, and Major Program of Hubei Province."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Peking University, Bytedance, and Alibaba Group. These are prominently displayed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSFC government grants and a government R&D program. The paper evaluates models from OpenAI, DeepSeek, BigCode, and Meta — not products of the funders. The Alibaba/Bytedance-affiliated authors' companies do not have models being evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Authors from Alibaba Group and Bytedance may have commercial interests related to code LLMs but this is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 states 'most code LLM's training data is up to September 2023.' Appendix E.1 provides specific cutoffs: gpt-4-1106 training data up to April 2023, gpt-3.5-turbo-1106 up to September 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 uses the CDD method to detect data leakage. Table 3 reports leakage ratios for all 8 LLMs on EvoCodeBench-2403, finding less than 3% potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark design addresses contamination via temporal splits (repos created Oct 2023 - Mar 2024, after training cutoffs). Section 3.2 validates this with CDD detection. Comparison to HumanEval's 41.47% leakage rate demonstrates improvement."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human subjects study. The human evaluation in Section 4 is quality assurance of annotations, not a study with human participants as subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The human evaluators performed quality checks on annotations, not a human-subjects research study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The evaluators are described as having '3 years of Python development experience' but this is quality assurance, not a study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper evaluates code LLMs on a benchmark, not human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-example inference costs are reported. The paper evaluates 8 LLMs generating 20 samples per requirement but does not quantify the cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 mentions '4 NVIDIA A100-40GB GPUs' but does not state total GPU hours, wall-clock time, or API spend for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed analysis is performed. Results are not reported across multiple random seeds despite using nucleus sampling."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states: 'When k=1, we use the greedy search and generate a single program per requirement. When k>1, we use the nucleus sampling... and sample 20 programs per requirement.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Generation hyperparameters (temperature=0.4, top-p=0.95, max length=500) are stated but no justification is given for these choices. No search budget or tuning process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported for all three experimental settings (without context, completion, infilling) and the RAG setting. No selective reporting of best configurations — all results are shown in Table 4."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many pairwise comparisons across 8 models, 3 settings, and 7+ domains without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate models on it. They do not acknowledge or discuss potential benchmark-creator bias, such as whether benchmark design choices favor certain model types."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes (6.7B to gpt-4) are compared without discussing compute budget differences. No performance-per-compute analysis is presented."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 discusses construct validity by showing EvoCodeBench-2403's code distribution and dependency distribution align with 500 real-world repositories (Table 2). The paper argues the benchmark reflects real-world development scenarios."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are evaluated directly via prompting in controlled settings."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark design uses temporal splits: repositories created October 2023 - March 2024, post-dating all evaluated models' training cutoffs (most up to September 2023). This is the paper's core contribution."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the code context provided in completion/infilling settings could leak answer information. The dramatic difference between without-context (7.27%) and infilling (20.73%) settings suggests context is highly informative, but whether this constitutes feature leakage in some cases is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple samples are drawn from the same repositories (e.g., 59 from litdata, 54 from camp_zipnerf per Table 9), but the paper does not discuss whether samples from the same repository are statistically independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 applies the CDD method (reference [6]) to all 8 LLMs on EvoCodeBench-2403. Table 3 reports detection results showing less than 3% leakage rate."
       }
     }
   },
   "claims": [
     {
       "claim": "EvoCodeBench significantly alleviates data leakage, reducing the potential leak rate from 41.47% (HumanEval) to 2.18%.",
       "evidence": "Table 3 shows CDD detection results: HumanEval leak ratio on gpt-3.5 is 41.47%, while EvoCodeBench-2403 leak ratios range from 0.73% to 2.18% across all 8 LLMs (Section 3.2).",
       "supported": "strong"
     },
     {
       "claim": "The highest Pass@1 of gpt-4 on EvoCodeBench-2403 is only 20.73%, far below prior benchmarks.",
       "evidence": "Table 4 shows gpt-4's Pass@1 in the Local File (Infilling) setting is 20.73. The paper notes this is much lower than 53.04 on the latest repo-level benchmark DevEval (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "LLMs benefit from more code contexts, with gpt-4's Pass@1 improving by 104% and 152% with context.",
       "evidence": "Table 4 shows gpt-4 without context: 7.27, completion: 17.45 (140% improvement), infilling: 20.73 (185% improvement). The stated 104% and 152% appear to be from a different computation base (Section 3.3).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs have distinct comfort and strange domains based on Domain-Specific Improvement (DSI).",
       "evidence": "Table 7 shows DSI values. gpt-4 has comfort domains (DSI>10%) in Database, System, Software Development, Scientific Engineering, and Multimedia but a strange domain in Internet (-28.59%). StarCoder 2-15B is comparable to gpt-4 in Database (Section 3.4).",
       "supported": "weak"
     },
     {
       "claim": "Auto-generated annotations (requirements and domain labels) are comparable to human-written ones in most cases (96.7% and 98.5% respectively).",
       "evidence": "Table 8 shows human evaluation: for requirements, gpt-4 wins 30, ties 236, loses 9. For domain labels: wins 3, ties 268, loses 4. Cohen's Kappa between evaluators is 0.9 (Section 4).",
       "supported": "moderate"
     },
     {
       "claim": "RAG with similar functions improves LLM performance in repo-level code generation.",
       "evidence": "Table 5 shows gpt-4 Pass@1 improves from 8.31 (without context) to 12.29 (similar functions). gpt-3.5 improves from 6.64 to 11.62 (Section 3.3).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny domain sample sizes",
       "detail": "Several domains have very few samples: Security (1), Utilities (2), Communications (8), Text Processing (12). Domain-specific claims (Tables 6-7) for these domains are unreliable. Even the excluded domains with <10 samples are acknowledged, but domains like Text Processing (12 samples) and Internet (15 samples) are still used for DSI analysis."
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "All model comparisons in Tables 4, 5, 6, and 7 are based on point estimates. With 275 samples (and much fewer per domain), small differences between models could easily be due to chance. No statistical tests are performed to validate any comparative claim."
     },
     {
       "flag": "Heavily unbalanced domain distribution",
       "detail": "120/275 samples (43.6%) are in Scientific Engineering. The 'diverse domains' claim is undermined by this extreme skew. Table 1 shows the distribution is far from balanced."
     },
     {
       "flag": "LLM-generated annotations used without full validation",
       "detail": "Requirements and domain labels were generated by gpt-4. While human evaluation shows 96.7% and 98.5% quality, this means ~9 requirements and ~4 domain labels may be incorrect. These errors propagate into evaluation results but their impact on Pass@k scores is not analyzed."
     },
     {
       "flag": "Performance drop attributed primarily to leakage without controlling for task difficulty",
       "detail": "The paper attributes the large performance gap between EvoCodeBench (20.73% Pass@1) and DevEval (53.04% Pass@1) primarily to data leakage. However, EvoCodeBench is a repo-level benchmark with different difficulty characteristics, making the comparison confounded."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval, the primary baseline benchmark compared against for data leakage analysis."
     },
     {
       "title": "DevEval: A manually-annotated code generation benchmark aligned with real-world code repositories",
       "authors": ["Jia Li", "Ge Li", "Yunfei Zhao"],
       "year": 2024,
       "relevance": "Prior repo-level code generation benchmark from the same group; EvoCodeBench extends and addresses its limitations."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2024,
       "relevance": "Repo-level code generation benchmark with dependency analysis; direct comparison point for EvoCodeBench."
     },
     {
       "title": "Evaluating large language models in class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2024,
       "relevance": "ClassEval benchmark for class-level code generation with domain labels, compared in related work."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark using latest competitive programming problems; addresses same data leakage concern at snippet level."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "EvoEval mutates HumanEval to create new benchmarks; addresses contamination at snippet level while EvoCodeBench targets repo level."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu"],
       "year": 2024,
       "relevance": "Provides the CDD contamination detection method used to validate EvoCodeBench's leakage-free status."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming - the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major code LLM evaluated in the benchmark; trained on 2T tokens of code."
     },
     {
       "title": "StarCoder 2 and the Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Code LLM evaluated in the benchmark; showed unexpected strength in specific domains."
     },
     {
       "title": "Code LLaMA: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family evaluated in the benchmark."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark and the unbiased Pass@k estimator used as an evaluation metric."
     },
     {
       "title": "CrossCodeEval: A diverse and multilingual benchmark for cross-file code completion",
       "authors": ["Yangruibo Ding", "Zijian Wang", "Wasi Uddin Ahmad"],
       "year": 2023,
       "relevance": "Cross-file code completion benchmark providing context for repo-level evaluation approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners can use EvoCodeBench to evaluate code LLMs for domain-specific selection, and the domain taxonomy provides actionable guidance."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that gpt-4 achieves only 20.73% Pass@1 (vs 53.04% on DevEval) suggests prior benchmarks may be leaked, though this concern was already growing in the community."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised by this benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit criticism that existing benchmarks (especially HumanEval with 41.47% leakage) are unreliable, but presented diplomatically."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released on GitHub and HuggingFace; researchers can download and evaluate their own models on the benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Peking University and Alibaba, published at NeurIPS; evaluates well-known models (gpt-4, DeepSeek Coder, StarCoder 2)."
     }
   }
 }