ai-research-survey

scan.json (33543B)

---

 {
   "paper": {
     "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Xuanming Zhang",
       "Yihong Dong",
       "Zhi Jin"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2404.00599",
     "doi": "10.48550/arXiv.2404.00599"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EvoCodeBench-2403 reveals a dramatic performance drop for LLMs compared to existing benchmarks: gpt-4's highest Pass@1 is only 20.73% (vs 80% on HumanEval), demonstrating that standalone function benchmarks overestimate real-world coding ability. Code context from local files improves gpt-4's Pass@1 by up to 152%, confirming the importance of repository context. Manual error analysis of 50 failed gpt-4 cases finds 58% are logic errors and 40% are due to missing cross-file context. The benchmark's code and dependency distributions match those observed across 500 real-world repositories.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided in footnote 1: https://github.com/seketeam/EvoCodeBench. The abstract states 'We release EvoCodeBench, all prompts, and LLMs' completions for further community analysis.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "EvoCodeBench-2403 (275 samples from 25 repositories) is released publicly along with all prompts and model completions, as stated in the abstract and Section 1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using pip3 and Pytest for test execution but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions for reproducing the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper itself does not contain step-by-step reproduction instructions. No README with commands, scripts, or 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 4 are point estimates (Pass@k and Recall@k values). No confidence intervals, error bars, or uncertainty measures are reported for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims multiple performance differences between LLMs (e.g., 'gpt-4 achieves the highest Pass@k') and between settings (e.g., '104% and 152% improvement') based solely on comparing numbers, with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: 'Pass@1 of gpt-4 is improved by 104% and 152%' from specific baseline values. Table 4 provides absolute scores allowing readers to compute effect magnitudes across all models and settings."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 275 samples from 25 repositories. No justification is given for why this number is sufficient, no power analysis is provided, and no discussion of whether 275 samples provides adequate statistical power for the comparisons made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported. For k=1, greedy decoding produces a single deterministic output. For k>1, 20 samples are generated per requirement but the unbiased Pass@k estimator yields only a point estimate with no reported spread across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Ten LLMs are compared against each other, and results are contrasted with performance on prior benchmarks (e.g., gpt-4's 88.4% Pass@1 on HumanEval vs 7.27% on EvoCodeBench without context)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include gpt-4-turbo-1106, DeepSeek Coder (Nov 2023), StarCoder 2 (Dec 2023), and Gemma (Feb 2024), all released within months of the paper. These were state-of-the-art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three experimental settings (without context, local file completion, local file infilling) serve as controlled ablations of context availability. Additionally, the RAG experiment (Section 5, Table 6) ablates the effect of retrieved similar functions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: Pass@k for functional correctness (test-case-based) and Recall@k for reference dependency recall. Both are reported at k=1,3,5,10."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 includes human evaluation of auto-generated requirements: two developers write requirements and two others evaluate them vs. gpt-4's output on 50 functions (Cohen's Kappa = 0.92). Section 4.4 includes manual analysis of 50 error cases. Section 5 also includes manual annotation of dependencies in 50 programs to gauge Recall@k bias."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EvoCodeBench-2403 is used purely for evaluation. No model tuning or development decisions are made on this data. The benchmark is collected from repositories created after the training cutoffs of evaluated models."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by program type (standalone vs non-standalone, Figure 4), dependency type (intra-class, intra-file, cross-file, Figure 5), and context setting (three settings in Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 'Error Analyses': 'we manually analyze 50 error cases of gpt-4 in the Local File (Infilling) setting. We found that most of the cases (29 cases) failed due to implementation logic errors. 20 cases failed since the necessary contexts were missing... one case failed because of the vague requirement.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central finding is that LLMs perform poorly: gpt-4's highest Pass@1 is only 20.73%. The paper explicitly frames low performance as a key result and discusses limitations of current LLMs. Section 4.5 summarizes the challenges that remain unsolved."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims gpt-4's highest Pass@1 is 20.73%, supported by Table 4 (Local File Infilling). Claims about alignment with real-world distributions are supported by Table 2 comparing EvoCodeBench-2403 statistics with 500 real repositories (27% standalone, 73% non-standalone, avg 3.46 vs 3.22 dependencies)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims code contexts improve performance ('Pass@1 of gpt-4 is improved by 104% and 152%'). This causal claim is supported by controlled comparison: the same models are evaluated under three conditions (no context, completion, infilling) with only the context variable changing, which constitutes adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims alignment with 'Real-World Code Repositories' and the abstract says results 'reveal the coding abilities of these LLMs in real-world repositories,' but the benchmark is Python-only with English requirements from 25 repositories. The paper acknowledges this in Section 8 but the abstract and title do not qualify the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes performance differences primarily to context availability and model design (instruction tuning vs. standard LM training) but does not consider other potential explanations such as prompt formatting effects, repository difficulty variance, or whether the specific 25 repositories are representative. The observation about GPT family's higher Pass@k but lower Recall@k receives only speculation, not substantive analysis."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@k (functional correctness via test cases) and Recall@k (dependency recall via static analysis). These are presented as specific metrics, not over-framed as broader constructs. The paper discusses the bias in Recall@k explicitly (Section 5), noting the parser may miss runtime-determined dependencies and quantifying the bias at 0.16."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 specifies exact versions: gpt-4-turbo-1106, gpt-3.5-turbo-1106. Open-source models are identified by name and parameter count (e.g., DeepSeek Coder 33B/6.7B, StarCoder 2 15B/7B, CodeLLaMa 13B/7B). Appendix B.1 provides further details including release dates and training data scope."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 6-10 and Appendix B.2, covering all four settings (without context, local file completion, local file infilling, similar functions). The paper also states 'We release... all prompts' via the GitHub repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'k=1, we use the greedy search... k>1, we use the nucleus sampling with a temperature 0.4 and sample 20 programs per requirement. We set the top-p to 0.95 and the max generation length to 500.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are directly prompted with requirements, signatures, and optional code contexts. No tools, retry logic, or multi-step workflows are involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the five-stage pipeline: repository selection (criteria: open-source Python, recent, non-fork, >50 stars, unit tests), function parsing (excluding trivial functions), test construction (pip3 + Pytest), deduplication (Jaccard similarity on code files and imports), and requirement annotation (gpt-4 with few-shot prompts)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' is a dedicated section with six specific limitation points covering monolingual scope, auto-generated requirement quality, Recall@k bias, limited LLMs evaluated, limited context exploration, and hyperparameter sensitivity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: 'EvoCodeBench is a monolingual benchmark (requirements in English and code in Python),' 'auto-generated requirements... may lack necessary details (e.g., hyper-parameters),' Recall@k bias quantified at 0.16, 'limited computing budgets' constrained the number of LLMs evaluated, and 'we do not carefully tune hyper-parameters and prompts.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was not tested: other programming languages, other natural languages, more LLMs, cross-file context extraction methods, and hyperparameter tuning. Each is identified as a specific gap with future work plans."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (275 samples with requirements, reference code, reference dependencies, repositories, and test cases) and all LLM completions are released via the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full collection procedure: GitHub crawling with criteria (open-source Python, created 2023-10 to 2024-2, non-fork, >50 stars, explicit unit tests), function extraction with static analysis, test case extraction using Pytest, deduplication, and LLM-based requirement annotation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The main data source is public GitHub repositories selected by automated criteria. No human participants are recruited for the main study. The small supplementary requirement evaluation (Section 5) involves hired developers but is not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the six-stage pipeline: repository selection → function parsing → test construction → deduplication → requirement annotation → benchmark construction. Table 7 provides details on the 25 selected repositories including creation date, stars, file counts, and sample counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are clearly identified as affiliated with 'School of Computer Science, Peking University.' No product being evaluated is affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding source is disclosed. The authors are academic researchers at Peking University with no apparent commercial stake, but the lack of disclosure prevents verification."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'the latest LLM's training data is up to 2023-9.' Appendix B.1 provides specific cutoffs: gpt-4-1106 training data up to April 2023, gpt-3.5-turbo-1106 training data up to September 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is a core design feature. Section 2.4: 'EvoCodeBench-2403 is collected from real-world repositories that were created from 2023-10 to 2024-2' — after all evaluated models' training cutoffs. The evolving benchmark design is explicitly motivated by avoiding data leakage."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination through temporal separation: repositories were created after training cutoffs. Section 1 lists 'Avoiding Data Leaking' as a key benchmark feature. The evolving update schedule (every ~6 months) is designed to maintain this property as new models emerge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper evaluates LLMs on a code generation benchmark. The small requirement quality evaluation (Section 5) with hired developers is a supplementary validation, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring IRB approval. The paper is a benchmark evaluation of LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study. The supplementary requirement evaluation (Section 5) hires developers but does not characterize them beyond their role."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Data is collected from public GitHub repositories with automated selection criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study. LLMs are evaluated on a fixed benchmark."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with participants who could attrit."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The requirement annotation cost is reported ($0.54 for gpt-4, Table 5), but no inference costs are reported for the main LLM evaluation experiments (20 samples per requirement × 275 samples × 10 models × 3 settings)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or hardware specifications are reported for the main experiments. Section 8 acknowledges 'limited computing budgets' but does not quantify them."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "For k=1, greedy decoding is deterministic (no seed variance). For k>1, nucleus sampling with temperature 0.4 is used to generate 20 samples, but no analysis of sensitivity to random seeds or across independent runs is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'When k=1, we use the greedy search and generate a single program per requirement. When k>1, we use the nucleus sampling... and sample 20 programs per requirement.' The number of generated programs is explicit."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 explicitly states: 'Due to the limited computing budgets, we do not carefully tune hyper-parameters and prompts.' No search budget is reported because no search was conducted, but the lack of tuning is transparently acknowledged."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a single fixed configuration for all models (temperature=0.4, top-p=0.95, max_length=500) without selecting a 'best' configuration. This uniform approach avoids cherry-picking, and the paper states: 'We ensure all LLMs are evaluated under the same experimental settings.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Ten LLMs are compared across three settings and four k values with no statistical tests performed at all, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate LLMs on it. They do not discuss potential biases in benchmark construction (e.g., whether the 25 selected repositories or 275 samples favor certain model architectures or coding patterns)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B to 33B parameters and closed-source models of unknown size are compared without discussion of compute costs. No performance-vs-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses what makes a good benchmark (Section 1, Table 1) and validates EvoCodeBench's construct validity by comparing code/dependency distributions against 500 real-world repositories (Table 2: 27% standalone matches real-world 27%, avg 3.46 dependencies vs 3.22). The contrast with HumanEval (100% standalone) directly addresses validity gaps."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are prompted directly with the same prompt templates. The evaluation is at the model level, not at a tool/scaffold level."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Core design feature: 'EvoCodeBench-2403 is collected from real-world repositories that were created from 2023-10 to 2024-2,' explicitly after the latest model's training cutoff (2023-9). The evolving benchmark design ensures temporal separation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The local file settings provide code context from the same file as the target function. The paper does not discuss whether this context might inadvertently contain patterns that leak information about the expected solution (e.g., similar functions in the same file that closely match the reference code)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 step 4: 'we perform repository-level deduplication based on the Jaccard similarities in code files and imports. The former removes duplicate repositories in text surfaces, and the latter removes repositories in domains that are too similar.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses temporal splits as a concrete leakage prevention method: repositories are selected based on creation dates after model training cutoffs. The automatic pipeline for updating EvoCodeBench from latest repositories is designed to maintain this temporal barrier for future model evaluations."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4's highest Pass@1 on EvoCodeBench-2403 is only 20.73%, compared to 80% on HumanEval.",
       "evidence": "Table 4 shows gpt-4 achieves Pass@1 of 20.73% in Local File (Infilling), 17.45% in Local File (Completion), and 7.27% without context. HumanEval comparison cited in Section 4.4.",
       "supported": "strong"
     },
     {
       "claim": "Code contexts from local files improve gpt-4's Pass@1 by 104% (completion) and 152% (infilling) over the no-context setting.",
       "evidence": "Table 4: gpt-4 Pass@1 goes from 7.27% (no context) to 17.45% (completion, +140% actually) and 20.73% (infilling, +185% actually). The paper states 104% and 152% in Section 4.4, though the math from Table 4 yields different percentages — the paper may be using different base numbers or rounding.",
       "supported": "moderate"
     },
     {
       "claim": "EvoCodeBench-2403's code distribution (27% standalone, 73% non-standalone) is consistent with that of 500 real-world repositories.",
       "evidence": "Table 2 shows EvoCodeBench-2403 has 27% standalone / 73% non-standalone, matching exactly the distribution of 500 real-world repositories (also 27% / 73%). Average dependencies: 3.46 vs 3.22.",
       "supported": "strong"
     },
     {
       "claim": "Most LLM failures on EvoCodeBench are due to logic errors (58%) and missing contexts (40%).",
       "evidence": "Section 4.4: manual analysis of 50 gpt-4 error cases in Local File (Infilling) found 29 logic errors, 20 missing context cases, and 1 vague requirement case.",
       "supported": "moderate"
     },
     {
       "claim": "Auto-generated requirements (by gpt-4) are comparable to human-written requirements in 92% of cases.",
       "evidence": "Table 5 and Section 5: On 50 randomly selected functions, gpt-4 and human developers tied on 41 functions, gpt-4 won 5, humans won 4. Cohen's Kappa between two evaluators is 0.92.",
       "supported": "moderate"
     },
     {
       "claim": "Retrieval-augmented generation with similar function names improves Pass@1 for gpt-4 from 8.31% to 12.29%.",
       "evidence": "Table 6 shows gpt-4 Pass@1 improves from 8.31% (without context) to 12.29% (similar functions), and gpt-3.5 from 6.64% to 11.62%. Recall@1 shows larger gains.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "All comparisons across 10 LLMs and 3 settings rely on point estimates. No error bars, confidence intervals, or significance tests are reported despite making numerous comparative claims (e.g., 'gpt-4 achieves the highest Pass@k'). With 275 samples, random variation could account for some observed differences."
     },
     {
       "flag": "Small benchmark with potential selection bias",
       "detail": "275 samples from only 25 repositories. While the code/dependency distributions match real-world statistics, 25 repositories from a 5-month window (Oct 2023 - Feb 2024) with >50 stars represent a narrow slice of Python development. The selection criteria (high stars, good tests) may bias toward well-maintained projects unrepresentative of typical repositories."
     },
     {
       "flag": "Percentage improvement calculation appears inconsistent",
       "detail": "The paper states gpt-4's Pass@1 'is improved by 104% and 152%' from without-context to completion and infilling settings. From Table 4: 7.27% → 17.45% is actually a 140% increase, and 7.27% → 20.73% is a 185% increase. The stated figures may use different baseline numbers but the discrepancy is unexplained."
     },
     {
       "flag": "No compute costs reported for main experiments",
       "detail": "Evaluating 10 LLMs × 275 samples × 20 generations (for k>1) × 3 settings represents substantial compute. No API costs, GPU hours, or wall-clock times are reported, making it impossible to assess the practical feasibility of the evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Introduced HumanEval, the most widely used code generation benchmark and primary comparison point for EvoCodeBench's claim of improved alignment with real-world coding."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Repository-level repair benchmark that focuses on issue resolution rather than code generation, contrasted with EvoCodeBench's code generation focus."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Most directly comparable prior work: a code generation benchmark from real repositories with non-standalone programs, but lacking real code distribution and robust dependency metrics."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark with 100 hand-crafted Python classes, compared as a predecessor that lacks real-world repository alignment."
     },
     {
       "title": "CrossCodeEval: A diverse and multilingual benchmark for cross-file code completion",
       "authors": ["Yangruibo Ding", "Zijian Wang"],
       "year": 2023,
       "arxiv_id": "2310.11248",
       "relevance": "Cross-file code completion benchmark addressing similar repository-level challenges but focused on completion rather than generation from requirements."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": ["Tianyang Liu", "Canwen Xu"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "relevance": "Repository-level code completion benchmark, contrasted with EvoCodeBench's code generation focus and comprehensive annotations."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the evaluated code LLMs, achieving competitive results on EvoCodeBench (Pass@1 19.64% for 33B in infilling setting)."
     },
     {
       "title": "StarCoder 2 and the Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Evaluated code LLM trained on Stack v2; its training data cutoff is used to determine EvoCodeBench's temporal boundary for contamination avoidance."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Evaluated code LLM series; performance on EvoCodeBench demonstrates the gap between standalone benchmark scores and real-world repository coding."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark and unbiased Pass@k estimator used as the primary evaluation metric in EvoCodeBench."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle"],
       "year": 2023,
       "relevance": "Prior work on extracting repository contexts for code generation, inspiring EvoCodeBench's context extraction approach."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "relevance": "Repository-level code completion via iterative retrieval, related to EvoCodeBench's RAG experiments with similar function names."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released benchmark and evaluation framework can be used by researchers and practitioners to evaluate code LLMs on realistic tasks, though it requires significant setup."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that gpt-4 drops from 80% to 20.73% on real-world code is notable but broadly expected by practitioners who know HumanEval is easy."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; purely a benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly argues that existing benchmarks like HumanEval are misleading about LLM coding abilities, but presents this as a gap to fill rather than a controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released on GitHub; researchers can download and run evaluations, though it requires setting up repositories and test environments."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Peking University (well-known in CS); evaluates recognizable models (gpt-4, DeepSeek Coder) but not from a major AI lab."
     }
   }
 }