ai-research-survey

scan.json (30531B)

---

 {
   "paper": {
     "title": "Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM",
     "authors": [
       "Chunqiu Steven Xia",
       "Yinlin Deng",
       "Lingming Zhang"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2403.19114",
     "doi": "10.48550/arXiv.2403.19114"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EvoEval, a suite of 828 benchmark problems evolved from HumanEval using GPT-4, reveals an average 39.4% performance drop across 51 LLMs compared to standard HumanEval, with per-model drops ranging from 19.6% to 47.7% and significant ranking changes among top models. Instruction-following LLMs are particularly brittle to rephrasing and subtle changes (7.6% drop vs <1% for base models on SUBTLE), while all models struggle with compositional generalization — the best model (GPT-4) can only compose 53.8% of individually-solved subproblems. The findings suggest potential overfitting to existing benchmarks and highlight gaps in tool-use capabilities of instruction-following models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We have open-sourced our benchmarks, tools, and complete LLM generations at https://github.com/evo-eval/evoeval' in both the abstract and conclusion."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks (828 problems with test cases and groundtruths) and complete LLM generations are released at the same GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions using vLLM for some models and building on EvalPlus but doesn't specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. It describes the methodology and points to the repository but lacks a dedicated 'Reproducing Results' section with specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 2 and throughout are reported as point estimates (pass@1 scores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'LLMs consistently perform worse', ranking changes) based solely on comparing pass@1 numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports percentage drops with baseline context, e.g., 'GPT-4 solve close to 85% of all HUMANEVAL problems but fall almost below 50% pass@1 when evaluated on the DIFFICULT problems' and 'on average 39.4% decrease' with per-benchmark breakdowns (Section 5.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 100 problems per semantic-altering benchmark or 164 for semantic-preserving benchmarks. No power analysis or discussion of whether these sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses greedy decoding (temperature=0) producing single deterministic results per problem. No variance across runs, seeds, or sampling strategies is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and HumanEval+ serve as baselines throughout. All EvoEval performance is compared against these established benchmarks (Table 2, Figures 4-7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 51 evaluated LLMs include the most contemporary models at time of writing: GPT-4-Turbo, Claude-3, Gemini, DeepSeek-Coder, StarCoder2 (Table 5)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like comparisons: TOOL USE vs TOOL USE-MAIN ONLY (Section 5.4, isolating helper function effects), COMBINE vs COMBINE-NAIVE (Section 5.2, isolating composition complexity), and DECOMPOSE analysis (Section 5.3). Each transformation type tests a specific capability dimension."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Beyond pass@1, the paper reports composition percentage (Table 3), decomposition percentage and recomposition percentage (Table 4), and normalized pass@1 for ranking analysis (Figure 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through differential testing against groundtruth outputs (Section 4, Appendix A.2). No human evaluation of LLM-generated code quality or correctness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EvoEval benchmarks are newly generated problems that were not available during any model's training, by design serving as held-out test data. The paper explicitly argues this avoids contamination from existing benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows per-benchmark breakdown for all 51 LLMs across 5 semantic-altering benchmarks. Figures 4-7 provide additional per-model, per-benchmark visualizations. Tables 3-4 provide detailed composition/decomposition breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C (Figures 21-27) shows concrete GPT-4 failure examples for each benchmark type with detailed analysis of why the generated solution is incorrect."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core finding IS a negative result: significant performance degradation on EvoEval vs HumanEval. Specific negative findings include instruction-following models' brittleness to rephrasing, poor compositional generalization (<54% for best model), and poor tool-use integration."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — 'significant drop in performance (on average 39.4%)' supported by Table 2, 'decrease can range from 19.6% to 47.7%' supported by per-model analysis, 'drastic ranking changes' supported by Figure 5, 'brittleness of instruction-following models' supported by Figure 7 and Section 5.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims about 'potential overfitting' and 'indicating potential memorization or contamination of prior evaluation benchmarks' (Section 5.1). While the instruction-following vs base model comparison on SUBTLE provides suggestive evidence, alternative explanations (e.g., format sensitivity unrelated to memorization, GPT-4 generation bias) are not discussed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only Python problems derived from HumanEval but makes broad claims about 'coding proficiency' and 'program synthesis ability of LLMs' (title and abstract). The scope is not bounded to Python or HumanEval-style function synthesis."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes performance drops to data leakage and overfitting without considering alternatives: evolved problems may be inherently harder in ways unrelated to leakage, GPT-4-generated problems may have systematic biases favoring/disfavoring certain models, or instruction-following sensitivity may reflect format calibration rather than memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@1 (functional correctness on test cases) but frames results as measuring 'coding proficiency' and 'program synthesis ability' (title, abstract). No discussion of the gap between passing specific test cases and actual coding proficiency."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix A.1) provides exact model IDs for all 51 models, including API endpoint names (e.g., 'gpt-4-0125-preview', 'claude-3-opus-20240229') and HuggingFace model names (e.g., 'deepseek-ai/deepseek-coder-33b-instruct')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides the exact transformation prompts for all 7 benchmarks plus refinement and I/O fixing prompts (Figures 11-20). Appendix A.2 shows an example input prompt for GPT-4 (Figure 10) and describes the input format for base vs instruction-following models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'greedy decoding (i.e., producing a deterministic sample per each problem with temperature = 0).' Appendix A.2 reports evaluation timeout settings (Tmax=1000ms, f=4) and exact-match thresholds (10^-6 for floating point)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is simple zero-shot prompting — models receive a function header with docstring and autocomplete (base models) or generate a complete solution (instruction-following models)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the full pipeline: targeted transformation → problem refinement with self-consistency → test generation → manual examination and test augmentation. The sanitization script for parsing LLM outputs is described in Appendix A.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated limitations or threats-to-validity section. The conclusion mentions future work directions but does not substantively discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no consideration of threats such as GPT-4 bias in problem generation, Python-only scope, or the limited problem origin (all from HumanEval)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no boundary statements about the limitations of Python-only testing, HumanEval-derived problems, or the generalizability of the findings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Complete LLM generations, benchmarks, and tools are released at the GitHub repository, enabling independent verification of all reported results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes in detail how benchmark problems were generated from HumanEval seeds using GPT-4 with targeted transformation prompts, including the refinement and self-consistency pipeline."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are the standard HumanEval benchmark and GPT-4-generated transformations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 2 and Figure 2: original problem → targeted transformation → refinement with self-consistency → manual examination → test augmentation. Table 1 shows problem counts and statistics at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The acknowledgment section (Section 8) thanks individuals for help and feedback but mentions no grants or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from University of Illinois Urbana-Champaign with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding cannot be assessed. The paper does not state whether the work was unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 51 evaluated LLMs. While EvoEval is designed to mitigate contamination, the training cutoffs are not explicitly reported."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is motivated by train/test overlap concerns. Section 1 discusses how 'example solutions can be readily found on the web and thus potentially in training data' and cites work showing 'substantial overlap between benchmark solutions and open-source training corpuses' [39]."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "EvoEval is specifically designed to address benchmark contamination by creating new problems that did not exist before. The paper argues these evolved problems cannot have been seen during training (Section 1). Results on SUBTLE further demonstrate contamination effects."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported despite evaluating 51 LLMs across 828+ problems, including commercial API calls to GPT-4, Claude-3, and Gemini."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no GPU hours, total API spend, or hardware specifications for running inference on the open-source models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only greedy decoding (temperature=0) is used, producing single deterministic results. No exploration of sensitivity to temperature, sampling strategy, or random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states 'greedy decoding (i.e., producing a deterministic sample per each problem with temperature = 0)', making clear each result comes from a single run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is discussed. Temperature=0 is used without justification for why this setting was chosen or whether other settings were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding is used as the sole configuration without justification for why it was selected over other temperature or sampling settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in the paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 is used to generate the benchmark problems and is also evaluated on them, creating a potential bias. The paper does not acknowledge or discuss this conflict — GPT-4-generated problems might systematically favor or disfavor GPT-4 models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models ranging from 2b to 70b+ parameters and proprietary models of unknown size without discussing compute differences. Performance is not analyzed relative to compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core thesis addresses construct validity: it argues existing benchmarks don't validly measure coding proficiency due to contamination and limited scope. The t-SNE visualization (Figure 3) demonstrates distributional differences, and multiple transformation types test different capability dimensions."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated via direct prompt-to-completion without any agentic framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core motivation is temporal leakage: 'prior benchmarks contain only a very limited set of problems... many benchmarks are prone to data leakage where example solutions can be readily found on the web' (Section 1). EvoEval is designed to create post-training problems to mitigate this."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage is not explicitly discussed. The paper doesn't address whether information in problem descriptions (derived from HumanEval) could provide signal through textual similarity to training data, beyond the direct contamination concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether evolved problems maintain structural similarity to HumanEval originals that could enable transfer from training data. All EvoEval problems derive from HumanEval seeds, creating potential non-independence between training examples and test problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. While the paper cites [39] on quantifying contamination and designs EvoEval to prevent leakage, it does not use canary strings, membership inference, n-gram overlap analysis, or other detection methods to verify that EvoEval is actually contamination-free."
       }
     }
   },
   "claims": [
     {
       "claim": "Compared to high performance on standard benchmarks like HumanEval, there is a significant drop in performance (on average 39.4%) when using EvoEval across 51 LLMs.",
       "evidence": "Table 2 shows per-benchmark average drops: DIFFICULT 58.7%, CREATIVE 50.2%, SUBTLE 5.0%, COMBINE 78.1%, TOOL USE 4.9%. Per-model drops range from 19.6% to 47.7% (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "EvoEval causes drastic ranking changes among top-performing LLMs compared to HumanEval leaderboards.",
       "evidence": "Figure 5 shows ranking changes across benchmarks. While top models differ by <10% on HumanEval, the difference on EvoEval averages over 20%. GPT-4 beats GPT-4-Turbo on DIFFICULT, CREATIVE, and COMBINE despite lower HumanEval scores (Table 2, Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Instruction-following LLMs are significantly more sensitive to subtle changes and rephrasing of problem descriptions than base models.",
       "evidence": "Figure 7 shows instruction-following LLMs drop on average 3.4% (VERBOSE) and 4.0% (CONCISE) while base models improve by 0.5% and 2.1%. On SUBTLE, instruction-following LLMs drop 7.6% vs <1% for base models (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Current state-of-the-art LLMs fail to effectively compose known programming concepts to solve combined problems.",
       "evidence": "Table 3 shows composition percentages: GPT-4 achieves only 53.8%, GPT-4-Turbo 48.1%, Claude-3 43.2%. On COMBINE-NAIVE (simpler sequential composition), best is GPT-4 at 75.2% (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Base (non-instruction-following) models benefit more from auxiliary helper functions than instruction-following models.",
       "evidence": "Section 5.4 reports 122.0% average improvement for base models vs 60.4% for instruction-following models when helper functions are provided (TOOL USE vs TOOL USE-MAIN ONLY). Figure 9 shows cases where base models outperform instruction-following counterparts with helpers.",
       "supported": "moderate"
     },
     {
       "claim": "High scores on existing benchmarks are potentially due to overfitting or data leakage rather than genuine coding capability.",
       "evidence": "Supported by the SUBTLE experiment showing instruction-following models drop significantly on minimally-changed problems, and by citing [39] showing overlap between benchmark solutions and training corpora. However, alternative explanations are not considered (Section 1, Section 5.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "GPT-4 used to generate benchmark AND evaluated on it",
       "detail": "GPT-4 is used to generate all EvoEval problems, groundtruth solutions, and test cases. GPT-4 and GPT-4-Turbo are then evaluated on these problems. This creates a potential systematic bias — problems generated by GPT-4 may favor or disfavor GPT-4's reasoning patterns in ways not applicable to other model families. The paper does not acknowledge this conflict."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across 51 LLMs and 7 benchmarks rely on raw pass@1 point estimates. Claims of 'significant drop' and 'drastic ranking changes' are stated without any statistical test. With 100 problems per benchmark, small ranking differences may not be statistically meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations, threats to validity, or scope boundaries. Key unaddressed limitations include Python-only evaluation, all problems derived from a single seed benchmark (HumanEval), and potential GPT-4 generation bias."
     },
     {
       "flag": "Broad claims from narrow evaluation scope",
       "detail": "The paper draws broad conclusions about 'coding proficiency' and 'program synthesis ability' from Python-only, function-level problems all derived from HumanEval. These are framed as general findings about LLM capabilities without bounding to the tested domain."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark that EvoEval evolves from and that dominates LLM code evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP, another widely-used code generation benchmark discussed as suffering from the same contamination issues as HumanEval."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus/HumanEval+ with augmented test suites for more rigorous code generation evaluation, directly built upon by EvoEval."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Real-world software engineering benchmark that addresses some limitations of HumanEval by using real GitHub issues."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "relevance": "Another contamination-free code benchmark using live programming contest problems, addressing similar concerns as EvoEval."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "arxiv_id": "2403.04811",
       "relevance": "Directly quantifies the overlap between benchmark solutions and training corpora, providing evidence for the contamination problem EvoEval addresses."
     },
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["Simone Balloccu", "Patrícia Schmidtová", "Mateusz Lango", "Ondřej Dušek"],
       "year": 2024,
       "arxiv_id": "2402.03927",
       "relevance": "Documents data contamination and evaluation malpractice in closed-source LLMs, including deliberate benchmark gaming."
     },
     {
       "title": "Magicoder: Source code is all you need",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Proposes OSS-Instruct for synthesizing instruction data from code, relevant to understanding how training data composition affects benchmark performance."
     },
     {
       "title": "WizardCoder: Empowering code large language models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Uses Evol-Instruct to create complex training data for code LLMs — a training-time approach paralleling EvoEval's test-time evolution approach."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major open-source code LLM family evaluated extensively in EvoEval across multiple model sizes and instruction variants."
     },
     {
       "title": "Code LLama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family evaluated in EvoEval, showing significant instruction-following sensitivity."
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "Extends HumanEval and MBPP to 18 languages, addressing the language diversity limitation that EvoEval also indirectly highlights."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Benchmark for class-level code generation, addressing the function-level limitation of HumanEval that EvoEval also builds upon."
     }
   ]
 }