ai-research-survey

scan.json (30414B)

---

 {
   "paper": {
     "title": "On Leakage of Code Generation Evaluation Datasets",
     "authors": [
       "Alexandre Matton",
       "Tom Sherborne",
       "Dennis Aumiller",
       "Elena Tommasone",
       "Milad Alizadeh",
       "Jingyi He",
       "Raymond Ma",
       "Maxime Voisin",
       "Ellen Gilsenan-McMahon",
       "Matthias Gallé"
     ],
     "year": 2024,
     "venue": "Conference on Empirical Methods in Natural Language Processing",
     "arxiv_id": "2407.07565",
     "doi": "10.48550/arXiv.2407.07565"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "HumanEval and MBPP are pervasively contaminated in LLM training data through three mechanisms: direct data leakage (every HumanEval prompt appears 43+ times on GitHub), indirect leakage through synthetic data pipelines (high embedding similarity between evol-instruct/Starcoder-Instruct and HumanEval/MBPP), and checkpoint selection overfitting. The authors release LBPP, a 161-problem uncontaminated Python benchmark on which SOTA models score up to 43% lower than HumanEval, with model rankings changing between contaminated and uncontaminated benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository is provided. Only the LBPP dataset is released on HuggingFace. No evaluation harness, embedding analysis scripts, or reproduction code is linked."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "LBPP is released at https://huggingface.co/datasets/CohereForAI/lbpp as stated in the abstract and Section 4."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency information is provided for reproducing the evaluation or analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the benchmark evaluations, GitHub search analysis, or embedding similarity experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports Pass@1 as single point estimates for all 28 models across 3 benchmarks. No confidence intervals or error bars are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ranking changes between benchmarks and performance drops of up to 43% without any statistical significance tests. The correlation in Fig. 3 is described as 'strong significant' but no test statistics or p-values are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Raw performance numbers are given for all models on all benchmarks (Table 2), with rank changes and absolute differences. The abstract states 'up to 43% worse' and Section 3.2 reports specific deltas (+9% HumanEval, +2% MBPP, unchanged LBPP) from fine-tuning."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "LBPP contains 161 prompts with no justification for this size. No power analysis or discussion of whether 161 problems are sufficient to reliably distinguish model capabilities."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All Pass@1 results are single-run point estimates. No standard deviation, interquartile range, or variance across runs is reported for any model or benchmark."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares 28 models across 5 families (Mistral, Meta, OpenAI, Anthropic, Qwen, Cohere, Deepseek, Databricks) on HumanEval, MBPP, and LBPP, serving as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include GPT-4o, Claude 3.5 Sonnet, Llama3 70B, Codestral, and Qwen 2—all contemporary at time of publication (2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "LBPP is a benchmark, not a multi-component system. There are no components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is reported. No other metrics (e.g., Pass@10, code quality metrics, or diversity measures) are used for any benchmark."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through unit test pass/fail. While human annotators created LBPP and reviewed prompts, no human evaluation of model outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LBPP is explicitly designed as a held-out, uncontaminated test set. The paper's core contribution is creating a benchmark that was not in any model's training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 provides a per-category error analysis: 21% 2D/3D array problems, 18% graph algorithms, 17% complex programming concepts, 8% bit arithmetic, 8% Pandas, 8% file IO."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 analyzes common failure patterns between Claude 3.5 Sonnet and Command R Refresh. Table 1 shows specific unsolved problems. Failure categories are enumerated."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper is fundamentally about negative results: contamination undermines benchmarks. Section 3.2 reports that fine-tuning with evol-instruct leaves LBPP unchanged despite boosting HumanEval +9%. Section 5 acknowledges LBPP will eventually become contaminated too."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about three contamination sources (direct, synthetic, checkpoint selection) are each supported with evidence in Sections 3.1, 3.2, and 3.3. The claim about 'up to 43% worse' on LBPP is supported by Table 2 (Mistral Large: 0.92 HumanEval vs 0.50 LBPP = 42pp drop)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims contamination causes inflated benchmark scores, but the evidence is primarily observational. The fine-tuning experiment (Section 3.2, evol-instruct) is a single model/dataset pair. The core difficulty confound—LBPP is intentionally harder—is not controlled for, making it impossible to separate contamination effects from difficulty effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'Code Generation Evaluation Datasets' generally, and the conclusion states 'contamination is likely unavoidable at the LLM scale,' but the analysis covers only HumanEval, MBPP, and two synthetic datasets. No other code generation benchmarks (APPS, CodeContests, SWE-bench) are analyzed."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly considers whether high similarity between synthetic data and benchmarks is due to 'synthetic data filling the space of problems similar to HumanEval/MBPP or more direct leakage.' The paper presents three distinct hypotheses for contamination rather than a single explanation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central argument is that Pass@1 on contaminated benchmarks is a poor proxy for code generation capability. They explicitly argue LBPP provides 'a less biased measure' and note that HumanEval/MBPP 'cannot be used as the only proxies to evaluate a model's code abilities' (Section 3.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists marketing names (GPT-4o, Claude-3.5-Sonnet, Llama3 70B Instruct) without API versions or snapshot dates. No model is specified with a version string like 'gpt-4-0613'. The schema requires exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 describes 'zero-shot' evaluation but the exact prompt format used to evaluate models is not shown. Table 4 shows partial prompts used for the memorization test, but the standard evaluation prompt template is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the 28 model evaluations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on code completion tasks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the LBPP creation pipeline: annotators with competitive programming experience create problems, model-in-the-loop filtering, about one-third disqualified, manual review by authors. Section 3.1 describes the GitHub keyword search methodology. Section 3.2 describes the embedding similarity analysis using Cohere embed v3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section discussing black-box analysis constraints and the inevitability of future LBPP contamination."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states specific limitations: 'All the model analysis was done black-box, without inspecting the model weights or the training set' and acknowledges LBPP 'will follow the same path' as HumanEval/MBPP regarding future contamination."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly bounds what the analysis does NOT show: they did not inspect model weights or training sets (except synthetic data analysis). They acknowledge they cannot definitively prove contamination, only present supporting evidence."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While LBPP is released on HuggingFace, the underlying analysis data is not: raw GitHub search results, embedding vectors, model outputs for Table 2, and fine-tuning experiment data are all unavailable for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes LBPP annotation in detail: annotators created novel problems, inspired by textbooks not freely available online, with model-in-the-loop filtering and manual review. Section 3.1 describes GitHub search methodology. Section 3.2 describes embedding comparison process."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Annotators are described as having 'competitive programming experience' and being 'paid above minimum wage,' but recruitment channels and selection process are not described. All authors are from Cohere, suggesting annotators may be employees/contractors, but this is not stated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The LBPP pipeline is documented: creation → model-in-the-loop filtering → review → ~1/3 disqualified → final manual review by authors. The contamination analysis pipeline is also documented: GitHub keyword search, embedding with Cohere embed v3, nearest-neighbor similarity comparison."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present. All authors are from Cohere but no corporate funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list Cohere as their affiliation on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work is produced by Cohere, which has commercial interest in (a) promoting its own LBPP benchmark, (b) demonstrating that competitors' benchmark scores may be inflated by contamination, and (c) evaluating its own Command R models alongside competitors. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. The authors work for Cohere which sells competing LLM products evaluated in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Despite the paper being about benchmark contamination, no training cutoff dates are stated for any of the 28 evaluated models. This makes it impossible to precisely determine which models could have seen which benchmark data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is the central topic of the paper. Section 3.1 quantifies HumanEval presence on GitHub (Fig. 1). Section 3.2 analyzes embedding similarity between training and test data (Fig. 2). Table 4 shows apparent memorization of HumanEval solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the paper's primary focus. Three contamination mechanisms are analyzed across Sections 3.1-3.3, and LBPP is designed specifically as an uncontaminated alternative."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. LBPP annotators created the benchmark but were not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency data reported despite evaluating 28 models across 3 benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget stated for model evaluations, the fine-tuning experiment, the embedding similarity analysis, or the GitHub search analysis."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds. All Pass@1 numbers in Table 2 appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is never stated. It is unclear whether Pass@1 values are from 1 sample or averaged over multiple generations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported for the fine-tuning experiment (Command R + evol-instruct) or any model evaluation configuration."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how model configurations were selected for evaluation (default settings? specific versions?). The fine-tuning experiment configuration is not justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Cohere employees evaluate Cohere models (Command R, R+, R Refresh, R+ Refresh) alongside competitors and promote Cohere's LBPP benchmark without acknowledging self-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B parameters to unknown large sizes are compared in Table 2 without any discussion of compute requirements or performance per unit compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central thesis is that HumanEval and MBPP lack construct validity due to contamination. Section 3.2 explicitly argues these benchmarks 'cannot be used as the only proxies to evaluate a model's code abilities' and Section 3.3 discusses overfitting to narrow metrics."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are evaluated directly on code completion."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses how HumanEval (published 2021) and MBPP predated most evaluated models' training, and Section 3 describes how benchmark data accumulates in training corpora over time through replication and synthetic data generation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether evaluation setups might leak answer information through context (e.g., whether prompt formats or test harness details provide hints unavailable in real usage)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Fig. 2 directly analyze the non-independence of training data (evol-instruct, Starcoder-Instruct) and test data (HumanEval, MBPP) through embedding similarity analysis, showing high cosine similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Multiple concrete detection methods are used: GitHub keyword search for direct contamination (Fig. 1), embedding cosine similarity for semantic overlap (Fig. 2), partial-prompt memorization tests on a commercial model (Table 4), and qualitative prompt matching (Tables 3, 5)."
       }
     }
   },
   "claims": [
     {
       "claim": "HumanEval prompts are pervasively present in public code repositories, with every prompt appearing at least 43 times on GitHub (median 99).",
       "evidence": "Fig. 1 shows a histogram of GitHub code search hits for all HumanEval prompts. Minimum is 43, median is 99, mean is 130.2 ± 153.7.",
       "supported": "strong"
     },
     {
       "claim": "Synthetic code training datasets have high semantic similarity to HumanEval and MBPP, suggesting indirect data leakage.",
       "evidence": "Fig. 2 shows cosine similarity histograms between HumanEval/MBPP and evol-instruct/Starcoder-Instruct. Table 5 shows near-identical prompt pairs. LBPP has uniformly lower similarity to these training datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Leading models perform up to 43% worse on the uncontaminated LBPP benchmark compared to HumanEval.",
       "evidence": "Table 2 shows Mistral Large drops from 0.92 (HumanEval) to 0.50 (LBPP), a 42 percentage-point drop. All 28 models perform worse on LBPP.",
       "supported": "moderate"
     },
     {
       "claim": "Model rankings change between contaminated benchmarks (HumanEval/MBPP) and the uncontaminated LBPP benchmark.",
       "evidence": "Table 2's rightmost column shows rank changes. For example, Claude-3-Haiku is rank 8 on HumanEval but rank 14 on LBPP. Mistral Large is rank 1 on HumanEval but rank 5 on LBPP.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning on evol-instruct increases HumanEval by +9% and MBPP by +2% but leaves LBPP unchanged, indicating differential contamination.",
       "evidence": "Section 3.2 reports this result for Command R Refresh. Only a single model/dataset pair, no variance or repeated trials.",
       "supported": "weak"
     },
     {
       "claim": "A major commercial model can reproduce exact HumanEval gold solutions from under-specified partial prompts, demonstrating memorization.",
       "evidence": "Table 4 in Appendix A shows four examples where a commercial system completes partial HumanEval prompts with solutions matching the gold standard exactly, despite deliberately ambiguous prompts.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest: company evaluating own models",
       "detail": "All authors are Cohere employees. The paper evaluates 4 Cohere models alongside competitors and promotes Cohere's LBPP benchmark. Cohere benefits commercially from showing competitors' scores may be inflated and from establishing LBPP as a standard benchmark."
     },
     {
       "flag": "No variance or error bars on any result",
       "detail": "All 84 Pass@1 values in Table 2 (28 models × 3 benchmarks) are single point estimates. With no variance, it is impossible to know whether observed ranking changes are meaningful or within run-to-run noise."
     },
     {
       "flag": "Difficulty confound not controlled",
       "detail": "LBPP is intentionally designed to be harder than HumanEval/MBPP ('adversarial collection resulted in more difficult problems'). Performance drops could be entirely explained by difficulty rather than contamination. The paper cannot separate these effects."
     },
     {
       "flag": "Single metric only",
       "detail": "Only Pass@1 is reported. No Pass@k for other k values, no code quality metrics, no partial credit. This limits the sensitivity of the comparison and may miss important performance patterns."
     },
     {
       "flag": "Anonymized evidence for strongest claim",
       "detail": "Table 4 demonstrates exact memorization of HumanEval solutions by a 'popular commercial system (kept anonymous).' Anonymizing the model prevents verification and limits the scientific value of this evidence."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, one of the two primary contaminated benchmarks studied in this paper."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP, the other primary contaminated benchmark studied in this paper."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Proposes continuously updated code evaluation to combat contamination, closely related to LBPP's motivation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Proposes a more challenging software engineering benchmark as an alternative to simpler code generation benchmarks."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "arxiv_id": "2403.04811",
       "relevance": "Directly quantifies code benchmark contamination in pretraining datasets, finding 12.2% of HumanEval in The Pile and 18.9% in The Stack."
     },
     {
       "title": "Starcoder: may the source be with you",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major code LLM whose decontamination is analyzed; shows limitations of exact-match decontamination for code."
     },
     {
       "title": "WizardLM: Empowering large pre-trained language models to follow complex instructions",
       "authors": ["Can Xu", "Qingfeng Sun", "Kai Zheng"],
       "year": 2023,
       "relevance": "Source of evol-instruct, a widely-used synthetic code dataset shown to have high similarity to HumanEval/MBPP."
     },
     {
       "title": "Magicoder: Source code is all you need",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Uses synthetic data pipeline for code LLM training; authors reported high similarity between evol-instruct and HumanEval."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang"],
       "year": 2024,
       "relevance": "Proposes additional tests for HumanEval/MBPP evaluation, addressing a related benchmark quality concern."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da", "Dean Lee"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Proposes hidden evaluation sets to combat contamination, a related approach discussed in Section 2."
     },
     {
       "title": "What's in my big data?",
       "authors": ["Yanai Elazar", "Akshita Bhagia", "Ian Helgi Magnusson"],
       "year": 2024,
       "relevance": "Reports that only 1.22% of verbatim HumanEval appears in OSCAR, illustrating limitations of surface-level decontamination."
     },
     {
       "title": "The Stack: 3 TB of permissively licensed source code",
       "authors": ["Denis Kocetkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2022,
       "arxiv_id": "2211.15533",
       "relevance": "Major code pretraining dataset found to contain 18.9% of HumanEval samples per Riddell et al. analysis."
     },
     {
       "title": "Deduplicating training data makes language models better",
       "authors": ["Katherine Lee", "Daphne Ippolito", "Andrew Nystrom"],
       "year": 2022,
       "relevance": "Foundational work on training data deduplication whose n-gram methods are shown to be insufficient for code decontamination."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "LBPP is a directly usable drop-in benchmark replacement, and the contamination findings inform how practitioners should interpret model evaluations."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges widely-cited HumanEval/MBPP rankings and suggests model scores are inflated, though contamination concerns are not entirely new in the community."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised; the paper is about evaluation methodology rather than dangerous capabilities."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "'Benchmarks are contaminated' narrative with concrete model rankings changing, plus anonymized evidence of a major commercial system memorizing benchmark solutions."
     },
     "demo_ability": {
       "score": 2,
       "justification": "LBPP is released on HuggingFace and can be used immediately for model evaluation, though no evaluation harness code is provided."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Cohere (a notable AI company) and evaluates GPT-4o, Claude 3.5 Sonnet, Llama3, and other well-known models."
     }
   }
 }