ai-research-survey

scan.json (33469B)

---

 {
   "paper": {
     "title": "The Emperor's New Clothes in Benchmarking? A Rigorous Examination of Mitigation Strategies for LLM Benchmark Data Contamination",
     "authors": [
       "Yifan Sun",
       "Han Wang",
       "Dongbai Li",
       "Gang Wang",
       "Huan Zhang"
     ],
     "year": 2025,
     "venue": "Preprint",
     "arxiv_id": "2503.16402",
     "doi": "10.48550/arXiv.2503.16402"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This paper introduces two metrics—fidelity and contamination resistance—for assessing benchmark data contamination (BDC) mitigation strategies at question level rather than aggregate accuracy. Testing 20 strategies across 10 LLMs (3B–34B) and 5 benchmarks under mild and intensive contamination, they find no existing strategy significantly outperforms the vanilla case (no update) in resistance across all benchmarks. Minor modifications preserve fidelity but offer negligible resistance; aggressive modifications improve resistance at the cost of fidelity, with no strategy achieving both simultaneously.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'Our code repository is available at https://github.com/ASTRAL-Group/BDC_mitigation_assessment.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five benchmarks used (Arc-C, MMLU, TruthfulQA, GSM8K, RepliQA) are publicly available standard datasets. The paper uses standard public splits documented in Table 8."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 12 lists GPU hardware (9x NVIDIA L40S) and training hyperparameters, but no Python version, library versions, requirements.txt, Dockerfile, or dependency specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a detailed pipeline description (Sections 4.1–4.5) and a code repository link, but no step-by-step reproduction instructions, README commands, or 'Reproducing Results' section are included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports averages over 10 LLMs but includes no confidence intervals, error bars, or ± notation. Only point estimates are presented."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "One-sided paired hypothesis testing at the 0.05 significance level is used throughout (Section 5.1). Green-highlighted values in Tables 3 and 4 indicate statistically significant improvements over vanilla."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fidelity and resistance scores are reported as absolute values alongside the vanilla baseline (e.g., MPA resistance 0.921 vs. vanilla 0.923 in Table 3), providing full context for the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 10 LLMs, 5 benchmarks, and 20 strategies. They explain model selection (14 candidates filtered to 10 via contamination checks) but provide no power analysis or justification that 10 LLMs is sufficient for their statistical claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 states 'Each value represents the average of 10 scores obtained using different LLMs' but no standard deviation, interquartile range, or other spread measure is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The 'vanilla' case (no benchmark update, fidelity=1.000) serves as the baseline throughout. All 20 mitigation strategies are compared against it in Tables 3 and 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 20 mitigation strategies are from contemporary work (Zhu et al. 2023b, 2024a/b; Ying et al. 2024; Wang et al. 2021). The vanilla baseline is the natural comparison point. The paper claims to 'comprehensively cover all existing BDC mitigation strategies proposed to date' (Section 4.3)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study compares 11 single strategies and their combinations (e.g., MPA = S2+S3+S4+S9+S10+S11 vs. individual components S2, S3, S4, etc.), showing how combining strategies affects fidelity and resistance. Two contamination recipes (mild and intensive) also serve as controlled variable manipulation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two complementary metrics are proposed and used: fidelity (matching between clean evaluations on original vs. updated benchmark) and contamination resistance (matching between clean and contaminated evaluations on updated benchmark). Section 3 defines both formally."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (evaluation vectors, normalized Hamming distance). The paper notes the need for manual validation of low-fidelity strategies (Section 5.3, Tables 5–6) but does not actually perform systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 specifies standard splits: Arc-C test (1172), MMLU test (50 per subset), TruthfulQA validation (817), GSM8K test (1319), RepliQA (1000). These are held out from contamination tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (5 benchmarks), per strategy (20 strategies), and per contamination type (mild vs. intensive) in Tables 3 and 4. Figure 4 aggregates at the benchmark level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides qualitative examples of strategy failures: Table 5 shows Analysis Extension dramatically increasing problem complexity, Table 6 shows MPA introducing incorrect constraints. Appendix C.4.2 shows LLM-generated wrong answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding is a negative result: 'no existing strategy significantly improves resistance over the vanilla case across all benchmarks' and 'none achieves strong fidelity and contamination resistance simultaneously' (Section 5.1, Section 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'no existing strategy significantly improves resistance over the vanilla case across all benchmarks' and 'none effectively balances fidelity and contamination resistance.' Both are supported by Table 3 (no green-highlighted values span all benchmarks for any strategy) and Figure 4 (no strategy in upper-right corner)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a controlled experimental design: deliberately selecting uncontaminated LLMs (verified via 3 detection methods), manually introducing contamination via fine-tuning, and measuring effects on evaluation vectors. This controlled manipulation supports the causal claim that contamination affects benchmark performance through specific mitigation strategies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested setting: 10 specific LLMs (3B–34B, listed in Table 7), 5 specific benchmarks (Table 8), 20 specific strategies (Table 2), and two contamination recipes. The paper states it covers 'all existing BDC mitigation strategies proposed to date' but does not overclaim to untested models or benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether fine-tuning-based contamination (their method) may behave fundamentally differently from pre-training contamination. It does not consider whether their metrics might be insensitive to certain forms of strategy effectiveness, or whether model size (3B–34B only) limits conclusions about frontier models."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exactly what it claims: question-level evaluation matching via normalized Hamming distance. The metrics (fidelity and resistance) directly correspond to the stated goals without proxy gaps. Section 3 provides formal definitions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 lists specific model versions: Llama-3.2-3B-Instruct, Yi-1.5-6B-Chat, vicuna-7b-v1.5, Llama-3.1-8B-Instruct, Falcon3-10B-Instruct, Qwen2.5-14B-Instruct, Phi-3-medium-128k-instruct, DeepSeek-V2-Lite-Chat, Qwen2.5-32B-Instruct, Yi-1.5-34B-Chat. GPT-4o-2024-08-06 is specified for mitigation strategy generation."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation template ('Question:{input}\\n Answer:') and full 5-shot GSM8K prompt are provided in Appendix C.5. However, the GPT-4o prompts used to generate the 20 mitigated benchmarks—a key part of the methodology—are not provided. Only natural-language descriptions and output examples are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 reports fine-tuning details: AdamW optimizer, batch sizes 2/3/4, learning rates 1e-5/3e-5, linear LR schedule, weight decay 0, warmup ratio 5%, epochs 1/3. GPT-4o temperature 0.7 is stated for mitigation. Evaluation settings (zero-shot, 5-shot, max tokens) are in Appendix C.5."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper uses standard LLM fine-tuning and evaluation without agent-based workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: 14 candidate LLMs filtered to 10 via 3 contamination detection methods (Tables 9–11), benchmark selection with rationale (Section 4.2), contamination recipes with validation checks (Section 4.4, Tables 13–15), and evaluation procedures (Section 4.5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated Limitations or Threats-to-Validity section. The 'Impact Statement' (after Section 6) is two sentences and does not discuss limitations of the methodology."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. Key unaddressed threats include: fine-tuning vs. pre-training contamination differences, model size limitations (3B–34B only), and whether Hamming distance captures all relevant dimensions of strategy effectiveness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge limitations of its model size range, benchmark selection, or contamination simulation method. No explicit boundary statements about untested settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "A code repository is provided but the paper does not explicitly state that raw evaluation vectors, contaminated model weights, or per-question results are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: LLM selection via 3 detection methods on 14 candidates (Section 4.2, Tables 9–11), benchmark selection with rationale and splits (Section 4.2, Table 8), contamination procedure (Section 4.4, Table 12), and evaluation procedure (Section 4.5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides an overview of the full pipeline. Each step is documented: LLM-benchmark pair selection with contamination checks, mitigation strategy application, model contamination with two recipes and three validation checks (accuracy inflation, retained correctness, perplexity), evaluation vector computation, and metric derivation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with University of Illinois Urbana-Champaign. No conflict with evaluated products (they evaluate open-source models and use GPT-4o only for benchmark generation, not as the subject of evaluation)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the 10 LLMs. The paper instead relies on contamination detection methods (Min-K% Prob, Sharded Rank Comparison Test, TS-Guessing) to verify non-contamination rather than using temporal cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is the central topic of the paper. Three BDC detection methods are applied to all 14 candidate LLM-benchmark pairs (Section 4.2, Tables 9–11). Only models passing all three detection methods on all benchmarks are retained."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper addresses benchmark contamination. They verify non-contamination before experiments (Section 4.2), deliberately introduce contamination (Section 4.4), validate contamination effectiveness (Tables 13–15), and measure mitigation strategy resistance to contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve LLMs evaluated on benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example compute costs are reported despite the study requiring extensive LLM evaluations across 10 models × 5 benchmarks × 20 strategies × 2 contamination recipes, plus GPT-4o calls for generating mitigated benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 12 mentions '9x NVIDIA L40S' GPUs but does not quantify total GPU hours, training time, or overall computational budget for the full experimental campaign."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Fine-tuning involves stochastic optimization, but no results across multiple random seeds are provided. Variation comes only from different LLMs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per LLM-benchmark-strategy combination is not stated. Each value is averaged across 10 different LLMs, but whether each individual experiment was run once or multiple times is not specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fixed hyperparameters are reported in Table 12 (e.g., learning rate 1e-5 or 3e-5 depending on benchmark type) but no hyperparameter search was described. The rationale for choosing these specific values is not provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Different learning rates are used for multiple-choice (1e-5) vs. open-ended (3e-5) benchmarks without justification for how these values were selected. No validation-based selection process is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "One-sided paired hypothesis tests are performed for each of 20 strategies across 5 benchmarks and 2 contamination types, yielding up to 200 comparisons. No multiple comparison correction (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper introduces two novel combination strategies (MPA-Ques+Trans-CN and MPA-Choice+Trans-CN) alongside 18 existing strategies but does not acknowledge potential bias in evaluating their own proposed combinations within the same framework."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different mitigation strategies have vastly different computational costs (e.g., adding a typo vs. translation via GPT-4o vs. generating entirely new questions), but performance is never analyzed as a function of compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The fidelity metric directly addresses construct validity: it measures whether the updated benchmark still evaluates the same model capability as the original. Section 3 and Figure 2(b) explicitly argue that matching aggregate accuracy is insufficient because the evaluation objective may have shifted. Tables 5–6 provide concrete examples."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper evaluates LLMs directly on benchmarks without agent scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "RepliQA is explicitly included because 'Its recent release (December 9, 2024) and non-factual nature ensure that none of the LLMs in our study have been contaminated by this benchmark' (Section 4.2). All other model-benchmark pairs are verified via three detection methods."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation format (e.g., multiple-choice probability-based selection) or prompt structure could leak answer information, or whether mitigation strategies inadvertently introduce feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential non-independence between training data and benchmark examples beyond the contamination detection step. Structural similarities or near-duplicates between benchmark questions and training corpora are not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Three concrete detection methods are applied: Min-K% Prob (token probability-based, Table 11), Sharded Rank Comparison Test (order-based, Table 9), and TS-Guessing (generation-based, Table 10). These are applied to all 14 candidate LLMs across 4 benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "No existing BDC mitigation strategy significantly improves contamination resistance over the vanilla case (no benchmark update) across all benchmarks.",
       "evidence": "Table 3 shows that while some strategies achieve statistically significant resistance improvements on specific benchmarks (green-highlighted values), no single strategy achieves significant improvement across all 5 benchmarks. One-sided paired hypothesis testing at 0.05 level (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "No existing mitigation strategy achieves both high fidelity and high contamination resistance simultaneously.",
       "evidence": "Figure 4 shows strategies clustering in either the lower-right (high fidelity, low resistance) or upper-left (high resistance, low fidelity) regions. No strategy occupies the upper-right corner. Section 5.1 discusses this trade-off explicitly.",
       "supported": "strong"
     },
     {
       "claim": "Minor modifications (typos, synonyms) achieve high fidelity (>0.9) but do not improve resistance beyond vanilla.",
       "evidence": "Table 3 shows typographical perturbation and synonym replacement achieve fidelity >0.9 on most benchmarks, but their resistance scores are not statistically significantly higher than vanilla on any benchmark.",
       "supported": "strong"
     },
     {
       "claim": "Semantic-altering strategies achieve statistically significantly higher resistance than vanilla but at the cost of ~0.15 lower fidelity.",
       "evidence": "Table 4 shows all four semantic-altering strategies (Mimicking, Remember-Understand, Application, Analysis Extension) achieve significant resistance improvements on Arc-C and MMLU (all green-highlighted), with fidelity ranging from 0.655 to 0.766 compared to ~0.85–0.95 for semantic-preserving strategies.",
       "supported": "strong"
     },
     {
       "claim": "Intensive contamination leads to lower resistance scores than mild contamination, but the relative ranking of strategies remains stable.",
       "evidence": "Table 3 shows resistance under intensive contamination is consistently lower than under mild contamination across all strategies and benchmarks. Section 5.1 states 'strategies that perform well under mild contamination continue to rank highly under intensive contamination.'",
       "supported": "strong"
     },
     {
       "claim": "Existing accuracy-based assessment methods (accuracy drop, accuracy matching) are incomplete and can be misleading.",
       "evidence": "Figure 2 illustrates the limitations: accuracy drop lacks a clean baseline reference, and accuracy matching can show matching scalar accuracy while question-level results differ significantly. This motivates the question-level metrics. Section 1.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "Despite making strong negative claims about all existing BDC mitigation strategies, the paper has no Limitations or Threats-to-Validity section. Key unaddressed limitations include: whether fine-tuning adequately simulates real-world pre-training contamination, the 3B–34B model size range excluding frontier models, and whether Hamming distance captures all relevant dimensions of strategy effectiveness."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "The paper performs up to 200 one-sided paired hypothesis tests (20 strategies × 5 benchmarks × 2 contamination types) without any correction for multiple comparisons. This inflates the family-wise error rate and could lead to false significant results in the green-highlighted cells."
     },
     {
       "flag": "Fine-tuning as contamination proxy",
       "detail": "Contamination is introduced via full-parameter fine-tuning (1 or 3 epochs), which is a much more direct form of memorization than incidental inclusion during pre-training on massive corpora. The paper's two recipes (mild and intensive) vary only in intensity, not in the fundamental mechanism. Results may not transfer to real-world contamination scenarios."
     },
     {
       "flag": "No variance reported for averaged metrics",
       "detail": "Table 3 averages fidelity and resistance over 10 LLMs without reporting any spread measure (std dev, IQR, range). The reader cannot assess whether the averaged metrics hide high variability across models."
     }
   ],
   "cited_papers": [
     {
       "title": "Don't make your LLM an evaluation benchmark cheater",
       "authors": ["Kun Zhou", "Yutao Zhu", "Zhipeng Chen"],
       "year": 2023,
       "arxiv_id": "2311.01964",
       "relevance": "Directly addresses LLM benchmark cheating and BDC detection, a foundational paper for the contamination mitigation space."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["Cheng Xu", "Shuhao Guan", "Derek Greene"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Comprehensive survey of LLM benchmark data contamination, directly relevant to the survey's evaluation reliability theme."
     },
     {
       "title": "Clean-eval: Clean evaluation on contaminated large language models",
       "authors": ["Wenhao Zhu", "Hongyi Hao", "Zhiwei He"],
       "year": 2023,
       "arxiv_id": "2311.09154",
       "relevance": "One of the key BDC mitigation strategies evaluated in this paper (S12), combining syntactic modification, synonym replacement, and back-translation."
     },
     {
       "title": "Inference-time decontamination: Reusing leaked benchmarks for large language model evaluation",
       "authors": ["Qin Zhu", "Qingyuan Cheng", "Runyu Peng"],
       "year": 2024,
       "arxiv_id": "2406.13990",
       "relevance": "Proposes the ITD mitigation strategy (S13) that is one of the most extensively evaluated approaches in this paper."
     },
     {
       "title": "Dynamic evaluation of large language models by meta probing agents",
       "authors": ["Kaijie Zhu", "Jiaao Wang", "Qiang Zhao"],
       "year": 2024,
       "relevance": "Proposes the MPA mitigation strategy (S14), one of the combined strategies that achieves higher resistance but lower fidelity."
     },
     {
       "title": "Automating dataset updates towards reliable and timely evaluation of large language models",
       "authors": ["Jiahao Ying", "Yixin Cao", "Yushi Bai"],
       "year": 2024,
       "relevance": "Proposes semantic-altering mitigation strategies (Mimicking, Remember-Understand, Application, Analysis Extension) that are the S17–S20 strategies evaluated."
     },
     {
       "title": "LiveBench: A challenging, contamination-free LLM benchmark",
       "authors": ["Colin White", "Samuel Dooley", "Manley Roberts"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Proposes a contamination-free benchmark approach as an alternative to mitigation strategies, relevant to evaluation reliability."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia"],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "Proposes Min-K% Prob, one of the three contamination detection methods used in this paper's pipeline to verify model non-contamination."
     },
     {
       "title": "Proving test set contamination in black box language models",
       "authors": ["Yonatan Oren", "Nicole Meister", "Niladri Chatterji"],
       "year": 2023,
       "arxiv_id": "2310.17623",
       "relevance": "Proposes the Sharded Rank Comparison Test used as one of three contamination detection methods in this paper."
     },
     {
       "title": "Investigating data contamination in modern benchmarks for large language models",
       "authors": ["Chunyuan Deng", "Yilun Zhao", "Xiangru Tang"],
       "year": 2023,
       "arxiv_id": "2311.09783",
       "relevance": "Proposes TS-Guessing, the generation-based contamination detection method used as one of three detection methods in this paper."
     },
     {
       "title": "Evading data contamination detection for language models is (too) easy",
       "authors": ["Jasper Dekoninck", "Mark Niklas Müller", "Maximilian Baader"],
       "year": 2024,
       "arxiv_id": "2402.02823",
       "relevance": "Demonstrates that contamination detection can be evaded, motivating the need for robust contamination mitigation assessment."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark, relevant to the survey's coverage of evaluation integrity for LLM code generation."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "Questions whether leaderboard rankings reflect true coding proficiency and proposes evolving benchmarks via LLMs, directly relevant to benchmark reliability."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The framework and metrics could be used by benchmark developers to assess mitigation strategies, but it requires significant compute infrastructure and is not a drop-in tool."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "The 'Emperor's New Clothes' framing directly challenges the widely held belief that benchmark update strategies effectively mitigate contamination—finding that none significantly outperforms doing nothing across all benchmarks."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about unreliable LLM evaluation due to contamination but does not demonstrate a novel attack or immediate safety risk."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The provocative title and finding that current mitigation strategies are essentially ineffective creates a 'benchmarks are broken' narrative, though it targets methodology rather than specific companies."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code repository is provided but running the full pipeline requires fine-tuning 10 LLMs across multiple configurations on multiple GPUs—not a quick demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From University of Illinois Urbana-Champaign, a well-known CS department but not a famous AI lab like OpenAI, Google DeepMind, or Anthropic."
     }
   }
 }