ai-research-survey

scan.json (23694B)

---

 {
   "paper": {
     "title": "A Survey on Data Contamination for Large Language Models",
     "authors": [
       "Yuxing Cheng",
       "Yi Chang",
       "Yuan Wu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.14425",
     "doi": "10.48550/arXiv.2502.14425"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey categorizes data contamination into phase-based (pre-training, fine-tuning, post-deployment, multi-modal) and benchmark-based (text, text-label, augmentation-based, benchmark-level) types. It reviews contamination-free evaluation strategies (data updating, data rewriting, prevention-based methods, dynamic evaluation, LLM-as-a-judge) and detection paradigms (white-box, gray-box, black-box). Five key characteristics of contamination are identified: inevitability at scale, scaling law effects (larger models memorize more), cross-stage dynamics, task-specificity, and cross-lingual transfer effects.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are released. The survey references external tools (Contamination Detector, Overlapy, etc. in Appendix C) but does not release its own artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of surveyed papers, extracted data tables, or supplementary materials are released. The survey lists external benchmarks (WikiMIA, BookMIA, MIMIR, PatentMIA, StackMIAsub) in Section 3.4 but does not release its own collected data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications provided. A survey could release a reproducible analysis environment but this one does not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions for the survey's paper selection or analysis process are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey paper with no experiments or statistical analysis of its own."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey paper with no experiments or comparative statistical claims of its own."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey paper with no experiments producing effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey paper; no experimental sample size to justify."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey paper with no experimental runs producing variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares this survey against five prior surveys (Ravaut et al., Xu et al., Fu et al., Chen et al., Deng et al.) across coverage dimensions (definition, detection, mitigation). The 'Difference with previous survey' subsection in Section 1 also positions against prior work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared prior surveys are all recent (2024-2025): Ravaut et al. 2024, Xu et al. 2024, Fu et al. 2024, Chen et al. 2025, Deng et al. 2024. These represent the current state of contamination survey literature."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system or components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments producing metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed categorical breakdowns: contamination types (phase-based vs benchmark-based, Section 2), mitigation strategies (data updating, rewriting, prevention in Section 3.1), evaluation approaches (dynamic, LLM-as-judge in Sections 3.2-3.3), and detection paradigms (white-box, gray-box, black-box in Section 4). Figure 1 provides a comprehensive structural overview."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses limitations of detection methods: 'Existing black-box contamination detection approaches rely on heuristic rules that Fu et al. (2024) showed fail under certain conditions.' Section 3.3 notes LLM-as-a-judge has preference contamination issues (Li et al. 2025). Section 5.1 notes unlearning may be 'fundamentally unachievable' (Shumailov et al. 2024)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The survey reports negative findings from the literature: detection methods fail under certain conditions (Section 5.2), safety mechanisms complicate detection, augmentation-based contamination evades traditional methods (Dekoninck et al. 2024b), and MIA methods have 'underwhelming performance on LLMs' (Duan et al. 2024 in Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to (1) examine definitions and impacts of data contamination (covered in Section 2), (2) review contamination-free evaluation methods (covered in Section 3), and (3) categorize detection methods into white-box, gray-box, and black-box (covered in Section 4). All three are substantively addressed."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey reports on others' causal findings but does not make original causal claims requiring its own study design. Statements like 'data contamination artificially inflates model performance' are summaries of cited work, not independent causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) explicitly bounds the scope: 'our focus is primarily on data contamination within the context of LLMs, and we may not have fully incorporated previous research on data contamination in other areas of machine learning.' The title itself scopes to LLMs."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey/taxonomy paper with no original empirical results requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own; no proxy-outcome gap to address."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not run any models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not describe its paper selection methodology. No search queries, databases searched, inclusion/exclusion criteria, or paper counts at each filtering stage are provided. It is unclear how the surveyed papers were identified and selected."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section with substantive discussion of the survey's coverage gaps."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific limitations: (1) new contamination mechanisms may not be captured, (2) focus on LLMs may miss other ML contamination research, (3) only some static benchmarks are listed, (4) related areas like MIA, machine unlearning, and LLM memorization are not fully covered. These are specific to this survey's scope decisions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what is not covered: 'our focus is primarily on data contamination within the context of LLMs,' 'we may not cover all related areas such as membership inference attacks (MIA), machine unlearning, and LLM memorization.' The paper is clear about its boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (e.g., list of all surveyed papers, extracted metadata, classification decisions) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not describe how papers were collected. No search databases, queries, time periods, or inclusion/exclusion criteria are specified. It is unclear how the corpus of reviewed work was assembled."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from initial paper discovery to final inclusion in the survey. The reader cannot determine how comprehensive or systematic the coverage is."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Jilin University (College of Software, School of Artificial Intelligence, Engineering Research Center of Knowledge-Driven Human-Machine Intelligence, International Center of Future Science)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding statement means the reader cannot determine whether any funder had a stake in the survey's conclusions."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model's capability on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no method to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computation performed."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no protocol registration, no structured search strategy with reproducible queries. The survey organizes by topic but does not describe a systematic paper collection methodology. The reader cannot determine how papers were found or whether the coverage is comprehensive."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey treats all cited papers equally regardless of their methodological quality. No quality scoring rubric, risk-of-bias assessment, or structured evaluation of included studies is performed. For example, papers with controlled contamination experiments are cited alongside informal observations with no differentiation."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether published contamination research skews toward certain findings (e.g., positive detection results) or whether negative detection results are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "Data contamination is inevitable as LLM training datasets expand, since web-scraped data inadvertently overlaps with evaluation benchmarks.",
       "evidence": "Section 2.2.4 cites Villalobos et al. (2024) on scaling data requirements and Deng et al. (2023) on web crawl overlap with benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "Larger models exhibit stronger contamination effects than smaller ones due to increased memorization capacity.",
       "evidence": "Section 2.2.4 cites Kocyigit et al. (2025) and Riddell et al. (2024) on scaling laws of memorization.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-lingual contamination can inflate English benchmark performance without direct exposure, while evading existing detection methods.",
       "evidence": "Section 2.2.4 cites Yao et al. (2024) on cross-lingual benchmark inflation and Zhang et al. (2024a) on evasion of detection.",
       "supported": "moderate"
     },
     {
       "claim": "Existing black-box contamination detection methods rely on heuristic rules that fail under certain conditions, raising concerns about their fundamental reliability.",
       "evidence": "Section 5.2 cites Fu et al. (2024) showing failure of black-box methods. Section 4.3 discusses assumptions underlying each method (detailed in Appendix B).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-as-a-judge evaluation may suffer from preference contamination, where models trained on synthetic data from architecturally similar foundations receive unfair preference.",
       "evidence": "Section 3.3 cites Li et al. (2025) identifying systematic bias in LLM-as-a-judge evaluations.",
       "supported": "weak"
     },
     {
       "claim": "This survey provides comprehensive coverage of data contamination definition, detection, and mitigation, unlike prior surveys which cover only partial aspects.",
       "evidence": "Table 1 compares against five prior surveys, claiming this survey covers all three areas comprehensively while others have gaps.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The survey does not describe how papers were identified, what databases were searched, what search terms were used, or what inclusion/exclusion criteria were applied. The reader cannot assess whether coverage is comprehensive or opportunistic."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "All cited papers are treated equally regardless of methodological rigor. Papers with controlled experiments are mixed with informal observations and position statements without any quality differentiation, potentially laundering weak findings through the survey."
     },
     {
       "flag": "Comprehensiveness claim weakly supported",
       "detail": "Table 1 claims 'Comprehensive' coverage of definitions versus partial coverage by competitors, but the criteria for this comparison are not defined. The self-assessment appears subjective — the authors grade themselves highest without explicit evaluation criteria."
     }
   ],
   "cited_papers": [
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["Simone Balloccu", "Patrícia Schmidtová", "Mateusz Lango", "Ondrej Dusek"],
       "year": 2024,
       "relevance": "Studies data contamination and evaluation malpractices in closed-source LLMs, directly relevant to benchmark evaluation integrity."
     },
     {
       "title": "Benchmarking benchmark leakage in large language models",
       "authors": ["Ruijie Xu", "Zengzhi Wang", "Run-Ze Fan", "Pengfei Liu"],
       "year": 2024,
       "arxiv_id": "2404.18824",
       "relevance": "Analyzes 31 LLMs for benchmark leakage in mathematical reasoning, quantifying widespread data contamination."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Foundational work on measuring LLM memorization of training data, key to understanding contamination mechanisms."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["Oscar Sainz", "Jon Campos", "Iker García-Ferrero", "Julen Etxaniz", "Oier Lopez de Lacalle", "Eneko Agirre"],
       "year": 2023,
       "relevance": "Argues that contamination evidence is fragmented and prevalence is underestimated, calling for per-benchmark contamination measurement."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia", "Yangsibo Huang", "Daogao Liu", "Terra Blevins", "Danqi Chen", "Luke Zettlemoyer"],
       "year": 2024,
       "arxiv_id": "2310.16789",
       "relevance": "Proposes Min-K% method and WikiMIA/BookMIA benchmarks for pretraining data detection, a key gray-box contamination detection technique."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "arxiv_id": "2403.04811",
       "relevance": "Quantifies contamination between code generation benchmarks and pretraining corpora, directly relevant to LLM code evaluation reliability."
     },
     {
       "title": "LiveBench: A challenging, contamination-free LLM benchmark",
       "authors": ["Colin White", "Samuel Dooley", "Manley Roberts"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Introduces a dynamically updated contamination-free benchmark with automated scoring, exemplifying data updating-based mitigation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Dynamic code evaluation benchmark that addresses contamination through continuous updating, relevant to contamination-free evaluation of code LLMs."
     },
     {
       "title": "Evading data contamination detection for language models is (too) easy",
       "authors": ["Jasper Dekoninck", "Mark Niklas Müller", "Maximilian Baader", "Marc Fischer", "Martin Vechev"],
       "year": 2024,
       "arxiv_id": "2402.02823",
       "relevance": "Demonstrates that traditional detection methods fail against augmentation-based contamination, highlighting fundamental challenges in contamination detection."
     },
     {
       "title": "Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?",
       "authors": ["Aaditya K Singh", "Muhammed Yusuf Kocyigit", "Andrew Poulton"],
       "year": 2024,
       "arxiv_id": "2411.03923",
       "relevance": "Proposes ConTAM contamination evaluation protocol and explores when contamination actually affects LLM evaluation results."
     },
     {
       "title": "Unveiling the spectrum of data contamination in language models: A survey from detection to remediation",
       "authors": ["Chunyuan Deng", "Yilun Zhao", "Yuzhao Heng", "Yitong Li", "Jiannan Cao", "Xiangru Tang", "Arman Cohan"],
       "year": 2024,
       "relevance": "Prior comprehensive contamination survey covering detection through remediation, one of the key comparison points for this work."
     },
     {
       "title": "Overestimation in LLM evaluation: A controlled large-scale study on data contamination's impact on machine translation",
       "authors": ["Muhammed Yusuf Kocyigit", "Eleftheria Briakou", "Daniel Deutsch"],
       "year": 2025,
       "arxiv_id": "2501.18771",
       "relevance": "Large-scale controlled study showing larger models exhibit stronger contamination effects and cross-lingual contamination thresholds, key empirical evidence for contamination scaling laws."
     }
   ]
 }