ai-research-survey

scan.json (24095B)

---

 {
   "paper": {
     "title": "NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark",
     "authors": [
       "Oscar Sainz",
       "Jon Ander Campos",
       "Iker García-Ferrero",
       "Julen Etxaniz",
       "Oier Lopez de Lacalle",
       "Eneko Agirre"
     ],
     "year": 2023,
     "venue": "Conference on Empirical Methods in Natural Language Processing",
     "arxiv_id": "2310.18018",
     "doi": "10.48550/arXiv.2310.18018"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["theoretical", "case-study"],
   "key_findings": "This position paper argues that LLM evaluation on standard NLP benchmarks is compromised by data contamination, where models have been trained on test data. The authors define three contamination types (guideline, raw text, annotation) and identify contamination at three training stages (pre-training, fine-tuning, post-deployment). Empirical demonstrations show ChatGPT, WizardCoder, and GitHub Copilot can reproduce CoNLL03 training data verbatim, confirming contamination. The paper proposes community measures including automatic contamination detection, a contamination registry (the LM Contamination Index), and peer-review mechanisms to flag compromised conclusions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository is provided in the paper. The LM Contamination Index website is referenced but no analysis scripts or contamination detection code are released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references the publicly available LM Contamination Index at hitz-zentroa.github.io/lm-contamination/ (footnote 1), a registry of contamination cases that is part of this work's contribution."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided for reproducing the contamination demonstrations shown in the appendix."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The appendix shows prompts and outputs but does not describe how to systematically reproduce the contamination demonstrations."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no quantitative experiments producing numerical results that would require confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no comparative quantitative claims requiring significance testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no quantitative experiments for which effect sizes would be applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no experimental sample to justify."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no experimental runs for which variance would be reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with illustrative demonstrations, not a system evaluation requiring baselines."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is conducted; baselines are not applicable to a position paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate; this is a position paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation with metrics; this is a position paper with qualitative demonstrations."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; the paper argues a position rather than presenting a system."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation requiring train/test splits; this is a position paper."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative results to break down by category; this is a position paper."
       },
       "failure_cases_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No system or method being evaluated that would have failure cases in the experimental sense."
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted that could produce negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that contamination causes overestimation and wrong scientific conclusions. The paper supports these with cited evidence (GPT-3 contamination bug in Brown et al. 2020, GPT-4's BIG-bench contamination in OpenAI 2023, empirical demonstrations in Appendix A) and logical argumentation throughout Sections 1-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central causal claim — that training on test data causes overestimation of performance — is well-supported by the established mechanism (memorization → inflated scores) and concrete examples such as GPT-3's acknowledged contamination bug (Brown et al. 2020) and the BIG-bench mixing into GPT-4 training (OpenAI 2023, Section 2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper scopes its claims to 'evaluating LLMs on standard academic benchmarks' (Section 7) and explicitly states 'there could exist other issues in current evaluations, but, they are out of the scope of this position paper.' The title and claims consistently focus on NLP benchmark evaluation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly raises one alternative in Section 5.2 — that 'the lack of memorization of a benchmark ensures that the LLM was not trained on that benchmark' and that performance might still be unaffected — but labels this 'currently speculation' without substantive discussion. No other alternative explanations for the observed phenomena are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central argument is precisely about the distinction between the proxy (benchmark scores from contaminated evaluations) and the outcome (actual model capability). Section 1 explicitly argues that contaminated scores 'overestimate' true performance and lead to wrong conclusions about model quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The appendix demonstrations use 'ChatGPT', 'WizardCoder', and 'GitHub Copilot' without specifying exact model versions, snapshot dates, or API versions. No version information is provided for any model used in the demonstrations."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The actual prompts used in the contamination demonstrations are shown in Figures 1, 2, and 3 in Appendix A. For example, Figure 1 shows: 'Please, generate the first instances of the CoNLL03 dataset train split in BIO format.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, sampling parameters, or other hyperparameters are reported for the LLM API calls used in the appendix demonstrations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; the demonstrations are simple single-turn prompts to LLMs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how the contamination demonstrations were selected or how the specific prompts were developed. There is no description of the process for choosing CoNLL03 or the three specific models."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations' and discusses the scope of the paper and the early-stage nature of proposed solutions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section contains generic statements: 'we are aware that these are early-stage solutions and that the proposed effort is really challenging' and 'there could exist other issues in current evaluations.' No specific threats to the validity of the paper's own arguments or demonstrations are identified."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'we address the problem of data contamination that occurs when evaluating LLMs on standard academic benchmarks' and that 'there could exist other issues in current evaluations, but, they are out of the scope of this position paper.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The appendix shows model outputs as figures but the raw model responses are not available for independent verification. Only selected portions of outputs are shown ('The output was shortened for commodity')."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the contamination demonstrations were conducted — when the models were queried, how many attempts were made, or whether the shown outputs are representative or cherry-picked."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper demonstrates model behavior through direct prompting."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The process from prompting models to selecting which outputs to show in the appendix is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgements section lists funding from the Basque Government (IT-1805-22), the Spanish Government (ILENIA project), and individual doctoral grants (PRE_2023_2_0137, PRE_2022_2_0208, PRE_2023_2_0060)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: five authors at HiTZ Center - University of the Basque Country, and one author (Jon Ander Campos) at Cohere, an LLM company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government research agencies (Basque Government, Spanish Government) that have no financial stake in the contamination findings. While one author is at Cohere, Cohere is not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. One author works at Cohere, an LLM company whose products could be affected by contamination discussions, but no declaration of competing interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It studies contamination itself by testing whether models can reproduce benchmark data, not measuring model performance on the benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper's entire topic is train/test overlap, but it does not itself evaluate model capability on benchmarks. The contamination checklist items are designed for papers that use benchmark evaluation, not papers studying contamination as a phenomenon."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper IS about benchmark contamination as its research topic, but it does not evaluate models on benchmarks in a way that would require addressing contamination of its own evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Position paper; no method with computational cost to report."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Position paper; no significant compute budget required."
       }
     }
   },
   "claims": [
     {
       "claim": "Data contamination causes overestimation of LLM performance on benchmarks, causing contaminated models to be preferred over non-contaminated counterparts.",
       "evidence": "Supported by cited examples: GPT-3 authors acknowledged a bug that contaminated several benchmarks (Brown et al. 2020, Section 2), OpenAI stated BIG-bench was mixed into GPT-4 training (OpenAI 2023, Section 2), and GSM-8K/MATH training data was deliberately included (Section 2).",
       "supported": "strong"
     },
     {
       "claim": "Papers using contaminated LLMs may draw wrong scientific conclusions about their hypotheses, invalidating alternative hypotheses that could be true.",
       "evidence": "Section 2 and Appendix A identify specific papers evaluating ChatGPT/GPT-3/Codex on CoNLL03 (Wei et al. 2023, Li et al. 2023a, Han et al. 2023, Li et al. 2023b), a benchmark shown to be contaminated. The argument is logical but no quantitative analysis of how contamination affected specific conclusions is provided.",
       "supported": "moderate"
     },
     {
       "claim": "ChatGPT, WizardCoder, and GitHub Copilot have memorized and can reproduce the CoNLL03 dataset.",
       "evidence": "Appendix A, Figures 1-3 show all three models generating the first lines of CoNLL03 training data in BIO format when prompted, producing near-verbatim reproductions.",
       "supported": "strong"
     },
     {
       "claim": "LLM memorization capabilities can be used to detect data contamination in closed models where training data is unavailable.",
       "evidence": "Section 5.2 proposes using extractability (adapted from Carlini et al. 2023) as a contamination metric, demonstrated anecdotally in Appendix A for three models. However, the paper acknowledges this remains speculative and notes 'further research on this topic is necessary.'",
       "supported": "moderate"
     },
     {
       "claim": "The community needs automatic measures, a contamination registry, and peer-review mechanisms to address data contamination.",
       "evidence": "Section 6 proposes four action items. The LM Contamination Index (footnote 1) is presented as an initial implementation. This is a normative proposal rather than an empirical claim.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Anecdotal empirical evidence",
       "detail": "The contamination demonstrations are limited to a single benchmark (CoNLL03) on three models. Despite arguing that contamination is widespread, no systematic quantification of contamination extent across benchmarks or models is provided."
     },
     {
       "flag": "Possible selection bias in demonstrations",
       "detail": "The paper shows cases where models successfully reproduce benchmark data but does not describe how many attempts were made, whether any models failed to reproduce the data, or whether the CoNLL03 example was selected because it works particularly well."
     },
     {
       "flag": "Lack of reproducibility details for demonstrations",
       "detail": "The appendix demonstrations lack model versions, dates of API access, temperature settings, and number of attempts. Outputs are described as 'shortened for commodity' without full outputs being available."
     }
   ],
   "cited_papers": [
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramèr", "Eric Wallace", "Matthew Jagielski", "Ariel Herbert-Voss", "Katherine Lee", "Adam Roberts", "Tom Brown", "Dawn Song", "Úlfar Erlingsson", "Alina Oprea", "Colin Raffel"],
       "year": 2021,
       "relevance": "Foundational work on extracting memorized training data from LLMs, directly relevant to data leakage and contamination detection."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang"],
       "year": 2023,
       "relevance": "Defines extractability of memorized training examples, the metric this paper adapts for measuring data contamination levels."
     },
     {
       "title": "Data contamination: From memorization to exploitation",
       "authors": ["Inbal Magar", "Roy Schwartz"],
       "year": 2022,
       "relevance": "Directly studies data contamination in LLMs, showing progression from memorization to exploitation of leaked data."
     },
     {
       "title": "Documenting large webtext corpora: A case study on the colossal clean crawled corpus",
       "authors": ["Jesse Dodge", "Maarten Sap", "Ana Marasović", "William Agnew", "Gabriel Ilharco", "Dirk Groeneveld", "Margaret Mitchell", "Matt Gardner"],
       "year": 2021,
       "relevance": "Showed that the C4 pre-training corpus contained test splits of several NLP benchmarks, providing direct evidence of contamination."
     },
     {
       "title": "Stop uploading test data in plain text: Practical strategies for mitigating data contamination by evaluation benchmarks",
       "authors": ["Alon Jacovi", "Avi Caciularu", "Omer Goldman", "Yoav Goldberg"],
       "year": 2023,
       "relevance": "Proposes preventative strategies to avoid benchmark contamination, complementary to this paper's detection-focused approach."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper that acknowledged a bug in their filtering script causing benchmark contamination during training."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Acknowledged that BIG-bench was inadvertently mixed into GPT-4 training and that MATH/GSM-8K training data was included to improve mathematical reasoning."
     },
     {
       "title": "Did chatgpt cheat on your test?",
       "authors": ["Oscar Sainz", "Jon Ander Campos", "Iker García-Ferrero", "Julen Etxaniz", "Eneko Agirre"],
       "year": 2023,
       "relevance": "Prior work by the same authors showing ChatGPT generates portions of popular NLP benchmarks, providing evidence of contamination."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Codex paper — one of the LLMs identified as potentially contaminated when evaluated on CoNLL03 by downstream papers."
     },
     {
       "title": "The ROOTS search tool: Data transparency for LLMs",
       "authors": ["Aleksandra Piktus", "Christopher Akiki", "Paulo Villegas", "Hugo Laurençon"],
       "year": 2023,
       "relevance": "Tool for searching pre-training data to detect contamination; found BLOOM should not be evaluated on XNLI due to contamination."
     },
     {
       "title": "Memorization vs. generalization: Quantifying data leakage in NLP performance evaluation",
       "authors": ["Aparna Elangovan", "Jiayuan He", "Karin Verspoor"],
       "year": 2021,
       "relevance": "Studies data leakage scenarios in NLP evaluation, quantifying the impact of memorization on performance estimates."
     },
     {
       "title": "Data portraits: Recording foundation model training data",
       "authors": ["Marc Marone", "Benjamin Van Durme"],
       "year": 2023,
       "relevance": "Tool for auditing foundation model training data, useful for contamination detection in open LLMs."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Proposes a contamination taxonomy and detection framework useful for researchers evaluating LLMs, but not a directly usable tool."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Argues that standard NLP evaluation is 'in trouble' and published results may be invalid, challenging confidence in benchmark leaderboards."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about unreliable scientific conclusions but focuses on evaluation integrity rather than AI safety or security."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Names specific models (ChatGPT, GPT-4, Copilot) as contaminated and identifies published papers with potentially invalid conclusions."
     },
     "demo_ability": {
       "score": 1,
       "justification": "References the LM Contamination Index website but provides no pip-installable tool or interactive demo."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Discusses ChatGPT, GPT-4, GitHub Copilot, and LLaMA; published at EMNLP, a top NLP venue."
     }
   }
 }