ai-research-survey

scan.json (30706B)

---

 {
   "paper": {
     "title": "Detecting Benchmark Contamination Through Watermarking",
     "authors": [
       "Tom Sander",
       "Pierre Fernandez",
       "Saeed Mahloujifar",
       "Alain Durmus",
       "Chuan Guo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.17259",
     "doi": "10.48550/arXiv.2502.17259"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Watermarking benchmark questions via LLM rephrasing preserves benchmark utility (models rank identically on original and watermarked versions) while enabling reliable contamination detection through radioactivity tests. Detection confidence scales with watermark strength, number of contaminations, and benchmark size — achieving p-values below 10^-5 when accuracy is inflated by just 5% on 5000 MMLU questions. The method works across different tokenizers with reasonable confidence loss, and substantially outperforms canary-based detection, which fails even with 10x more contamination exposures.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. They reference Meta Lingua (Videau et al., 2024) as the training framework but do not release their own watermarking or detection code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The original benchmarks (ARC, MMLU) are public, but the watermarked versions of the benchmarks and the trained model checkpoints are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "They mention A-100 GPUs with 80GB memory and the Meta Lingua framework, but provide no requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The method and hyperparameters are described but there is no README, script, or explicit guide to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Accuracy numbers and p-values are reported as point estimates. No confidence intervals or error bars appear on accuracy results. The paper notes that [-1, 0] is a 90% CI for log10(p) under H0, but this is a theoretical property of the test, not an uncertainty measure on the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The core contribution is a binomial significance test for contamination detection (Equation 2, Proposition 1). P-values are reported throughout (Table 1, Figures 3b, 4, 8). The test has a formally proven False Positive Rate equal to α."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports accuracy improvements with baseline context: e.g., '+4.3%' (from 53.5% to 57.9%), '+9.5%' (from 53.5% to 63.0%), '+18.2%' (from 53.5% to 71.7%) for ARC-Easy at different contamination levels."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "MMLU* uses 5000 questions 'to accelerate experimentation and maintain a comparable size to the other benchmarks,' but no power analysis or formal justification for any of the benchmark sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single training runs. No standard deviation, variance across seeds, or spread measures are reported for accuracy or detection metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 compares against canary-based contamination detection as an alternative baseline (Table 3). They also compare contaminated vs. uncontaminated models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The canary method from BIG-bench (Srivastava et al., 2022) is a standard and commonly used approach for contamination detection. Membership inference attacks (Duan et al., 2024) are discussed as contemporary alternatives that are ineffective."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides extensive ablations: watermark strength δ (0, 1, 2, 4), window size k (0, 1, 2), model size (135M, 360M, 1B), number of contaminations (0, 4, 8, 16), rephrasing model (8B vs 70B), and tokenizer (Section 4.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on two distinct metrics: benchmark accuracy (performance impact) and detection confidence (log10 p-value from the radioactivity test). Both are reported in Table 1 and throughout Section 4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the watermarked questions' quality. Benchmark utility is assessed entirely through automated comparison of model performance on original vs. watermarked versions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are trained with controlled contamination and evaluated on the benchmark test sets. An out-of-distribution (OOD) evaluation template is used to further separate training and evaluation contexts (Section 4.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (ARC-Easy, ARC-Challenge, MMLU*) in Table 1, per model size in Figure 4, per window size in Table 2 and Figures 7-8, and per watermark strength in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows a concrete rephrasing failure on a math question. Section 4.3 discusses how smaller benchmarks lead to weaker detection (ARC-Challenge with 4 contaminations has 'doubtful' p-value). Section 5 discusses evasion scenarios."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows the canary-based method fails even with 160 contaminations (p-val=0.19). They report that detection on ARC-Challenge with 4 contaminations is 'doubtful' (log10(p)=-1.2). Figure 6 shows the 8B rephrasing model failing on math questions."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'p-val = 10^-3 for +5% on ARC-Easy.' Table 1 shows 4 contaminations on ARC-Easy: log10(p)=-3.0, Acc +4.3%, which is consistent. Claims of preserved utility are supported by Figure 3a and Figure 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's experimental design controls contamination levels by training models from scratch with and without benchmark data, enabling causal inference about the relationship between contamination and detection confidence. This is a controlled experiment with single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Results are tested on 3 benchmarks, 3 model sizes, and 4 tokenizer families. The limitations section explicitly bounds scope: 'primarily designed for unintentional contamination' and acknowledges that math/code questions are harder to watermark."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Proposition 1 explicitly distinguishes three meanings of 'contaminated': (i) trained on test set, (ii) memorized the watermark, (iii) performance artificially enhanced. The paragraph below Proposition 1 acknowledges the scenario where a model performs well without overfitting to watermark biases."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between what is measured (watermark memorization, Definition ii) and what is claimed (benchmark contamination). The text below Proposition 1 states: 'This is distinct from the hypothesis that the model's test performance is not artificially enhanced due to training on the benchmark.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: 'Llama-3.1-8B-Instruct' for rephrasing, 'Llama-3.3-1B', 'Llama-3.3-3B', 'Llama-3.1-8B' for evaluation. Tokenizer experiments use Llama-1/2, Llama-3, Gemma-1/2, Gemma-3 (Table 4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the exact system prompt ('You are a problem rephrasing assistant...') and instruction. Section 4.2 provides the exact evaluation templates for both in-distribution and OOD evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports top-p=0.7, temperature=0.5, k=2, γ=50%. Section 4.2 reports hidden dim=2048, 25 layers, 16 attention heads, LR=3×10^-3, weight decay=0.033, warmup=5000 steps, batch size=4, sequence length=4096, gradient clipping=1.0."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method involves direct LLM generation for rephrasing and standard model training/evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the contamination procedure in detail: batches taken between steps 2500-7500, format template provided, DCLM as base training data. Section 4.1 describes MMLU* as a random subset of 5000 questions across all disciplines."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations & Conclusion' and contains two substantive bullet points addressing rephrasing impact and intentional evasion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 identifies study-specific threats: 'some questions lose coherence after rephrasing' (Figure 6 demonstrates this), and 'Malicious actors could rephrase questions to weaken the watermark or train only on answers conditioned on questions, which would bypass radioactivity detection.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the method is 'primarily designed for unintentional contamination.' It explicitly acknowledges that intentional evasion (rephrasing to remove watermarks, training only on answers) would bypass detection. Section 4.3 notes that math/code questions present challenges for rephrasing."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No trained model checkpoints, watermarked benchmark datasets, or raw experimental outputs are released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Training data source is DCLM (Li et al., 2024). Benchmarks are ARC-Easy (1172 questions), ARC-Challenge (2372 questions), MMLU* (5000 randomly selected questions). Contamination injection procedure is fully described in Section 4.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (ARC, MMLU) and a public training corpus (DCLM)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: benchmark selection → watermarked rephrasing with specified parameters → training with controlled contamination injection at specified steps → evaluation with specified templates → radioactivity detection scoring."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present. Authors are Meta FAIR employees and École polytechnique affiliates, implying corporate funding, but this is not explicitly disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: 'Meta FAIR' and 'École polytechnique CMAP' in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta has a vested interest in the integrity of LLM benchmarks, as it trains and releases Llama models that are evaluated on them. The paper uses exclusively Meta models (Llama family) for all experiments. No statement of funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure section is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper trains models from scratch with fully controlled training data (10B tokens from DCLM with specified contamination injections). The entire training pipeline is described, so the 'cutoff' is effectively complete — the authors know exactly what the model saw."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train-test overlap is the central topic of the paper. They intentionally control contamination levels (0, 4, 8, 16 injections) and analyze the resulting overlap's impact on detection and performance."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the paper's primary research question. The entire method is designed to detect it, and experiments systematically vary contamination levels to validate detection."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve training and evaluating language models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states: 'Each radioactivity detection test took less than 30 minutes on a single GPU.' This quantifies the cost of the detection method."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed compute: '1B models was conducted on 8 nodes (so 64 GPUs) and took approximately six hours.' Total budget estimated at 'approximately 10,000 GPU hours' including all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All experiments appear to be single-run training with single-seed evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Results appear to come from single training runs for each configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Training hyperparameters are listed but no indication of how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across all tested configurations (multiple δ values, window sizes, model sizes, contamination levels, tokenizers) rather than selecting a single best. The paper sweeps over parameter space rather than optimizing."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple statistical tests are performed across 3 benchmarks, 4 contamination levels, 4 watermark strengths, and 3 window sizes. No correction for multiple comparisons is applied, though many p-values are so extreme (10^-12) that correction would not change conclusions."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the watermarking method and evaluate it themselves. No discussion of self-comparison bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While different model sizes are tested (135M, 360M, 1B), performance is not systematically analyzed as a function of compute budget. The focus is on contamination level, not compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ARC-Easy, ARC-Challenge, and MMLU actually measure the capabilities they claim to measure. The benchmarks are treated as given without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated directly via loss comparison on answer choices."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper trains models from scratch on controlled data, fully addressing temporal leakage by design. The authors know exactly when each piece of data enters training (contamination injected between steps 2500 and 7500)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 uses an out-of-distribution (OOD) evaluation template distinct from the contamination template to ensure the model isn't merely memorizing the formatting context. Results are reported for both in-distribution and OOD evaluation."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Each benchmark uses a different secret watermarking key s, ensuring independence between detection tests. The base training corpus (DCLM) is separate from the benchmarks, and the paper controls exactly which benchmark data enters training."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper's entire contribution IS a concrete leakage detection method: watermark embedding via rephrasing followed by a binomial radioactivity test with formal p-value guarantees (Proposition 1, Algorithm 1)."
       }
     }
   },
   "claims": [
     {
       "claim": "Watermarking benchmark questions through LLM rephrasing preserves benchmark utility — models perform similarly and rank identically on watermarked vs. original versions.",
       "evidence": "Figure 3a shows Llama-3.2-1B, 3B, and 8B perform comparably on original and watermarked ARC-Easy across all δ values (0-4). Figure 7 extends this across all benchmarks and window sizes. Some MMLU* discrepancies noted but occur even without watermarking.",
       "supported": "strong"
     },
     {
       "claim": "Contamination detection achieves p-value below 10^-5 when accuracy is inflated by 5% on 5000 MMLU questions.",
       "evidence": "Table 1 shows 4 contaminations on MMLU*: log10(p)=-5.7 with +5.1% accuracy gain. For ARC-Easy with 4 contaminations: log10(p)=-3.0 with +4.3% gain.",
       "supported": "strong"
     },
     {
       "claim": "Detection confidence increases with both watermark strength δ and number of contaminations.",
       "evidence": "Figure 3b and Table 1 show monotonic increase in -log10(p) across contamination levels (0, 4, 8, 16) and δ values (0, 1, 2, 4). With 16 contaminations and δ=4, all benchmarks achieve log10(p) < -12.",
       "supported": "strong"
     },
     {
       "claim": "The method works across different tokenizers with reasonable loss in detection confidence.",
       "evidence": "Table 4 shows detection remains reliable across Llama-1/2, Llama-3, Gemma-1/2, and Gemma-3 tokenizers. P-values range from -7 (Llama-1/2) to -15 (Gemma-3) with contamination, and near 0 without.",
       "supported": "strong"
     },
     {
       "claim": "Watermark radioactivity substantially outperforms canary-based contamination detection.",
       "evidence": "Table 3 shows a 64-digit canary seen 160 times (10x more than max watermark contaminations) yields p-val=0.19, while watermarking achieves p-val < 10^-12 with just 16 contaminations.",
       "supported": "strong"
     },
     {
       "claim": "For a fixed performance gain, p-values are consistent across model sizes.",
       "evidence": "Section 4.3 and Figure 4 show that 1B (4 contaminations), 360M (8 contaminations), and 135M (16 contaminations) all achieve ~+6% gain on MMLU* with detection p-values around 10^-5.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Meta evaluating using exclusively Meta models",
       "detail": "All experiments use Meta's Llama models for both watermark embedding and evaluation. While the paper is about a detection method rather than model capability, there is no validation with non-Meta models for the rephrasing step. The authors acknowledge this partially by testing different tokenizers."
     },
     {
       "flag": "No variance across experimental runs",
       "detail": "All results appear to be from single training runs. Given the stochasticity of pretraining, detection p-values and accuracy numbers could vary across seeds. The absence of multi-seed results means the reported p-values may not be representative."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite proposing a method that benchmark creators should adopt, neither the watermarking code, detection code, watermarked benchmarks, nor model checkpoints are released, making independent verification impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "A watermark for large language models",
       "authors": ["John Kirchenbauer", "Jonas Geiping", "Yuxin Wen", "Jonathan Katz", "Ian Miers", "Tom Goldstein"],
       "year": 2023,
       "arxiv_id": "2301.10226",
       "relevance": "Foundation watermarking scheme (green/red list) that this paper builds upon for benchmark contamination detection."
     },
     {
       "title": "Watermarking makes language models radioactive",
       "authors": ["Tom Sander", "Pierre Fernandez", "Alain Durmus", "Matthijs Douze", "Teddy Furon"],
       "year": 2024,
       "arxiv_id": "2402.14904",
       "relevance": "Prior work by same authors demonstrating watermark radioactivity in fine-tuning; this paper extends the concept to pre-training contamination detection."
     },
     {
       "title": "Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?",
       "authors": ["Aaditya K Singh", "Muhammed Yusuf Kocyigit", "Andrew Poulton", "David Esiobu", "Maria Lomeli", "Gergely Szilvasy", "Dieuwke Hupkes"],
       "year": 2024,
       "arxiv_id": "2411.03923",
       "relevance": "Directly addresses the benchmark contamination measurement problem that this paper proposes to solve via watermarking."
     },
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["Simone Balloccu", "Patrícia Schmidtová", "Mateusz Lango", "Ondřej Dušek"],
       "year": 2024,
       "arxiv_id": "2402.03927",
       "relevance": "Documents data contamination and evaluation malpractice in closed-source LLMs, motivating the need for proactive contamination detection."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da", "Dean Lee", "Vaughn Robinson", "Catherine Wu", "Will Song"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Demonstrated significant performance drops on new GSM8K-like questions, providing key evidence of benchmark contamination in LLMs."
     },
     {
       "title": "Do membership inference attacks work on large language models?",
       "authors": ["Michael Duan", "Anshuman Suri", "Niloofar Mireshghallah", "Sewon Min", "Weijia Shi"],
       "year": 2024,
       "arxiv_id": "2402.07841",
       "relevance": "Shows membership inference is ineffective for LLMs in realistic scenarios, motivating alternative contamination detection approaches like watermarking."
     },
     {
       "title": "Investigating data contamination for pre-training language models",
       "authors": ["Minhao Jiang", "Ken Ziyu Liu", "Ming Zhong", "Rylan Schaeffer", "Siru Ouyang", "Jiawei Han", "Sanmi Koyejo"],
       "year": 2024,
       "arxiv_id": "2401.06059",
       "relevance": "Studies data contamination during pre-training showing even small models exhibit benchmark performance gains from contamination."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Foundational work on quantifying memorization in language models, closely related to the contamination detection problem."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "Introduces MMLU, one of the three benchmarks used in this paper's contamination detection experiments."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri", "Abhinav Pandey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Describes Llama-3 models used throughout this paper for both watermark embedding and model evaluation."
     },
     {
       "title": "WARD: Provable RAG dataset inference via LLM watermarks",
       "authors": ["Nikola Jovanović", "Robin Staab", "Maximilian Baader", "Martin Vechev"],
       "year": 2024,
       "arxiv_id": "2410.03537",
       "relevance": "Applies watermark radioactivity concepts to RAG systems, extending the contamination detection paradigm to retrieval-augmented generation."
     },
     {
       "title": "Rethinking benchmark and contamination for language models with rephrased samples",
       "authors": ["Shuo Yang", "Wei-Lin Chiang", "Lianmin Zheng", "Joseph E Gonzalez", "Ion Stoica"],
       "year": 2023,
       "arxiv_id": "2311.04850",
       "relevance": "Shows training on reformulated questions boosts performance on originals, establishing that contamination via rephrasing is a real threat."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark creators could adopt this watermarking technique before releasing new benchmarks, though no code is provided for immediate use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the known problem of benchmark contamination and proposes a solution; does not challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Highlights the unreliability of current benchmark evaluations due to contamination, a mild integrity concern rather than a safety threat."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Feeds into the 'benchmarks are unreliable' narrative and implicitly suggests models may be scoring higher than they should due to contamination."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released; the method cannot be tried without reimplementation."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Meta FAIR is a well-known AI research lab, though the paper is not about a flagship product."
     }
   }
 }