ai-research-survey

scan.json (32340B)

---

 {
   "paper": {
     "title": "On The Fragility of Benchmark Contamination Detection in Reasoning Models",
     "authors": ["Han Wang", "Haoyu Li", "Brian Ko", "Huan Zhang"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.02386",
     "doi": "10.48550/arXiv.2510.02386"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Existing benchmark contamination detection methods are fragile against LRM contamination. RL training (GRPO) with clean data conceals SFT contamination evidence, reducing detection AUROC from ~75% to ~50% (random guessing), with PPO-style importance sampling/clipping identified as the root cause via theoretical analysis and ablation. For advanced LRMs, SFT contamination with chain-of-thought reasoning yields up to 11.76% performance inflation while making all 10 tested detection methods perform near chance, suggesting LRMs generalize from contamination rather than memorizing.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at https://github.com/ASTRAL-Group/LRM_Conta_Detection_Arena.git' — a working repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks are publicly available (AIME 2024/2025, AMC 2023, GPQA Diamond, OlympiadBench, Minerva Math). Clean training data comes from public datasets (OpenThoughts3, DeepMath-103K)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix F lists hardware (9× NVIDIA L40S, CUDA 12.8, Ubuntu 22.04) and mentions frameworks (LLaMA-Factory, Verl, FlashAttention-2, DeepSpeed ZeRO-1, Liger kernels, vLLM), but no requirements.txt, Dockerfile, or versioned dependency list is provided. Framework versions are cited only by arXiv paper references."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides detailed experimental setup in Appendix D (contamination pipelines, hyperparameters, prompt templates) and a code repository URL, but no step-by-step reproduction instructions (e.g., 'run this command') are included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-5 report point estimates for AUROC and Pass@1 with no confidence intervals or error bars. Results are 'averaged over detection scores from 8 rollouts' but no uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims that GRPO 'conceals' contamination and that detection methods 'perform near random guess' are based on comparing raw AUROC numbers without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, and 5 include a Δ column reporting absolute AUROC changes relative to the baseline (e.g., -14.22, -16.42 for GRPO concealment). Table 1 reports Pass@1 with absolute differences across conditions, providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for using these particular benchmarks or their sizes. Some benchmarks are very small (AIME 2024/2025 have only 30 problems each, yielding 15-member evaluation sets), and no power analysis or sample size rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "AUROC is averaged over 8 rollouts and Pass@1 over 3-10 rollouts, but no standard deviations, interquartile ranges, or other spread measures are reported for any results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "10 representative contamination detection methods are evaluated spanning generation-based, perturbation-based, reference-based, and reference-free approaches (Table 2). Multiple training regimes (SFT only, SFT+GRPO, SFT+RAFT, SFT+RAFT++) serve as comparison conditions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Detection methods range from 2018-2025, including recent work like Min-K%++ (Zhang et al., 2024), DICE (Tu et al., 2024), CDD (Dong et al., 2024), and Verbatim (Wu et al., 2025). The RL algorithms (GRPO, RAFT++) are also contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents a carefully designed ablation isolating the effect of importance sampling and clipping. RAFT (no clipping), RAFT++ (with/without clipping), and GRPO (with/without clipping) are compared, directly testing the theoretical prediction."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both AUROC for detection performance and Pass@1 for benchmark performance/contamination inflation, providing complementary views of the contamination problem."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated (AUROC computation, Pass@1 with automated answer checking). Human expert review of detection failures or contamination evidence could have strengthened the analysis."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For each benchmark, 'we randomly sample half of the questions as the member set (used for contamination) and leave the remaining half as the non-member set (for detection evaluation)' (Section 3). The member/non-member split provides clear separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by each of the 6 benchmarks (OlympiadBench, GPQA, AIME25, AIME24, Minerva, AMC23) and by each detection method category (generation-based, perturbation-based, reference-based, reference-free) in all tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis is provided: log-probability distributions show why detection fails (Figures 3-4), the discussion in Section 4 explains why LRMs generalize rather than memorize, and embedding visualizations (Appendix E.6, Figures 10-12) show member/non-member indistinguishability."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RAFT does NOT conceal contamination (Table 3, Δ=+2.03), RL contamination provides 'no significant difference compared to using a clean RL training set' (Section 3.1), and further SFT does NOT conceal contamination (Figure 2, Table 14). These are explicitly reported negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that (1) GRPO conceals contamination signals — supported by Table 2 showing AUROC drops averaging -14 to -20 points; (2) PPO-style clipping is the root cause — supported by Theorem 3.1 and ablation in Table 3; (3) detection methods perform near random on advanced LRMs — supported by Table 5 averaging ~50-55% AUROC."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that 'PPO-style importance sampling and clipping objectives are the root cause of detection concealment' is supported by (1) theoretical analysis (Theorem 3.1), (2) controlled ablation removing clipping from RAFT++/GRPO (Table 3), and (3) comparison of RAFT (no clipping, no concealment) vs RAFT++ (with clipping, concealment)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims fragility of 'benchmark contamination detection in reasoning models' broadly, but experiments use only 7-8B parameter models (Qwen2.5-7B-Instruct, DeepSeek-R1-Distill-Llama-8B/Qwen-7B) on 6 math/science benchmarks. No results on larger models, coding benchmarks, or other reasoning domains. The paper does not bound its claims to the tested scale or domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly rules out alternative explanations: (1) 'simply training with more clean samples' by comparing continued SFT vs GRPO (Figure 2, Table 14); (2) 'further training makes models forget contamination' by showing Pass@1 inflation persists (Table 1). Section 4 discusses the alternative that LRMs generalize rather than memorize."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures AUROC for detection and Pass@1 for benchmark performance — both directly match the claimed constructs (detection effectiveness and performance inflation). No proxy gap exists between measurements and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model checkpoints are named: 'Qwen2.5-7B-Instruct', 'DeepSeek-R1-Distill-Llama-8B', 'DeepSeek-R1-Distill-Qwen-7B', 'QwQ-32B' for distillation, and 'bespokelabs/Bespoke-Stratos-7B' as the reference model. These are identifiable checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.4 provides the actual prompt templates used: the math reasoning template ('{question}\\nPlease reason step by step, and put your final answer within \\boxed{}.') and the multiple-choice template, with concrete examples showing how they are instantiated."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.4 provides comprehensive hyperparameter tables for SFT (batch size 128, LR 4e-5, 5 epochs, cosine scheduler, etc.) and RL (batch size 64, ε=0.2, LR 1e-6, rollout num 4, temp 0.6) training, plus inference settings (temperature=0.6, top_p=0.95, max_new_tokens=32768)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains and evaluates language models directly without any scaffolding layer."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 describes the contamination pipeline: 10K clean samples from OpenThoughts3, distillation with QwQ-32B using rejection sampling (64 rollouts), 3× replication of member data. Deduplication uses '13-gram overlap deduplication' against evaluation benchmarks. Table 6 shows the proportion of questions solved after distillation rollouts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A contains a dedicated 'LIMITATIONS' section discussing the scope of the findings and acknowledging that no new detection algorithm is proposed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section (Appendix A) discusses the implications of their findings but does not address specific threats to their own study's validity, such as whether results generalize beyond 7-8B models, whether the small benchmark sizes (15 member items in AIME) introduce noise, or whether base model pre-training contamination could confound results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It generalizes from a few 7-8B models to 'LRMs' broadly without bounding claims to the tested scale, model families, or benchmark domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and benchmarks are public, the paper does not release raw detection scores, contaminated model checkpoints, or intermediate experimental data needed for independent verification of the reported AUROC and Pass@1 values."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D describes data collection in detail: contamination pipeline construction (D.1), detection method implementations (D.2), benchmark descriptions with sizes (D.3), and SFT/RL implementation specifics (D.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks (AIME, AMC, GPQA Diamond, OlympiadBench, Minerva Math) and public training datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: base model → SFT contamination (10K clean + 1,866 member × 3 replications = 11,866 samples) → RL training (4,096 clean + optional members). Detection pipeline: 8 rollouts per question → compute detection score per rollout → average. Deduplication step is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding, acknowledgments, or grants section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Illinois Urbana-Champaign and University of Washington. None of the authors are affiliated with the model providers being evaluated (DeepSeek, Qwen/Alibaba)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding source is disclosed. The absence of a funding disclosure makes this unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for the base models (Qwen2.5-7B-Instruct, DeepSeek-R1-Distill models). While the paper controls its own contamination, the base models may have already been exposed to the evaluation benchmarks during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is the central topic. The paper explicitly splits benchmarks into member/non-member halves and uses 13-gram deduplication on clean training data against evaluation benchmarks to prevent unintended overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the entire subject of the paper. They control contamination as an experimental variable and explicitly address it through member/non-member splits and deduplication of clean data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example wall-clock times are reported. The paper generates many rollouts per question (8 for detection, 3-10 for Pass@1 across 6 benchmarks) but does not quantify the computational cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix F states hardware specs (9× NVIDIA L40S GPUs, 48 GiB each) but does not report total GPU hours, wall-clock training time, or total compute spent across all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. The member/non-member split is performed once ('we randomly sample half of the questions') with no sensitivity analysis across different random splits."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Each AUROC is averaged over detection scores from 8 rollouts' and 'we evaluate pass@1 and run 10 rollouts on AIME 2024 & 2025, AMC 2023, and 3 rollouts on OlympiadBench, GPQA Diamond, and Minerva Math' (Appendix D.4)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The paper adopts SFT hyperparameters 'suggested by OpenThought3' and uses fixed RL hyperparameters without stating whether alternatives were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses recommended hyperparameters from prior work ('we adopt the SFT hyperparameters suggested by OpenThought3 for medium dataset scales', Appendix D.4), providing transparent justification for configuration choices rather than cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement 10 detection methods from other groups but do not acknowledge the potential bias of their own implementations of these baselines. No independent evaluation or verification of implementation correctness is discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 and Tables 11-12 show AUROC as a function of RL training steps (0, 64, 110, 156 steps), providing performance-vs-compute curves that reveal the monotonic decline in detection with more training."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses AUROC as the sole measure of detection effectiveness without discussing whether it adequately captures real-world detection utility (e.g., at what false positive rate a method becomes useful). No discussion of whether the member/non-member random split on small benchmarks is a valid construct for evaluating practical detectability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are trained and evaluated directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the base models (Qwen2.5-7B-Instruct, DeepSeek-R1-Distill) may have already been exposed to the evaluation benchmarks (AIME, GPQA, etc.) during pre-training. AIME 2024 problems may have appeared in 2025-era model training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup inadvertently leaks information. For example, non-members from the same benchmark may share structural features with members, making them not truly 'unseen' in distribution."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Members and non-members are randomly split from the same benchmarks but share strong distributional similarity (same competition, same difficulty distribution). This non-independence is not discussed and could affect AUROC measurements."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses '13-gram overlap deduplication' on clean training datasets against evaluation benchmarks 'to ensure conclusive results' (Appendix D.4). This is a concrete decontamination method applied to their own training data."
       }
     }
   },
   "claims": [
     {
       "claim": "Contamination inflation mainly comes from SFT, not RL training",
       "evidence": "Table 1 shows SFT contamination yields 8.82% additional Pass@1 inflation across six benchmarks, while RL contamination shows 'no significant difference compared to using a clean RL training set.'",
       "supported": "strong"
     },
     {
       "claim": "SFT contamination is initially detectable by existing methods",
       "evidence": "Table 2 shows reference-free methods achieve ~73-75% AUROC and LiRA achieves 89.13% AUROC on SFT-contaminated models before RL training.",
       "supported": "strong"
     },
     {
       "claim": "GRPO training conceals SFT contamination evidence across all detection methods",
       "evidence": "Table 2 shows consistent AUROC decreases after GRPO across all 10 detection methods and 6 benchmarks. Average drops range from -0.60 (Verbatim) to -19.84 (Max-K%). Figure 2 shows monotonic AUROC decline with more GRPO steps.",
       "supported": "strong"
     },
     {
       "claim": "PPO-style importance sampling and clipping is the root cause of detection concealment",
       "evidence": "Theorem 3.1 provides theoretical analysis; Table 3 shows ablation where removing clipping from GRPO raises AUROC from 61.26% to 73.28% (Δ from -14.22 to -2.20), and RAFT (no clipping) shows Δ=+2.03 while RAFT++ (with clipping) shows Δ=-17.91.",
       "supported": "strong"
     },
     {
       "claim": "Extensive SFT contamination with CoT on advanced LRMs leaves little detectable evidence",
       "evidence": "Table 5 shows almost all detection methods perform near random guessing (AUROC ≈50%) across three advanced LRMs and six benchmarks. Best method (LiRA) achieves only 58.74% average AUROC.",
       "supported": "strong"
     },
     {
       "claim": "LRMs generalize from contamination rather than memorizing specific sequences",
       "evidence": "Figures 4, 8, and 9 show that after SFT contamination, log-probability distributions shift upward for both members and non-members at similar margins, and embedding visualizations (Figures 10-12) show member/non-member embeddings are indistinguishable. Discussed in Section 4.",
       "supported": "moderate"
     },
     {
       "claim": "Further GRPO training preserves performance inflation while reducing detectable evidence",
       "evidence": "Table 1 shows the SFT+GRPO contaminated model retains 7.14% average performance inflation vs clean SFT, while Table 2 and Figure 2 show AUROC drops toward random guessing.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "No error bars, confidence intervals, or standard deviations are reported for any AUROC or Pass@1 results despite averaging over multiple rollouts. With very small evaluation sets (15 member items for AIME), point estimates may be highly unstable."
     },
     {
       "flag": "Very small evaluation set sizes",
       "detail": "AIME 2024 and 2025 have only 30 problems each (15 members, 15 non-members). AMC 2023 has 40 (20/20). AUROC computed on 15-20 items is inherently noisy and may not reliably discriminate between methods. No discussion of the statistical power implications."
     },
     {
       "flag": "Limited model scale",
       "detail": "All experiments use 7-8B parameter models only. The paper's claims about 'LRMs' broadly are not tested at larger scales (e.g., 70B, 405B) where contamination dynamics and detection may differ substantially."
     },
     {
       "flag": "No significance testing for key claims",
       "detail": "Claims that detection methods 'perform near random guess' or that GRPO 'conceals' contamination are based on comparing raw numbers without statistical tests. Some reported AUROC values (e.g., 55-62%) may not be statistically distinguishable from 50% given the small sample sizes."
     }
   ],
   "cited_papers": [
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["Simone Balloccu", "Patrícia Schmidtová", "Mateusz Lango", "Ondřej Dušek"],
       "year": 2024,
       "arxiv_id": "2402.03927",
       "relevance": "Documents data contamination and evaluation malpractices in closed-source LLMs, directly relevant to benchmark integrity."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia"],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "Proposes Min-K% detection method for pre-training data, a key baseline evaluated in this paper."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Proposes the CDD generation-based contamination detection method evaluated in this paper."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Foundational LRM work whose distilled models are used as experimental subjects for Stage II contamination."
     },
     {
       "title": "The emperor's new clothes in benchmarking? A rigorous examination of mitigation strategies for LLM benchmark data contamination",
       "authors": ["Yifan Sun", "Han Wang", "Dongbai Li"],
       "year": 2025,
       "arxiv_id": "2503.16402",
       "relevance": "Examines mitigation strategies for LLM benchmark contamination, directly related to evaluation integrity."
     },
     {
       "title": "Does data contamination detection work (well) for LLMs? A survey and evaluation on detection assumptions",
       "authors": ["Yujuan Fu", "Ozlem Uzuner", "Meliha Yetisgen", "Fei Xia"],
       "year": 2024,
       "arxiv_id": "2410.18966",
       "relevance": "Surveys contamination detection assumptions and their effectiveness, providing foundational context for this work."
     },
     {
       "title": "Evading data contamination detection for language models is (too) easy",
       "authors": ["Jasper Dekoninck", "Mark Niklas Müller", "Maximilian Baader"],
       "year": 2024,
       "arxiv_id": "2402.02823",
       "relevance": "Demonstrates evasion of contamination detection through benchmark augmentation, a precursor to this paper's findings on RL-based concealment."
     },
     {
       "title": "Reasoning or memorization? Unreliable results of reinforcement learning due to data contamination",
       "authors": ["Mingqi Wu", "Zhihao Zhang", "Qiaole Dong"],
       "year": 2025,
       "arxiv_id": "2507.10532",
       "relevance": "Studies how RL training results become unreliable due to data contamination, complementary to this paper's findings."
     },
     {
       "title": "How much do language models memorize?",
       "authors": ["John X Morris", "Chawin Sitawarin", "Chuan Guo"],
       "year": 2025,
       "arxiv_id": "2505.24832",
       "relevance": "Studies memorization in language models, providing theoretical grounding for the memorization assumptions that this paper challenges."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["Cheng Xu", "Shuhao Guan", "Derek Greene"],
       "year": 2024,
       "arxiv_id": "2406.04244",
       "relevance": "Comprehensive survey of benchmark contamination in LLMs, providing the taxonomic framework for detection methods."
     },
     {
       "title": "LLMs can easily learn to reason from demonstrations structure, not content, is what matters!",
       "authors": ["Dacheng Li", "Shiyi Cao", "Tyler Griggs"],
       "year": 2025,
       "arxiv_id": "2502.07374",
       "relevance": "Demonstrates that LLMs learn reasoning structure rather than content from demonstrations, relevant to why contamination with CoT generalizes."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO training algorithm that this paper identifies as a key mechanism for contamination concealment."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Findings inform contamination detection researchers and leaderboard operators, but not directly usable as a tool by practitioners."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "Directly challenges the widely-held assumption that contamination detection methods work, showing they are trivially evaded by standard RL training."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises alarm about the integrity of LLM leaderboards and the ease of undetectable benchmark gaming, undermining trust in model evaluations."
     },
     "drama_conflict": {
       "score": 3,
       "justification": "Strong 'leaderboards are fake' angle — demonstrates that model developers could easily contaminate LRMs to achieve inflated rankings while evading all 10 tested detection methods."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but this is a research pipeline requiring significant compute (9× L40S GPUs), not something easily tried."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University researchers (UIUC, UW) without major lab affiliation. Uses DeepSeek models which have moderate recognition."
     }
   }
 }