ai-research-survey

scan-v5.json (29039B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "On The Fragility of Benchmark Contamination Detection in Reasoning Models",
     "authors": [
       "Han Wang",
       "Haoyu Li",
       "Brian Ko",
       "Huan Zhang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.02386",
     "doi": "10.48550/arXiv.2510.02386"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims — that GRPO conceals SFT contamination, that CoT contamination on LRMs evades detection near random guess, and that PPO-style clipping is the root cause — are directly supported by Tables 1–5 and Theorem 3.1.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that PPO-style importance sampling/clipping causes concealment is supported by controlled ablations removing the clipping term (Table 3) and a formal mathematical proof in Appendix C; both independently reach the same conclusion.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'a broad class of RL methods may inherently exhibit similar concealment capability' based solely on GRPO and RAFT++; this generalization is not bounded to which specific methods share the PPO clipping objective and which do not.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly addresses and rules out the alternative that GRPO simply causes models to forget contamination, using two experiments: RL on contaminated data showing comparable AUROC drops, and further clean SFT failing to conceal contamination (Figure 2, Table 14).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "AUROC measures detection performance and pass@1 measures performance inflation; the paper distinguishes these throughout, and Section 3.1 explicitly discusses that performance inflation persists even when contamination is undetectable.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A is titled 'LIMITATIONS' and discusses that no new detection algorithm is proposed and that current detectors' failure stems from log-probability assumptions.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section does not discuss specific threats to experimental validity, such as the single base model used (Qwen2.5-7B-Instruct) for Stage I, the extremely small member/non-member sets (15 samples each for AIME), or the limited 156 RL training steps relative to real-world scales.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show; there is no discussion of whether findings generalize beyond the tested model sizes, model families, or contamination ratios, and the limitations section focuses on what was not built rather than what was not proven.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section appears anywhere in the paper; no grants, industry support, or other funding sources are mentioned.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors' institutional affiliations (University of Illinois Urbana-Champaign and University of Washington) are clearly disclosed on the first page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations appear anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms including 'benchmark contamination,' 'SFT contamination,' 'RL contamination,' 'member/non-member sets,' and 'LRMs' are all explicitly defined in Section 1 and Section 3's contamination/detection setups.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it presents 'the first systematic study of benchmark contamination in LRMs' structured around two contamination scenarios, contributing empirical evidence of detection fragility and a theoretical explanation of the mechanism.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides dedicated related work covering LRMs, contamination detection methods (5 categories, 10+ methods), and prior concealment work, explicitly positioning this as the first algorithmic-level concealment analysis.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code is explicitly stated as available at https://github.com/ASTRAL-Group/LRM_Conta_Detection_Arena.git in the abstract.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks used (AIME 2024/2025, GPQA Diamond, OlympiadBench, Minerva Math, AMC 2023) are standard publicly available datasets used without modification.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix F describes hardware and names frameworks (LLaMA-Factory, Verl, vLLM, FlashAttention-2) but provides no requirements.txt, Dockerfile, or pinned dependency versions; exact software environment is not reproducible from this information alone.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Detailed hyperparameters are scattered across Appendix D.4 but no step-by-step reproduction instructions exist; a reader would need to reverse-engineer the pipeline from the code and appendix tables without a clear entry point.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All AUROC values in Tables 2, 3, 5, 11, and 12 are reported as point estimates only; no confidence intervals or error bars are reported despite results being averaged over 8 rollouts.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are performed for any comparative claims; AUROC differences between training conditions are presented without formal testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Delta (∆) columns in Tables 2, 3, 11, and 12 report absolute AUROC differences from baseline, providing effect size context for all comparative claims across detection methods and benchmarks.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 8 rollouts and 50/50 member/non-member split is not justified; AIME benchmarks have only 30 problems total, yielding 15 members and 15 non-members — too small for reliable AUROC estimation and not discussed as a concern.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No standard deviations or variance are reported across the 8 rollouts used to average detection scores; all result tables report only mean AUROC values.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Ten representative detection methods spanning generation-based, perturbation-based, reference-based, and reference-free approaches serve as baselines, compared before and after RL training.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Detection baselines span 2021–2025 including recent methods Min-K%++ (2024), CDD (2024), LLM Dataset Inference (2024), and Verbatim (2025), covering the current state of the art.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 ablates the clipping term in RAFT++ and GRPO, directly isolating its contribution to contamination concealment and confirming the theoretical prediction.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both AUROC (contamination detection performance) and pass@1 (benchmark performance inflation) are reported throughout, measuring detection ability and practical impact separately.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "The evaluation involves automated scoring on math and science reasoning benchmarks; human evaluation is not relevant.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Each benchmark is split 50/50 into member (training) and non-member (held-out) sets; AUROC is computed by comparing detection scores between these disjoint sets.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "All result tables provide per-benchmark breakdowns across six benchmarks (OlympiadBench, GPQA Diamond, AIME 2025, AIME 2024, Minerva Math, AMC 2023), allowing assessment of variability across task types.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4's discussion analyzes why detection fails on LRMs — LRMs generalize broadly from member to non-member distributions after CoT contamination, undermining the memorization assumption — providing mechanistic understanding of failure modes.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that RL contamination provides no performance benefit (Table 1), RAFT cannot conceal contamination (Tables 3, 12), and embedding-based methods fail to distinguish members from non-members even before contamination (Figures 10–12).",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model names are provided throughout: Qwen2.5-7B-Instruct, DeepSeek-R1-Distill-Llama-8B, DeepSeek-R1-Distill-Qwen-7B, OpenThought3-7B (15K), QwQ-32B (for distillation), and Bespoke-Stratos-7B (reference model).",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.4 provides exact prompt templates with full text for both math reasoning and multiple-choice formats, including complete worked examples.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.4 provides complete hyperparameter tables for SFT (batch size, LR, context length, epochs, scheduler, warmup ratio) and RL (batch size, prompt/response length, ε, LR, rollout number, temperature) stages.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.2 provides precise formulas for all 10 detection methods with implementation choices documented (e.g., α=0.5 for CDD, k=20 for Min-K%, why response tokens are used over question tokens).",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.4 documents 13-gram overlap deduplication against benchmarks; Appendix D.1 describes the contamination pipeline including QwQ-32B distillation settings (temperature 0.6, top-p 0.95, 64 rollouts) and 3x question replication.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The contaminated training datasets (member sets with QwQ-32B-distilled reasoning chains) are not released as standalone artifacts; reproducing them requires running the full distillation pipeline.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Appendices D.1 and D.3 describe the contamination pipelines and benchmark sources in detail, including exact split procedures, benchmark sizes, and distillation parameters.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; standard public benchmarks are used without any participant recruitment.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The complete pipeline from benchmark selection → 50/50 member/non-member split → SFT/RL contamination training → detection evaluation is documented across Sections 3–4 and Appendices D.1–D.4.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for the base models (Qwen2.5-7B-Instruct, DeepSeek-R1 distillations) are not stated; prior exposure to these benchmarks in pretraining is not discussed.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Train-test overlap is the paper's central subject; the authors use 13-gram deduplication of clean training data against benchmarks and explicitly control contamination through the experimental design.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Benchmark contamination is the core focus; the paper directly controls and measures it throughout, studying both deliberate contamination and RL-induced concealment in systematic experiments.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost or latency is reported; vLLM is mentioned and 8 rollouts per question are used, but no runtime or cost estimates are provided.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Appendix F specifies the hardware platform (9x NVIDIA L40S GPUs, 48 GiB each) but does not state total GPU-hours, wall-clock training time, or dollar cost for any experiment.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "GRPO training with clean samples markedly conceals SFT contamination signals, with AUROC dropping 8–20pp across all detection methods",
       "evidence": "Table 2 shows systematic AUROC drops after 156 GRPO steps (Max-K%: −19.84pp, Loss: −16.68pp, Min-K%: −16.42pp); Figure 2 shows monotonic decline with each additional step",
       "supported": "strong"
     },
     {
       "claim": "PPO-style importance sampling and clipping are the root cause of contamination concealment, not generic further training",
       "evidence": "Table 3 ablation: removing clipping from RAFT++ reduces its concealment effect from Δ−17.91 to Δ−1.09; RAFT (no clipping) shows no concealment (Δ+2.03); supported by Theorem 3.1",
       "supported": "strong"
     },
     {
       "claim": "Contamination inflation comes primarily from SFT, with RL contamination providing negligible additional benefit",
       "evidence": "Table 1: SFT contamination adds 8.82pp pass@1 on average; adding RL contamination shows no significant difference vs. clean RL training",
       "supported": "strong"
     },
     {
       "claim": "Extensive CoT contamination of advanced LRMs at the final training stage leaves barely any detectable evidence",
       "evidence": "Table 5: all 10 detection methods achieve near-random AUROC (~50%) across three LRM models and six benchmarks after extensive SFT CoT contamination",
       "supported": "strong"
     },
     {
       "claim": "LRMs generalize from member to non-member distributions after CoT contamination, invalidating the memorization assumption behind all current detectors",
       "evidence": "Figure 4 shows log-prob of non-members increases at similar margin as members after contamination; Section 4 provides mechanistic analysis",
       "supported": "moderate"
     },
     {
       "claim": "Further SFT training on clean data cannot make models forget contamination or suppress performance inflation",
       "evidence": "Table 14: additional 4 epochs of clean SFT on contaminated model shows pass@1 continuing to rise by 0.25pp and contamination remaining undetectable",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "Existing contamination detection methods are fundamentally fragile against Large Reasoning Models. In Stage I, GRPO training with clean samples after SFT contamination systematically reduces detection AUROC by 8–20pp, with the PPO-style importance sampling/clipping mechanism identified as the root cause via controlled ablation (removing clipping eliminates concealment) and formal proof. In Stage II, contaminating advanced LRMs with CoT data reduces all detection methods to near-random guessing (~50% AUROC) because LRMs generalize broadly from member to similar non-member distributions, invalidating the memorization assumption underlying all current detection methods. Together these findings reveal that developers could deliberately contaminate LRMs to inflate leaderboard scores while leaving minimal detectable traces.",
   "red_flags": [
     {
       "flag": "Single base model for Stage I",
       "detail": "All Stage I experiments use only Qwen2.5-7B-Instruct; generalization to other model families, scales, or architectures is asserted but not empirically tested."
     },
     {
       "flag": "Tiny member/non-member sets for AIME",
       "detail": "AIME 2024 and 2025 benchmarks have only 30 problems total, yielding 15 members and 15 non-members per set — insufficient for reliable AUROC estimation, yet used for primary claims."
     },
     {
       "flag": "No confidence intervals or significance tests",
       "detail": "All AUROC values are point estimates across 8 rollouts with no error bars or significance testing; it is impossible to assess whether reported differences are statistically meaningful."
     },
     {
       "flag": "Limited RL training steps vs. real-world scales",
       "detail": "Maximum 156 GRPO steps used; real-world LRM training uses orders of magnitude more steps. The paper acknowledges this gap but extrapolates that 'extensive GRPO training would render all existing detection methods to near-random performance.'"
     },
     {
       "flag": "Training cutoffs of base models unstated",
       "detail": "The paper does not state whether AIME, GPQA, or other benchmarks appeared in the pretraining data of Qwen2.5-7B-Instruct or DeepSeek distillation models, potentially confounding the controlled contamination setup."
     }
   ],
   "cited_papers": [
     {
       "title": "Extracting training data from large language models (Carlini et al., 2021)",
       "relevance": "Foundational membership inference work providing Loss, Ref, and Zlib detection methods — three of the ten baselines directly evaluated in this paper"
     },
     {
       "title": "Detecting pretraining data from large language models (Shi et al., 2023)",
       "relevance": "Min-K% detection method — one of the primary detection baselines evaluated across all experiments"
     },
     {
       "title": "Min-K%++: Improved baseline for detecting pre-training data from large language models (Zhang et al., 2024)",
       "relevance": "Min-K%++ detection method — evaluated as the improved version of Min-K%, showing similar fragility under RL concealment"
     },
     {
       "title": "LLM dataset inference: Did you train on my dataset? (Maini et al., 2024)",
       "relevance": "Max-K% detection method and dataset inference framing — directly evaluated and shown to be fragile"
     },
     {
       "title": "An empirical analysis of memorization in fine-tuned autoregressive language models (Mireshghallah et al., 2022)",
       "relevance": "LiRA detection method — the highest-performing baseline in Stage I (89.13% AUROC) that still drops significantly after GRPO"
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models (Dong et al., 2024)",
       "relevance": "CDD detection method and key prior work on contamination in non-reasoning models; this paper extends that work to the LRM setting"
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning (Guo et al., 2025)",
       "relevance": "Source of DeepSeek-R1 distillation models used in Stage II experiments and the GRPO algorithm central to Stage I analysis"
     },
     {
       "title": "Does data contamination detection work (well) for LLMs? A survey and evaluation on detection assumptions (Fu et al., 2024)",
       "relevance": "Survey of contamination detection methods and their underlying assumptions — directly relevant to the paper's finding that those assumptions fail for LRMs"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly threatens the integrity of all LLM leaderboards and benchmark evaluations used for model selection, procurement, and safety assessments."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "Shows that standard RL training inadvertently (or deliberately) erases contamination traces, upending the assumption that contamination is detectable if you look hard enough."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises concrete concerns about systematic capability overstating in frontier AI evaluations, with direct implications for AI governance and capability assessments."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implies model developers can game leaderboards with near-zero detection risk, framing the LLM evaluation ecosystem as a fundamentally adversarial arms race developers are currently winning."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released but requires training LLMs from scratch on 9x NVIDIA L40S GPUs, making live demonstration impractical for most readers."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From UIUC and University of Washington — respected institutions but not top commercial AI labs with immediate leaderboard authority."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45693591",
         "title": "ChunkLLM: A Lightweight Pluggable Framework for Accelerating LLMs Inference",
         "points": 96,
         "comments": 8,
         "url": "https://news.ycombinator.com/item?id=45693591"
       },
       {
         "hn_id": "45235119",
         "title": "Instruction-Following Pruning for Large Language Models",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45235119"
       },
       {
         "hn_id": "37921371",
         "title": "Quantum Computing: Principles and Applications",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37921371"
       },
       {
         "hn_id": "45843183",
         "title": "Mathematical Exploration and Discovery at Scale",
         "points": 4,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=45843183"
       },
       {
         "hn_id": "45862519",
         "title": "Mathematical exploration and discovery at scale – Terence Tao et al.",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45862519"
       },
       {
         "hn_id": "45721820",
         "title": "The Fragility of Benchmark Contamination Detection in Reasoning Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45721820"
       },
       {
         "hn_id": "45837025",
         "title": "Mathematical Exploration and Discovery at Scale",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45837025"
       },
       {
         "hn_id": "41801438",
         "title": "Comprehensive Survey of Mamba Architectures for Medical Image Analysis,Beyond",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41801438"
       },
       {
         "hn_id": "45607220",
         "title": "Conceptualizing/Modeling Communication-Based Cyberattacks on Automated Vehicles",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45607220"
       }
     ],
     "top_points": 96,
     "total_points": 121,
     "total_comments": 11
   }
 }