ai-research-survey

scan-v4.json (32272B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Detecting Benchmark Contamination Through Watermarking",
     "authors": [
       "Tom Sander",
       "Pierre Fernandez",
       "Saeed Mahloujifar",
       "Alain Durmus",
       "Chuan Guo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.17259",
     "doi": "10.48550/arXiv.2502.17259"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'p-val = 10^-3 for +5% on ARC-Easy.' Table 1 shows 4 contaminations on ARC-Easy: log10(p)=-3.0, Acc +4.3%, which is consistent. Claims of preserved utility are supported by Figure 3a and Figure 7.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's experimental design controls contamination levels by training models from scratch with and without benchmark data, enabling causal inference about the relationship between contamination and detection confidence. This is a controlled experiment with single-variable manipulation.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Results are tested on 3 benchmarks, 3 model sizes, and 4 tokenizer families. The limitations section explicitly bounds scope: 'primarily designed for unintentional contamination' and acknowledges that math/code questions are harder to watermark.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Proposition 1 explicitly distinguishes three meanings of 'contaminated': (i) trained on test set, (ii) memorized the watermark, (iii) performance artificially enhanced. The paragraph below Proposition 1 acknowledges the scenario where a model performs well without overfitting to watermark biases.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between what is measured (watermark memorization, Definition ii) and what is claimed (benchmark contamination). The text below Proposition 1 states: 'This is distinct from the hypothesis that the model's test performance is not artificially enhanced due to training on the benchmark.'",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations & Conclusion' and contains two substantive bullet points addressing rephrasing impact and intentional evasion.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 identifies study-specific threats: 'some questions lose coherence after rephrasing' (Figure 6 demonstrates this), and 'Malicious actors could rephrase questions to weaken the watermark or train only on answers conditioned on questions, which would bypass radioactivity detection.'",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the method is 'primarily designed for unintentional contamination.' It explicitly acknowledges that intentional evasion (rephrasing to remove watermarks, training only on answers) would bypass detection. Section 4.3 notes that math/code questions present challenges for rephrasing.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present. Authors are Meta FAIR employees and École polytechnique affiliates, implying corporate funding, but this is not explicitly disclosed.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: 'Meta FAIR' and 'École polytechnique CMAP' in the paper header.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta has a vested interest in the integrity of LLM benchmarks, as it trains and releases Llama models that are evaluated on them. The paper uses exclusively Meta models (Llama family) for all experiments. No statement of funder independence.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure section is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'contamination' is given three explicit definitions in Section 3.2, 'radioactivity' is defined via reference to Sander et al. (2024), 'benchmark utility' is operationalized as similar accuracy across watermarked and original versions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction lists three explicit contributions: (1) rephrasing benchmarks with watermarking, (2) extending watermark radioactivity to pre-training, and (3) a new detection algorithm for different tokenizers.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides a thorough related work review covering membership inference limitations, canary approaches, prior watermarking schemes, and radioactivity, explicitly explaining why existing methods fail and how this work extends Sander et al. (2024).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository URL is provided in the paper. They reference Meta Lingua (Videau et al., 2024) as the training framework but do not release their own watermarking or detection code.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The original benchmarks (ARC, MMLU) are public, but the watermarked versions of the benchmarks and the trained model checkpoints are not released.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "They mention A-100 GPUs with 80GB memory and the Meta Lingua framework, but provide no requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided. The method and hyperparameters are described but there is no README, script, or explicit guide to replicate experiments.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Accuracy numbers and p-values are reported as point estimates. No confidence intervals or error bars appear on accuracy results. The paper notes that [-1, 0] is a 90% CI for log10(p) under H0, but this is a theoretical property of the test, not an uncertainty measure on the main results.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "The core contribution is a binomial significance test for contamination detection (Equation 2, Proposition 1). P-values are reported throughout (Table 1, Figures 3b, 4, 8). The test has a formally proven False Positive Rate equal to α.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports accuracy improvements with baseline context: e.g., '+4.3%' (from 53.5% to 57.9%), '+9.5%' (from 53.5% to 63.0%), '+18.2%' (from 53.5% to 71.7%) for ARC-Easy at different contamination levels.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "MMLU* uses 5000 questions 'to accelerate experimentation and maintain a comparable size to the other benchmarks,' but no power analysis or formal justification for any of the benchmark sizes is provided.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be from single training runs. No standard deviation, variance across seeds, or spread measures are reported for accuracy or detection metrics.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.3 compares against canary-based contamination detection as an alternative baseline (Table 3). They also compare contaminated vs. uncontaminated models.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The canary method from BIG-bench (Srivastava et al., 2022) is a standard and commonly used approach for contamination detection. Membership inference attacks (Duan et al., 2024) are discussed as contemporary alternatives that are ineffective.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.3 provides extensive ablations: watermark strength δ (0, 1, 2, 4), window size k (0, 1, 2), model size (135M, 360M, 1B), number of contaminations (0, 4, 8, 16), rephrasing model (8B vs 70B), and tokenizer (Section 4.4).",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Results are reported on two distinct metrics: benchmark accuracy (performance impact) and detection confidence (log10 p-value from the radioactivity test). Both are reported in Table 1 and throughout Section 4.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of the watermarked questions' quality. Benchmark utility is assessed entirely through automated comparison of model performance on original vs. watermarked versions.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Models are trained with controlled contamination and evaluated on the benchmark test sets. An out-of-distribution (OOD) evaluation template is used to further separate training and evaluation contexts (Section 4.2).",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per benchmark (ARC-Easy, ARC-Challenge, MMLU*) in Table 1, per model size in Figure 4, per window size in Table 2 and Figures 7-8, and per watermark strength in Figure 3.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Figure 6 shows a concrete rephrasing failure on a math question. Section 4.3 discusses how smaller benchmarks lead to weaker detection (ARC-Challenge with 4 contaminations has 'doubtful' p-value). Section 5 discusses evasion scenarios.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 shows the canary-based method fails even with 160 contaminations (p-val=0.19). They report that detection on ARC-Challenge with 4 contaminations is 'doubtful' (log10(p)=-1.2). Figure 6 shows the 8B rephrasing model failing on math questions.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are given: 'Llama-3.1-8B-Instruct' for rephrasing, 'Llama-3.3-1B', 'Llama-3.3-3B', 'Llama-3.1-8B' for evaluation. Tokenizer experiments use Llama-1/2, Llama-3, Gemma-1/2, Gemma-3 (Table 4).",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figure 5 provides the exact system prompt ('You are a problem rephrasing assistant...') and instruction. Section 4.2 provides the exact evaluation templates for both in-distribution and OOD evaluation.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 reports top-p=0.7, temperature=0.5, k=2, γ=50%. Section 4.2 reports hidden dim=2048, 25 layers, 16 attention heads, LR=3×10^-3, weight decay=0.033, warmup=5000 steps, batch size=4, sequence length=4096, gradient clipping=1.0.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The method involves direct LLM generation for rephrasing and standard model training/evaluation.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 describes the contamination procedure in detail: batches taken between steps 2500-7500, format template provided, DCLM as base training data. Section 4.1 describes MMLU* as a random subset of 5000 questions across all disciplines.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No trained model checkpoints, watermarked benchmark datasets, or raw experimental outputs are released for independent verification.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Training data source is DCLM (Li et al., 2024). Benchmarks are ARC-Easy (1172 questions), ARC-Challenge (2372 questions), MMLU* (5000 randomly selected questions). Contamination injection procedure is fully described in Section 4.2.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data sources are standard public benchmarks (ARC, MMLU) and a public training corpus (DCLM).",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline is documented: benchmark selection → watermarked rephrasing with specified parameters → training with controlled contamination injection at specified steps → evaluation with specified templates → radioactivity detection scoring.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "The paper trains models from scratch with fully controlled training data (10B tokens from DCLM with specified contamination injections). The entire training pipeline is described, so the 'cutoff' is effectively complete — the authors know exactly what the model saw.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Train-test overlap is the central topic of the paper. They intentionally control contamination levels (0, 4, 8, 16 injections) and analyze the resulting overlap's impact on detection and performance.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Benchmark contamination is the paper's primary research question. The entire method is designed to detect it, and experiments systematically vary contamination levels to validate detection.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. All experiments involve training and evaluating language models.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C states: 'Each radioactivity detection test took less than 30 minutes on a single GPU.' This quantifies the cost of the detection method.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C provides detailed compute: '1B models was conducted on 8 nodes (so 64 GPUs) and took approximately six hours.' Total budget estimated at 'approximately 10,000 GPU hours' including all experiments.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No mention of multiple random seeds. All experiments appear to be single-run training with single-seed evaluation.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is not explicitly stated. Results appear to come from single training runs for each configuration.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget is reported. Training hyperparameters are listed but no indication of how they were selected or how many configurations were tried.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "Results are reported across all tested configurations (multiple δ values, window sizes, model sizes, contamination levels, tokenizers) rather than selecting a single best. The paper sweeps over parameter space rather than optimizing.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Multiple statistical tests are performed across 3 benchmarks, 4 contamination levels, 4 watermark strengths, and 3 window sizes. No correction for multiple comparisons is applied, though many p-values are so extreme (10^-12) that correction would not change conclusions.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors propose the watermarking method and evaluate it themselves. No discussion of self-comparison bias or independent evaluation.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "While different model sizes are tested (135M, 360M, 1B), performance is not systematically analyzed as a function of compute budget. The focus is on contamination level, not compute.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether ARC-Easy, ARC-Challenge, and MMLU actually measure the capabilities they claim to measure. The benchmarks are treated as given without questioning their construct validity.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is used. Models are evaluated directly via loss comparison on answer choices.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The paper trains models from scratch on controlled data, fully addressing temporal leakage by design. The authors know exactly when each piece of data enters training (contamination injected between steps 2500 and 7500).",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 uses an out-of-distribution (OOD) evaluation template distinct from the contamination template to ensure the model isn't merely memorizing the formatting context. Results are reported for both in-distribution and OOD evaluation.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Each benchmark uses a different secret watermarking key s, ensuring independence between detection tests. The base training corpus (DCLM) is separate from the benchmarks, and the paper controls exactly which benchmark data enters training.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": true,
           "justification": "The paper's entire contribution IS a concrete leakage detection method: watermark embedding via rephrasing followed by a binomial radioactivity test with formal p-value guarantees (Proposition 1, Algorithm 1).",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Watermarking benchmarks via LLM rephrasing preserves benchmark utility: models rank the same and achieve similar accuracy on watermarked vs. original versions even at 80% green-token density.",
       "evidence": "Figure 3a and Figure 7 show Llama-3.2-1B/3B and Llama-3.1-8B achieve comparable accuracy on all watermarked versions of ARC-Easy; Table 1 reports baseline accuracy with 0 contaminations matching original benchmark performance.",
       "supported": "strong"
     },
     {
       "claim": "Benchmark contamination can be reliably detected via radioactivity testing with p-values as low as 10^-6 even for modest performance gains (~10%).",
       "evidence": "Table 1 shows log10(p) < -12 for 8 contaminations (+9.5% gain on ARC-Easy) and log10(p) = -5.7 for 4 contaminations (+5.1% gain on MMLU*).",
       "supported": "strong"
     },
     {
       "claim": "Canary insertion (as used in BIG-bench) is less effective than radioactivity for contamination detection.",
       "evidence": "Table 3 shows a 360M model trained with 160 MMLU* contaminations (10× more than the most contaminated radioactivity setup) fails to memorize a 64-digit canary sufficiently to yield a low p-value.",
       "supported": "strong"
     },
     {
       "claim": "The detection method generalizes across different tokenizers used by the watermarking and suspect models.",
       "evidence": "Table 4 shows detection confidence log10(p) between -7 and -15 for contaminated models using Llama-1/2, Gemma-1/2, and Gemma-3 tokenizers, while uncontaminated models yield p-values near 0.5.",
       "supported": "strong"
     },
     {
       "claim": "Larger benchmarks, stronger watermarks, and more contamination passes all increase detection confidence proportionally.",
       "evidence": "Table 1 and Figure 3b demonstrate monotonic increases in -log10(p) with both δ (watermark strength) and number of contaminations; Table 2 shows benchmark size directly increases the number of scored tokens and thus detection confidence.",
       "supported": "strong"
     },
     {
       "claim": "The false positive rate of the contamination test equals the significance threshold α by design.",
       "evidence": "Proposition 1 provides a formal proof based on independence of token predictions under H0, confirmed empirically by uncontaminated models consistently yielding log10(p) ≈ -0.3.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "Benchmark contamination can be reliably detected by embedding LLM watermarks into benchmark questions during rephrasing before release; models trained on contaminated watermarked data retain detectable radioactive traces even for modest performance gains (p-val ≤ 10^-5 for +5% accuracy gain on MMLU*). Benchmark utility is preserved across all watermark strengths tested (up to 80% green tokens), with model rankings unchanged. The canary-based alternative completely fails under the same conditions (160 contaminations, still no significant detection). The method extends to cross-tokenizer scenarios with only modest loss in detection confidence, and detection scales predictably with benchmark size, watermark strength, and number of contamination passes.",
   "red_flags": [
     {
       "flag": "No code or model release",
       "detail": "Despite sufficient implementation detail to describe the method, no code repository or watermarked benchmark versions are released, making independent replication require full re-implementation."
     },
     {
       "flag": "Only 1B parameter models tested",
       "detail": "All training experiments use models up to 1B parameters; whether the method scales to the 70B–405B range of production LLMs is not validated, only speculated."
     },
     {
       "flag": "Requires white-box model access",
       "detail": "The radioactivity detection requires full model weights for the 'reading mode'; the method does not apply to closed-source APIs, which is where contamination is most suspected and hardest to audit."
     },
     {
       "flag": "No variance across runs",
       "detail": "All results reflect single training runs; there is no statistical characterization of run-to-run variability in either accuracy or detection p-values."
     },
     {
       "flag": "Meta FAIR affiliation and undisclosed funding",
       "detail": "All primary authors are from Meta FAIR, a major LLM developer with commercial interest in contamination detection, and no funding disclosure or competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Watermarking makes language models radioactive",
       "relevance": "Direct predecessor: the paper extends this fine-tuning radioactivity result to the pre-training / benchmark contamination setting."
     },
     {
       "title": "A watermark for large language models",
       "relevance": "The green/red list watermarking scheme (Kirchenbauer et al. 2023a/b) is the core technical primitive used throughout the paper."
     },
     {
       "title": "Do membership inference attacks work on large language models?",
       "relevance": "Establishes that membership inference is ineffective for LLMs, motivating the need for the watermarking-based alternative approach."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "relevance": "Provides empirical evidence that models exhibit performance drops on fresh questions vs. contaminated benchmarks, motivating the contamination problem."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "relevance": "MMLU is one of the three benchmarks watermarked and tested in the experiments."
     },
     {
       "title": "Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?",
       "relevance": "Recent work surveying contamination measurement approaches, providing context for the problem the paper addresses."
     },
     {
       "title": "Investigating data contamination for pre-training language models",
       "relevance": "Shows that even small contamination can improve benchmark performance in pre-training, directly motivating the detection method."
     },
     {
       "title": "WARD: Provable RAG dataset inference via LLM watermarks",
       "relevance": "Related work applying watermark radioactivity to a different setting (RAG dataset inference), showing the general applicability of the radioactivity concept."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark creators could adopt this watermarking technique before releasing new benchmarks, though no code is provided for immediate use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the known problem of benchmark contamination and proposes a solution; does not challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Highlights the unreliability of current benchmark evaluations due to contamination, a mild integrity concern rather than a safety threat."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Feeds into the 'benchmarks are unreliable' narrative and implicitly suggests models may be scoring higher than they should due to contamination."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released; the method cannot be tried without reimplementation."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Meta FAIR is a well-known AI research lab, though the paper is not about a flagship product."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43258481",
         "title": "HumT DumT: Measuring and controlling human-like language in LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43258481"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 0
   }
 }