ai-research-survey

scan.json (30941B)

---

 {
   "paper": {
     "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
     "authors": [
       "Shuo Yang",
       "Wei-Lin Chiang",
       "Lianmin Zheng",
       "Joseph E. Gonzalez",
       "Ion Stoica"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2311.04850",
     "doi": "10.48550/arXiv.2311.04850"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Simple rephrasing or translation of benchmark test cases bypasses standard decontamination methods (n-gram overlap, embedding similarity) while enabling a 13B model to match GPT-4 performance on MMLU, HumanEval, and GSM-8k. The proposed LLM-based decontaminator, which combines embedding similarity search with GPT-4 judgment, achieves significantly higher detection F1 scores. Applied to real-world pre-training and fine-tuning datasets (RedPajama, StarCoder-Data, CodeAlpaca, MATH, FLAN), the method reveals 0.5-18.9% previously unknown benchmark overlap, including unintentional contamination in GPT-generated synthetic datasets.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public GitHub repository: https://github.com/lm-sys/llm-decontaminator, stated in the abstract and conclusion."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (MMLU, HumanEval, GSM-8k) are publicly available, and the training datasets analyzed (RedPajama, CodeAlpaca, StarCoder-Data, The Stack, FLAN, etc.) are all public. The decontamination tool is also released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency listing is provided in the paper. Only model names are mentioned without environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The algorithms are described (Algorithms 1-2) but there are no commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-7 are point estimates with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares detection F1 scores and benchmark accuracies across methods without any statistical significance tests. Claims like 'the LLM decontaminator works significantly better' are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Baseline and post-contamination scores are reported together (e.g., Llama-2-13B from 54.8 to 89.9 on MMLU, CodeLlama-13B from 36.0 to 81.1 on HumanEval), providing sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the detection evaluation benchmark size (200 prompt pairs: 100 random + 100 rephrased). The choice of 3 MMLU subjects out of 57 for detection evaluation is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Detection methods are compared: n-gram overlap, embedding similarity search (with multiple embedding models), and the proposed LLM decontaminator, with random detection as baseline (Tables 5-6). For contamination effects, original model scores serve as baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models (Llama-2, CodeLlama, 2023) and GPT-4 as a reference point. Detection baselines use current embedding models (sentence-BERT variants). All baselines are from 2019-2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple rephrasing variants are tested: English vs. Chinese translation (Table 2), question-only vs. full prompt (Table 2), single-language vs. multi-language code translation (Table 3). These systematically vary components of the rephrasing approach."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy for MMLU and GSM-8k, Pass@1 for HumanEval, and F1 score for detection method evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All detection accuracy and benchmark scoring is automated. Human judgment could have validated whether the LLM decontaminator's decisions align with human assessment of contamination."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets (MMLU, HumanEval, GSM-8k) are used for evaluation. The training data consists of rephrased versions of the test data, which is the intended experimental design — the test sets themselves are held out from the exact training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detection F1 scores are broken down by MMLU subject (Abstract Algebra, Sociology, US History) in Table 5 and by programming language (Python, C, JavaScript) in Table 6. Table 7 breaks down contamination by dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Specific failure cases are shown: Figure 1 demonstrates an n-gram overlap failure on MMLU, Example 2 shows a false positive from n-gram overlap on multiple-choice questions, and Table 5 reveals embedding similarity search failures on translated samples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: multilingual BERT fails on US History (F1=0.111 in Table 5), single-language C translation underperforms multi-language for HumanEval (45.7 vs 59.8 in Table 3), and multi-QA BERT completely fails on translated samples (F1=0 for Chinese in US History)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'a 13B model can easily overfit a test benchmark and achieve drastically high performance, on par with GPT-4' is confirmed by Tables 2-4. '8-18% of HumanEval benchmark overlaps' is confirmed by Table 7. 'n-gram overlap insufficient' is confirmed by Tables 5-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The central causal claim — training on rephrased test data causes benchmark score inflation — is supported by controlled experiments where the only manipulation is the inclusion of rephrased data in fine-tuning. The before/after comparison with the same model isolates the effect of rephrased data."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Rethinking Benchmark and Contamination for Language Models' generalizes beyond the tested scope. Results are from two model families (Llama-2, CodeLlama) on three benchmarks, but the paper urges 'the community' to act broadly. No explicit statement of what settings were NOT tested or where findings might not hold."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its main findings. For instance, whether fine-tuning on rephrased test data could impart legitimate problem-solving skills rather than pure memorization is not addressed. Section 6.1 raises the number-substituted case as an open question but does not discuss alternative interpretations of the main results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's measurements (benchmark accuracy, F1 for detection) directly match its claims about contamination effects and detection accuracy. The paper explicitly argues that high benchmark scores from rephrased training do not reflect real capability, properly distinguishing the measured proxy from the intended outcome."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Llama-2-7b, Llama-2-13b, CodeLlama 7B, and CodeLlama 13B are named with sizes, but GPT-4 — used as both the rephrasing engine and the LLM decontaminator — has no version or snapshot date specified. GPT-4 behavior varies across versions, making this a significant omission."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides full actual prompt text for MMLU rephrase/translate instructions and HumanEval rephrase/translate instructions. The decontaminator tool code is released at the GitHub repository, which would contain the detection prompt template."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only partial hyperparameters are reported: 16 epochs for fine-tuning, 'non-zero initial temperature' for rephrasing (without stating the value), k=1 for some decontaminator runs, and embedding thresholds (0.5, 0.6, 0.9). Missing: GPT-4 temperature/sampling settings, learning rate, batch size, optimizer, max tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a two-step pipeline (embedding similarity search + LLM judgment), not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The rephrasing algorithm (Algorithm 1) and detection algorithm (Algorithm 2) are documented, but fine-tuning data preparation is underdocumented. For real-world dataset analysis, sampling procedures are vaguely described ('we sample 16G of data from the GitHub subset') without detailing the sampling method or filtering steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 6 ('Discussion') covers related topics including contamination definitions and future directions but does not systematically discuss limitations of the proposed method or experiments."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the LLM decontaminator's dependence on GPT-4 quality, sensitivity to the embedding similarity threshold k, or whether the detection accuracy generalizes to other domains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not state what models, benchmarks, or domains were NOT tested, or where the LLM decontaminator might fail. Section 6.3 acknowledges 'how to detect contamination without access to training data remains an open problem' but does not bound the current study's scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The rephrased test sets, fine-tuned model weights, and raw detection results are not released for independent verification. Only the decontamination tool is released. Original benchmarks and training datasets are public but the specific experimental artifacts are not."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: rephrased samples are generated using Algorithm 1 with GPT-4, detection evaluation uses 200 prompt pairs (100 random + 100 rephrased) per subject/benchmark, and real-world datasets are identified by name with subset sizes (e.g., RedPajama 16G GitHub subset)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks and publicly available training datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The rephrasing and detection pipelines are algorithmically described (Algorithms 1-2), but the fine-tuning pipeline lacks detail (only '16 epochs' stated). For real-world contamination analysis, the number of examples at each filtering stage is not reported — only final contamination counts appear in Table 7."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists gifts from Anyscale, Astronomer, Google, IBM, Intel, Lacework, Microsoft, MBZUAI, Samsung SDS, Uber, and VMware. Lianmin Zheng's Meta PhD Fellowship is also disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: UC Berkeley and Shanghai Jiao Tong University. The authors are not employed by the companies whose models or datasets are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funding appears to be general research support from multiple tech companies, not tied to specific outcomes. The paper evaluates open-source datasets and models, not products of the funders. No funder has a direct stake in the contamination detection results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for Llama-2 and CodeLlama are not stated in this paper, despite these models being fine-tuned and evaluated on benchmarks that predate their training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about train/test overlap. It systematically studies how rephrased test data in training sets affects benchmark scores and proposes detection methods."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the central topic. The paper demonstrates contamination effects on MMLU, HumanEval, and GSM-8k, proposes a stronger detection method, and reveals contamination in real-world datasets (Section 5.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 lists computational complexity classes (O(MN), O(N)) for detection methods but no actual costs (API dollars, wall-clock time, tokens consumed) are reported for any experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or training time is reported. The fine-tuning of Llama-2/CodeLlama models and the GPT-4 API calls for rephrasing and detection would involve significant compute, but this is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single runs without seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The fine-tuning configuration (16 epochs) appears chosen without justification, and no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how configurations were selected. Embedding similarity thresholds (0.5, 0.6, 0.9) and n-gram sizes (10-gram, 50-character) are used without justification for these specific values."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across detection methods, benchmarks, subjects, and languages without any correction for multiple testing. No formal statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baseline detection methods themselves and compare against their proposed LLM decontaminator. No acknowledgment of potential bias from implementing baselines or possible suboptimal baseline tuning."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 shows theoretical complexity classes but no empirical comparison of compute budget vs. detection performance. The LLM decontaminator uses expensive GPT-4 calls while baselines are much cheaper, but this cost-performance tradeoff is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper questions benchmark construct validity by demonstrating that high scores on MMLU, HumanEval, and GSM-8k can be achieved through contamination rather than genuine capability. Section 6.3 advocates for fresh one-time exams to address this validity problem."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used in these experiments. Models are directly fine-tuned and evaluated on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss the temporal relationship between benchmark creation dates (MMLU 2020, HumanEval 2021, GSM-8k 2021) and the training data cutoffs of Llama-2/CodeLlama, despite this being relevant to assessing pre-existing contamination in the base models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage is not discussed. For example, whether the rephrasing process preserves structural features that could serve as hints is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The independence between rephrased training samples and original test samples is not formally analyzed. By design, rephrased samples are semantically identical to test samples, but the paper does not discuss what degree of semantic overlap constitutes non-independence in the general case."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper proposes and applies concrete leakage detection methods: n-gram overlap, embedding similarity search, and the novel LLM decontaminator (Algorithm 2). These are applied to real-world datasets to detect contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing decontamination methods (n-gram overlap) fail to detect rephrased test samples that preserve the same semantics.",
       "evidence": "Tables 5-6 show F1 scores of 0 for 10-gram overlap on all rephrased (English and Chinese) MMLU samples and rephrased HumanEval samples across Python, C, and JavaScript. Figure 1 provides a concrete failure example.",
       "supported": "strong"
     },
     {
       "claim": "A 13B model fine-tuned on rephrased benchmark data achieves performance on par with GPT-4.",
       "evidence": "Table 2: Llama-2-13B reaches 85.9-89.9 on MMLU (GPT-4 ~86.4). Table 3: CodeLlama-13B reaches 81.1 on HumanEval (GPT-4 67.0). Table 4: Llama-2-13B reaches 95.3 on GSM-8k.",
       "supported": "strong"
     },
     {
       "claim": "The proposed LLM decontaminator significantly outperforms existing detection methods for finding rephrased samples.",
       "evidence": "Tables 5-6 show the LLM decontaminator achieves minimum F1 of 0.940 across all MMLU conditions and 0.974 across all HumanEval conditions, while n-gram overlap scores 0 on all rephrased samples and embedding similarity varies widely (0 to 0.990).",
       "supported": "strong"
     },
     {
       "claim": "8-18% of HumanEval benchmark overlaps with popular pre-training datasets.",
       "evidence": "Table 7 and Figure 3: The Stack (18.9%), StarCoder-Data (15.9%), CodeAlpaca (12.8%), RedPajama (8.5%). Specific examples shown in Examples 3-4 and Appendix B.",
       "supported": "moderate"
     },
     {
       "claim": "Synthetic data generated by LLMs contains unintentional contamination from benchmarks.",
       "evidence": "Section 5.3: CodeAlpaca (generated by GPT-3.5 Davinci-003) contains 21 rephrased HumanEval samples (12.8%). Example 3 shows a concrete instance. Phi-1 report is also cited as corroborating evidence.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-language data augmentation for code yields better contamination results than single-language translation.",
       "evidence": "Table 3: CodeLlama-13B achieves 67.1 on multi-language vs. 48.2 on C-only translation. CodeLlama-7B achieves 59.8 vs. 45.7.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All benchmark scores and detection F1 scores are single point estimates with no error bars, confidence intervals, or multi-run variance. The stability of the fine-tuning results (e.g., whether 85.9 MMLU is consistent across runs) is unknown."
     },
     {
       "flag": "GPT-4 used as both creator and judge",
       "detail": "GPT-4 is used to generate rephrased samples (Algorithm 1) and also serves as the LLM decontaminator (Algorithm 2). This circular dependency could inflate detection accuracy — the detector may recognize its own rephrasing style more easily than other forms of contamination."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated discussion of its own limitations. Key unaddressed issues include: sensitivity to the embedding threshold k, scalability to larger datasets, false positive rates of the LLM decontaminator in real-world settings, and whether findings generalize beyond the three benchmarks tested."
     },
     {
       "flag": "Incomplete training details",
       "detail": "Fine-tuning experiments report only '16 epochs' — no learning rate, optimizer, batch size, warmup schedule, or data formatting details. This makes reproduction of the contamination effect experiments difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, central to the contamination analysis in this paper and widely used for code generation evaluation."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "Introduces MMLU benchmark, one of the three main benchmarks used to demonstrate contamination via rephrased samples."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM-8k benchmark used in this paper's contamination experiments."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "relevance": "Primary model family used in contamination experiments; also cited for its own contamination analysis showing 10% of MMLU test samples highly contaminated."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Reports 25% HumanEval contamination in GPT-4 training data; uses 50-character overlap detection that this paper shows is insufficient."
     },
     {
       "title": "PaLM 2 technical report",
       "authors": ["Rohan Anil", "Andrew M. Dai", "Orhan Firat"],
       "year": 2023,
       "relevance": "Major LLM development that uses n-gram overlap for decontamination, which this paper shows is inadequate."
     },
     {
       "title": "Textbooks are all you need",
       "authors": ["Suriya Gunasekar", "Yi Zhang", "Jyoti Aneja"],
       "year": 2023,
       "relevance": "Phi-1 report that discovered significant contamination from LLM-generated synthetic data, corroborating this paper's findings about synthetic data contamination."
     },
     {
       "title": "Proving test set contamination in black box language models",
       "authors": ["Yonatan Oren", "Nicole Meister", "Niladri Chatterji"],
       "year": 2023,
       "relevance": "Proposes exchange detection for contamination in black-box models; alternative contamination detection method discussed in related work."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia"],
       "year": 2023,
       "relevance": "Proposes min-k prob detection for LLM contamination without training data access; complementary contamination detection approach."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "relevance": "Code pre-training dataset analyzed in this paper, found to have 15.9% HumanEval contamination."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski"],
       "year": 2023,
       "relevance": "Studies LLM memorization and the boundary between generalization and memorization, directly relevant to understanding contamination effects."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Yupeng Chang", "Xu Wang", "Jindong Wang"],
       "year": 2023,
       "arxiv_id": "2307.03109",
       "relevance": "Surveys LLM evaluation challenges including benchmark trustworthiness concerns that motivate this contamination study."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Released an immediately usable open-source decontamination tool at GitHub that practitioners can apply to their own datasets."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Quantifies a suspected but under-demonstrated problem: that simple rephrasing defeats standard decontamination, enabling a 13B model to match GPT-4."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about benchmark trustworthiness and potential for gaming but does not demonstrate novel attacks or existential risks."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The 'a 13B model matches GPT-4 via contamination' finding and revelation of contamination in popular datasets like RedPajama has a 'benchmarks are unreliable' angle."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Open-source GitHub tool available for download and use, though not a live web demo."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From UC Berkeley / LMSYS group (known for Chatbot Arena); uses well-known models (Llama-2, GPT-4) and datasets."
     }
   }
 }