ai-research-survey

scan.json (31285B)

---

 {
   "paper": {
     "title": "MMLU-CF: A Contamination-free Multi-task Language Understanding Benchmark",
     "authors": [
       "Qihao Zhao",
       "Yangyu Huang",
       "Tengchao Lv",
       "Lei Cui",
       "Furu Wei",
       "Qinzheng Sun",
       "Ying Xin",
       "Shaoguang Mao",
       "Xin Zhang",
       "Qiufeng Yin",
       "Scarlett Li"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2412.15194",
     "doi": "10.48550/arXiv.2412.15194"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided in the paper. They mention a 'project homepage' for evaluation submission but no actual code link."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The validation set is described as 'open-source' but the paper uses future tense ('we will publicly release the validation set') and provides no download URL. The test set is explicitly closed-source."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency information is provided. They mention using OpenCompass for evaluation but provide no setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the dataset construction pipeline conceptually but gives no runnable instructions for reproducing the evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., '73.4%') with no confidence intervals or error bars. No uncertainty quantification is provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparative claims are made (e.g., 'GPT-4o emerges as the strongest model', 'Qwen2.5-72B-instruct outperforms its peers') based solely on comparing point estimates. No statistical significance tests are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided in context: MMLU vs MMLU-CF scores are reported side-by-side (e.g., GPT-4o drops from 88.0% to 73.4%), and the Δ column shows absolute differences between test and validation sets. The ablation in Table 2 also shows incremental performance changes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 10,000 test and 10,000 validation questions, but no justification is given for why this specific size was chosen. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or repeated-run statistics are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "MMLU scores are provided as a baseline comparison for all models in Table 1, enabling direct assessment of the contamination-free benchmark's impact."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes contemporary models as of late 2024: GPT-4o, Gemini-1.5-Flash, Qwen2.5, Llama-3.3, Phi-4. These represent the state of the art at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents an ablation study of the three decontamination rules (rephrase, shuffle choices, random replace) applied incrementally, showing each rule's contribution to performance change on GPT-4o, GPT-3.5-Turbo, and Llama-3.1-8b."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The sole evaluation metric is accuracy (%). While both 5-shot and 0-shot settings are used, these are different evaluation configurations of the same metric, not different metrics (e.g., no F1, calibration, per-question analysis)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is conducted. LLMs (GPT-4o, Gemini, Claude) are used to check question quality in the construction pipeline, but no humans evaluate the evaluation results."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark explicitly separates test (10,000 questions, closed-source) and validation (10,000 questions, open-source) sets. Section 4.5 discusses the properties of this partitioning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-discipline breakdowns across 14 fields (Math, Physics, Chemistry, Law, etc.) for six major models on the 5-shot test set."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that 'models perform worst in Computer Science' (Appendix A.1) but provides no detailed error analysis, qualitative failure examples, or discussion of where the benchmark approach breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every decontamination rule is presented as effective (Table 2 shows monotonic performance decreases). No unsuccessful approaches, failed rules, or abandoned methods are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-4o achieves 73.4% (5-shot) and 71.9% (0-shot), which Table 1 confirms. The claim that these scores indicate contamination-free evaluation is supported by the decontamination pipeline and ablation study."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Table 2 uses controlled incremental addition of decontamination rules (adding Rule 1, then Rule 2, then Rule 3), which is an adequate design for the causal claim that these rules reduce contamination-inflated scores."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'Contamination-free' as an absolute property, but the decontamination rules can only reduce contamination, not guarantee its elimination. Rephrasing questions doesn't prevent a model from recognizing the underlying concept. The claim overstates what the methodology can actually prove."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The performance drops from MMLU to MMLU-CF are attributed to contamination, but alternative explanations are not discussed. The questions may be inherently harder because they come from different, broader sources. Rule 3 (replacing choices with 'None of the other choices') increases difficulty independent of contamination. These confounds are not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames MCQ accuracy as measuring 'understanding' and 'problem-solving abilities' without discussing the gap between MCQ performance and actual language understanding. No discussion of whether MCQ accuracy is a valid proxy for the claimed construct."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "API model versions are specified with snapshot dates: GPT-4o (v2024-10-1), GPT-4o-mini (v2024-10-1), GPT-4-Turbo (v2024-2-15), GPT-3.5-Turbo (v2024-2-15). Open-source models use standard versioned names (e.g., Qwen2.5-72B-instruct, Llama-3.3-70B-instruct)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the MCQ evaluation prompt ('There is a single choice question. Answer the question by replying A, B, C or D.'). Table 4 provides the full LLMs checking prompt. The difficulty rating prompt is quoted in Section 3.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the evaluation runs or the LLM checking pipeline. These settings significantly affect model output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly via standard 5-shot and 0-shot prompting through OpenCompass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Figure 3 document the full 5-step pipeline with counts at each stage: 200B+ documents → 2.7M MCQs → 1.66M after cleaning → 50K after difficulty sampling → 20K after LLM checking → 10K test + 10K validation. Filtering criteria are described at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' provides a dedicated limitations discussion covering potential remaining errors and the scope limitation to MCQs and language modalities."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 offers mostly generic limitations: 'some errors may remain' and 'primarily focuses on multiple-choice questions and language modalities.' No specific threats like 'our use of GPT-4o for difficulty rating may introduce systematic bias' or 'web-sourced questions may contain errors despite LLM checking.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'this dataset primarily focuses on multiple-choice questions and language modalities. However, other aspects of large models' capabilities, such as math and code reasoning, multi-modal understanding (e.g., image and audio), and specific domain expertise, still require evaluation with similarly unbiased and contamination-free benchmarks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The test set is explicitly closed-source. The validation set is described as forthcoming ('we will publicly release') but no download URL is provided. Raw source data (the 2.7M extracted questions) is not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes collection from 200+ billion documents across 3,000+ public website domains, using rule-based extraction to obtain 2.7 million MCQs spanning 14 fields."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Data was sourced from public open-source websites using rule-based MCQ extraction across 3,000+ domains. The sourcing method and domain diversity are described in Section 3.2, Step 1."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Section 3.2 document the full pipeline: 200B+ documents → 2.7M (rule-based extraction) → 1.66M (cleaning) → 50K (difficulty sampling) → 20K (LLM quality checking) → 10K test + 10K validation (contamination-free processing). Counts and criteria at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Microsoft Research. This is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors work at Microsoft Research. Microsoft has a major investment in OpenAI, whose GPT-4o model emerges as the top performer on MMLU-CF. This financial relationship creates a non-independent interest in outcomes favorable to GPT-4o."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. No disclosure of patents, equity, or financial interests related to the evaluated models or the benchmark itself."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Despite the paper being fundamentally about contamination, no training data cutoff dates are stated for any of the 40+ evaluated models. This information would be crucial for assessing whether MMLU-CF questions predate model training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about preventing train/test overlap. Three decontamination rules (rephrasing, shuffling, random replacement) are specifically designed to break any memorized associations. Section 4.5 also introduces the Δ metric to monitor for future contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the central contribution. The paper employs three decontamination rules, sources from diverse domains (3000+), keeps the test set closed-source, and demonstrates through ablation (Table 2) and Figure 1 that MMLU suffers from contamination while MMLU-CF mitigates it."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLMs on a benchmark dataset."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported for any of the 40+ model evaluations despite using both open-source and commercial API models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated for the evaluation runs, the LLM-based quality checking pipeline, or the difficulty sampling process, all of which required substantial API calls."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single evaluation runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is never stated. It is unclear whether results are from single runs or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The difficulty threshold (level 6 center) and quality score cutoff (>4) appear tuned but no search process is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses standard configurations (5-shot and 0-shot) that are conventional in LLM benchmarking, with both settings reported transparently in Table 1."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Over 40 models are compared with no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft Research authors evaluate models including GPT-4o (Microsoft/OpenAI partnership) and Phi-4 (Microsoft). No discussion of potential self-evaluation bias despite clear organizational interest."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models are categorized by parameter count (Mini/Small/Medium/Large) but no analysis of compute budget vs. performance is provided. Evaluation compute costs are not reported."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether MCQ accuracy actually measures 'language understanding' or 'problem-solving abilities' as claimed. No analysis of construct validity for the benchmark's claims."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated through the OpenCompass platform (Section 4.1) with standardized 5-shot and 0-shot prompting, ensuring consistent evaluation scaffolding across all models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution addresses temporal leakage: questions are rephrased (Rule 1) to break memorization of previously seen questions, and the test set is kept closed-source to prevent future temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of feature leakage. While the MCQ format is standard, the paper does not discuss whether the evaluation setup could leak answer information through context or formatting cues."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The three decontamination rules (rephrase, shuffle, replace) are specifically designed to break dependencies between MMLU-CF questions and any similar questions in model training data. Section 3.2 also uses GPT-4o for redundancy detection on semantically identical questions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Multiple concrete methods are used: (1) Three decontamination rules as prevention, (2) GPT-4o redundancy detection for semantic deduplication, (3) The prompt-based memorization test in Figure 1 showing models can recall MMLU choices but not MMLU-CF choices, (4) Δ metric between test/validation as ongoing contamination monitoring."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "GPT-4o achieves only 73.4% (5-shot) and 71.9% (0-shot) on MMLU-CF, significantly lower than 88.0% on MMLU.",
       "evidence": "Table 1 shows GPT-4o scores: MMLU 88.0%, MMLU-CF 5-shot test 73.4%, 0-shot test 71.9%. A 14.6 percentage point drop.",
       "supported": "strong"
     },
     {
       "claim": "Three decontamination rules cumulatively reduce model performance, demonstrating contamination in standard benchmarks.",
       "evidence": "Table 2 ablation: GPT-4o drops from 79.8% (no rules) to 73.4% (all rules). GPT-3.5-Turbo drops from 65.3% to 58.2%. Llama-3.1-8b drops from 63.8% to 57.1%. Each rule added incrementally reduces scores.",
       "supported": "moderate"
     },
     {
       "claim": "Test and validation sets have similar difficulty, with most Δ values below 1.0%.",
       "evidence": "Table 1 shows about 60% of Δ values less than 0.5% and 96% below 1.0% across all models in both 5-shot and 0-shot settings.",
       "supported": "strong"
     },
     {
       "claim": "MMLU-CF sources from a broader domain (3,000+ website domains) than previous benchmarks to reduce contamination risk.",
       "evidence": "Section 3.2 states questions come from 'over 3,000 different website domains' extracted from '200+ billion documents,' compared to MMLU which 'relied on a few sources.'",
       "supported": "moderate"
     },
     {
       "claim": "Smaller models are more sensitive to decontamination rules than larger models.",
       "evidence": "Table 2 shows larger relative drops for GPT-3.5-Turbo (-7.1pp) and Llama-3.1-8b (-6.7pp) compared to GPT-4o (-6.4pp) when all three rules are applied. Section 4.6 interprets this as smaller models being 'more sensitive.'",
       "supported": "weak"
     },
     {
       "claim": "Some LLMs have memorized MMLU questions and can reproduce the exact choices when given only the question text.",
       "evidence": "Figure 1 demonstrates that when given only MMLU questions as prompts, certain LLMs produce choices identical to the MMLU test set, while they produce guessed (different) choices for MMLU-CF questions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MMLU-CF is a contamination-free MCQ benchmark of 20,000 questions (10K closed test + 10K open validation) sourced from 3,000+ web domains with three decontamination rules. GPT-4o scores 73.4% (5-shot), a 14.6pp drop from 88.0% on MMLU, suggesting substantial contamination in standard benchmarks. An ablation study shows each decontamination rule (rephrasing, shuffling choices, random replacement) contributes to performance reduction, with cumulative effects strongest on smaller models. Test and validation set scores are highly consistent (96% of models within 1.0pp), enabling contamination monitoring.",
   "red_flags": [
     {
       "flag": "Conflict of interest: Microsoft Research evaluates Microsoft-partnered models",
       "detail": "All authors are from Microsoft Research. GPT-4o (OpenAI/Microsoft partnership) and Phi-4 (Microsoft) are among the evaluated models, with GPT-4o emerging as the top performer. No conflict of interest statement is provided."
     },
     {
       "flag": "Confound between contamination and difficulty in Rule 3",
       "detail": "Rule 3 replaces a random choice with 'None of the other choices' at 50% probability, which increases question difficulty independent of contamination. The ablation (Table 2) shows the largest performance drop when Rule 3 is added, but this cannot be cleanly attributed to decontamination versus added difficulty."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Over 40 models are compared and ranked based solely on point estimates from apparently single runs. No confidence intervals, significance tests, or variance measures are reported, making it impossible to assess whether observed differences are meaningful."
     },
     {
       "flag": "Overclaimed property: 'Contamination-free'",
       "detail": "The title claims 'Contamination-free' but the methods can only reduce contamination. Rephrasing questions does not prevent a model from recognizing underlying concepts. A model trained on medical knowledge will still answer medical MCQs correctly regardless of rephrasing — this is knowledge, not memorization."
     },
     {
       "flag": "Using LLMs to validate a benchmark for LLMs",
       "detail": "GPT-4o is used for difficulty rating, quality checking, and redundancy detection in the construction pipeline. The same model (and model family) is then evaluated on the resulting benchmark. Any systematic biases in GPT-4o's quality judgments would be baked into the benchmark."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt"],
       "relevance": "The original MMLU benchmark that MMLU-CF is designed to replace; foundational benchmark for LLM evaluation."
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-task Language Understanding Benchmark",
       "authors": ["Yubo Wang", "Xueguang Ma", "Ge Zhang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Enhanced MMLU variant with reasoning-focused questions and expanded choices; directly related benchmark in the MMLU family."
     },
     {
       "title": "A Careful Examination of Large Language Model Performance on Grade School Arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da", "Dean Lee"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "GSM1K contamination-free math benchmark that revealed memorization in LLMs; shares MMLU-CF's motivation."
     },
     {
       "title": "LiveBench: A Challenging, Contamination-Free LLM Benchmark",
       "authors": ["Colin White", "Samuel Dooley", "Manley Roberts"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Contamination-free benchmark using frequently updated questions; alternative approach to the same contamination problem."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free coding benchmark with continuous collection; relevant to AI code generation evaluation."
     },
     {
       "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
       "authors": ["Chunyuan Deng", "Yilun Zhao", "Xiangru Tang", "Mark Gerstein", "Arman Cohan"],
       "year": 2024,
       "relevance": "Systematic study of data contamination in LLM benchmarks; foundational work on the contamination problem MMLU-CF addresses."
     },
     {
       "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
       "authors": ["Shuo Yang", "Wei-Lin Chiang", "Lianmin Zheng", "Joseph E. Gonzalez", "Ion Stoica"],
       "year": 2023,
       "arxiv_id": "2311.04850",
       "relevance": "Decontaminator approach using rephrased samples; directly inspired MMLU-CF's decontamination rules."
     },
     {
       "title": "Quantifying Memorization Across Neural Language Models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang"],
       "year": 2023,
       "relevance": "Foundational study of LLM memorization; provides theoretical basis for MMLU-CF's approach to preventing memorized answers."
     },
     {
       "title": "Detecting Pretraining Data from Large Language Models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia"],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "WIKIMIA dynamic benchmark for detecting pretraining data contamination; related approach to the contamination detection problem."
     },
     {
       "title": "Inference-time Decontamination: Reusing Leaked Benchmarks for Large Language Model Evaluation",
       "authors": ["Qin Zhu", "Qingyuan Cheng", "Runyu Peng"],
       "year": 2024,
       "arxiv_id": "2406.13990",
       "relevance": "Proposes inference-time approaches to address benchmark contamination; complementary technique to MMLU-CF's construction-time decontamination."
     },
     {
       "title": "Changing Answer Order Can Decrease MMLU Accuracy",
       "authors": ["Vipul Gupta", "David Pantoja", "Candace Ross", "Adina Williams", "Megan Ung"],
       "year": 2024,
       "arxiv_id": "2406.19470",
       "relevance": "Demonstrates that choice order affects LLM accuracy on MMLU; directly motivates MMLU-CF's Rule 2 (shuffle choices)."
     },
     {
       "title": "Are We Done with MMLU?",
       "authors": ["Aryo Pradipta Gema", "Joshua Ong Jun Leang"],
       "year": 2024,
       "arxiv_id": "2406.04127",
       "relevance": "Critical examination of MMLU's continued validity as a benchmark; provides context for MMLU-CF's creation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners evaluating LLMs could use MMLU-CF for more reliable model comparisons, though the test set is closed-source and the validation set was not yet released."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Shows GPT-4o drops 14.6pp from MMLU to MMLU-CF, directly challenging the reliability of widely-cited MMLU leaderboard rankings."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns that LLM evaluations may be unreliable due to contamination, but does not demonstrate direct safety or security risks."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implies that MMLU scores — widely used for model marketing — are inflated by contamination, with some models potentially having been maliciously trained on benchmark data."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The validation set was announced as forthcoming but not yet available; evaluation requires submission through a project homepage."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Microsoft Research, evaluates well-known models (GPT-4o, Gemini, Llama), and builds on the famous MMLU benchmark."
     }
   }
 }