ai-research-survey

scan.json (29511B)

---

 {
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "paper": {
     "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
     "authors": [
       "Chunyuan Deng",
       "Yilun Zhao",
       "Xiangru Tang",
       "Mark Gerstein",
       "Arman Cohan"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2311.09783"
   },
   "methodology_tags": [
     "benchmark-eval"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code found anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (MMLU, TruthfulQA, HellaSwag, WinoGrande, GSM8K, OpenbookQA, PIQA) and publicly available pretraining corpora (The Pile, C4). All datasets are publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions Pyserini but does not give version details or dependency lists."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method is described at a high level but without sufficient detail to reproduce exact results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 are point estimates with no confidence intervals or error bars reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., GPT-4 vs ChatGPT, stronger vs weaker models) but uses no statistical significance tests. Differences are compared by raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports exact match rates with context: e.g., 'ChatGPT and GPT-4 demonstrated an exact match rate of 52% and 57%, respectively' (§4.2.2), with baselines like LLaMa 2-13B at 0% for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The human evaluation uses 100 data points from 7 benchmarks and 17 annotators, but no power analysis or justification for these numbers is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models serve as implicit baselines against each other (ChatGPT, GPT-4, Claude-2, Claude-instant-1, LLaMa 2-13B, Mistral-7B). Open-source models with transparent training data serve as baselines for proprietary models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models (GPT-4, ChatGPT, Claude-2, Mistral-7B, LLaMa 2-13B) were state-of-the-art at the time of writing (2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates query types (question-only, label-only, question+label) in Table 4, tests different hint types (type-hint, category-hint, url-hint) in Table 2, and varies top-k retrieval settings (1, 5, 10) in Table 1."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: BM25, SacreBLEU, Rouge-L, BLEURT, GPTscore for retrieval (Table 1); Exact Match and Rouge-L F1 for TS-Guessing (Table 3); Spearman correlation for metric validation (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "17 NLP volunteers evaluated 100 data points on a binary scale for contamination. Inter-annotator agreement was measured via Krippendorff's alpha (0.8673). Described in §4.1.1."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets (MMLU test, TruthfulQA, etc.). For benchmarks without public test labels (HellaSwag, WinoGrande, PIQA), development sets are used, which is explicitly stated (§4.2.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (7 benchmarks in Table 1, 6 in Table 3), per model, per query type, and per hint type. Individual benchmark contamination levels are discussed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) discusses failure modes: BM25 limitations, open-source models failing to follow TS-Guessing instructions, and models predicting correct answers instead of masked wrong answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that PIQA does not show high contamination despite word overlaps (§4.1.2), that open-source models (LLaMa 2, Mistral) show near-zero EM in multichoice guessing on several benchmarks (Table 3), and that TruthfulQA has negative correlation between TS-Guessing and task accuracy (Table 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ChatGPT/GPT-4 achieve 52%/57% EM on MMLU, which matches Table 3. Claims about the TS-Guessing method and retrieval system are supported by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies that high TS-Guessing scores are caused by training data contamination, but acknowledges this is only 'suspicion' — it cannot rule out that models are reasoning to the answer. The contamination probing experiment (§4.3) partially addresses this by showing fully contaminated ChatGPT reaches ~100% EM, but the causal link from high EM to contamination in the 'clean' model remains unproven."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Modern Benchmarks' broadly but tests only 7 QA-style benchmarks. The TS-Guessing method is tested on multiple-choice formats only. The paper does not bound its claims to these specific benchmark types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses tradeoffs and limitations of both methods. The paper acknowledges that TS-Guessing success could partly reflect reasoning rather than memorization, and that PIQA's overlapping words don't indicate contamination due to its physical reasoning nature (§4.1.2). Appendix D discusses the correlation between TS-Guessing and actual task performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exact match rates for slot guessing and retrieval overlap rates, and frames findings at that granularity: 'ChatGPT and GPT-4 demonstrated an exact match rate of 52% and 57%.' The paper explicitly distinguishes between detecting contamination signals (what is measured) and proving benchmark inflation (the broader concern), noting limitations of each method."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'ChatGPT (GPT-3.5-turbo)', 'GPT-4', 'Claude-instant-1-100k', 'Claude-2', 'LLaMa 2-13B', 'Mistral-7B'. No snapshot dates or API versions are specified for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are shown in Figure 2 for both Question-based and Question-Multichoice settings, including the actual text used with masking instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling parameters are reported for any of the LLM API calls. The Rouge-L threshold of 0.65 for filtering is mentioned, but inference hyperparameters are absent."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper makes direct API calls to models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pre-filtering steps are well-documented in §4.2.1: removal of short questions (≤4 words), removal of 'Indexical Error' category from TruthfulQA, removal of Yes/No and True/False options, Rouge-L threshold filtering at 0.65. Keyword searching process using Stanford POS Tagger and ChatGPT ICL is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is a dedicated section discussing multiple specific limitations of both the retrieval and TS-Guessing methods."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: BM25 may miss contamination, computation time makes the system impractical without high-performance hardware, text generation scores are superficial for true contamination detection, open-source models tend to predict correct answers instead of following TS-Guessing instructions, and models may be overfitting to multi-choice format (§7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. It doesn't bound claims to specific benchmark types (only QA-style) or acknowledge that TS-Guessing cannot work for non-multiple-choice benchmarks. The title implies broad coverage without explicit scoping."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (retrieval results, per-example TS-Guessing outputs, human annotations) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is clearly described: 7 public benchmarks are named with citations, 2 pretraining corpora (The Pile, C4) are specified, 100 data points randomly sampled for human evaluation from 7 benchmarks."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "17 volunteers with NLP backgrounds participated in human evaluation (§4.1.1), but no details on how they were recruited, their experience level, or potential selection bias are provided."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: indexing corpora with Pyserini → BM25 retrieval with top-k → 13-gram tokenization → overlap scoring. For TS-Guessing: filtering → keyword extraction → masking → model querying → EM/Rouge-L scoring."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgement section thanks reviewers and colleagues but mentions no grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Georgia Institute of Technology, Yale University, Allen Institute for AI. None of the affiliations are companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. One author is from Allen Institute for AI, but no funding statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'According to OpenAI, their training data is current up to September 2021' (§4.2.2). For open-source models, the training corpora (The Pile, C4) are identified."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is the central focus of the paper. The entire retrieval-based method (§3.1) directly measures train-test overlap, and TS-Guessing (§3.2) probes for memorization of test data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses that MMLU, TruthfulQA, and other benchmarks were available before model training cutoffs and could be in training data. This is the paper's core contribution."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation component (17 annotators, 100 data points) is not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics approval is mentioned despite using human annotators. The Ethics Statement (§8) discusses data ethics but not human subjects approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Annotators are described only as '17 volunteers with backgrounds in NLP.' No further demographics (experience level, education, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria for annotator selection are stated beyond 'backgrounds in NLP.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a randomized experiment comparing conditions for human participants; annotators all performed the same evaluation task."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable — annotators evaluated contamination overlap, not a treatment condition."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether all 17 annotators completed the task or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or total token consumption is reported despite making extensive API calls to GPT-4, ChatGPT, and Claude models. Only mentions '2-3 minutes per data point' for retrieval in §7."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions needing approximately 4TB (reduced to 2TB) disk space for corpora and notes the system requires a 'high-performance computer' (§7), but no GPU hours, total API spend, or hardware specs are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple random seeds or sensitivity analysis is reported. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for any experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The Rouge-L threshold of 0.65 is described as 'chosen based on initial experiments' (§4.2.1 footnote) but no search budget or alternatives tried are reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The filtering threshold of 0.65 and choice of top-k values are not justified beyond brief mentions. No validation set-based selection is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own TS-Guessing method without acknowledging potential bias in evaluating their own system."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the methods don't involve training or compute-intensive baselines where compute budget differences would affect comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly questions whether high TS-Guessing scores truly indicate contamination versus reasoning ability. Section 5 discusses tradeoffs and §4.2.2 notes that the method 'may not heavily rely on advanced reasoning skills, although its performance may vary depending on the training data available.' The contamination probing experiment (§4.3) validates the construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper probes models directly via prompting without any scaffolding. No multi-model comparisons are made through different scaffolds. The study evaluates contamination signals, not model capability through a scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses temporal leakage: TruthfulQA's camera-ready was May 2022 but source data from Wikipedia predates training cutoffs (§4.2.2). This temporal analysis is central to the paper."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses how TruthfulQA metadata (type, category, URL) acts as hints that could leak information. The hint-augmented experiments (Table 2) explicitly test whether metadata provides unfair signal."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The retrieval-based approach (§3.1) directly measures overlap between training corpora (Pile, C4) and benchmark instances, testing whether benchmark items appear in training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Two concrete detection methods are applied: (1) BM25 retrieval with 13-gram overlap scoring on The Pile and C4, and (2) TS-Guessing protocol that probes for memorization via slot-filling. Both are applied and results reported."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT and GPT-4 can guess missing incorrect options in MMLU with 52% and 57% exact match rates respectively.",
       "evidence": "Table 3 shows EM rates of 0.52 (ChatGPT) and 0.57 (GPT-4) for MMLU Question-Multichoice guessing, after filtering correlated options.",
       "supported": "strong"
     },
     {
       "claim": "Stronger models do not necessarily show higher proficiency in TS-Guessing, suggesting the protocol measures memorization rather than reasoning.",
       "evidence": "Table 2 shows only 1% difference between ChatGPT and GPT-4 in Question-based guessing without hints. Claude-instant-1 sometimes outperforms Claude-2 (Table 2).",
       "supported": "moderate"
     },
     {
       "claim": "After fully contaminating ChatGPT with MMLU test data via fine-tuning, TS-Guessing EM rate reaches nearly 100%, validating the method's sensitivity.",
       "evidence": "Figure 4 shows ~100% EM for both question-based and question-multichoice formats after fine-tuning on MMLU test set (§4.3).",
       "supported": "strong"
     },
     {
       "claim": "TruthfulQA has significant overlap with pretraining corpora due to its web-sourced content.",
       "evidence": "Table 1 shows high BM25 and GPTscore for TruthfulQA-C4 overlap. Appendix C provides a specific contamination example from C4. Human annotators identified 23/100 examples as contaminated (§4.1.2).",
       "supported": "strong"
     },
     {
       "claim": "GPTscore aligns more closely with human evaluation than traditional metrics for contamination detection.",
       "evidence": "Figure 3 shows Spearman correlation between metrics and human scores across 100 examples. GPTscore achieves the highest correlation.",
       "supported": "moderate"
     }
   ],
   "key_findings": "The paper proposes two methods for detecting benchmark data contamination in LLMs: retrieval-based search in pretraining corpora and a novel TS-Guessing protocol that masks test set elements and asks models to fill them in. ChatGPT achieves 52% exact match guessing missing incorrect MMLU options, suggesting potential contamination. TruthfulQA shows significant overlap with the C4 corpus. A validation experiment fine-tuning ChatGPT on MMLU reaches ~100% EM, confirming the method's sensitivity to contamination.",
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "Despite proposing a contamination detection methodology, no code, scripts, or raw experimental outputs are released for others to verify or apply the methods."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates with no error bars, confidence intervals, or variance across runs. The TS-Guessing results could be sensitive to prompt variations or API non-determinism."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "No API snapshot dates or model versions specified. ChatGPT and GPT-4 behavior changes across versions, making results non-reproducible."
     },
     {
       "flag": "Causal inference gap",
       "detail": "High TS-Guessing scores are interpreted as evidence of contamination, but the paper cannot rule out that models are reasoning about likely wrong answers from the structure of multiple-choice questions."
     }
   ],
   "cited_papers": [
     {
       "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",
       "authors": [
         "Shahriar Golchin",
         "Mihai Surdeanu"
       ],
       "year": 2023,
       "relevance": "Proposes alternative contamination detection method using prompt-based probing of LLMs."
     },
     {
       "title": "Proving Test Set Contamination in Black Box Language Models",
       "authors": [
         "Yonatan Oren",
         "Nicole Meister",
         "Niladri Chatterji",
         "Faisal Ladhak",
         "Tatsunori B. Hashimoto"
       ],
       "year": 2023,
       "relevance": "Presents probing method for contamination detection using canonical data order, applicable to black-box models."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": [
         "Oscar Sainz",
         "Jon Campos",
         "Iker García-Ferrero",
         "Julen Etxaniz",
         "Oier Lopez de Lacalle",
         "Eneko Agirre"
       ],
       "year": 2023,
       "relevance": "Argues for systematic contamination measurement across NLP benchmarks."
     },
     {
       "title": "Detecting Pretraining Data from Large Language Models",
       "authors": [
         "Weijia Shi",
         "Anirudh Ajith",
         "Mengzhou Xia",
         "Yangsibo Huang",
         "Daogao Liu",
         "Terra Blevins",
         "Danqi Chen",
         "Luke Zettlemoyer"
       ],
       "year": 2023,
       "relevance": "Proposes membership inference methods for detecting whether specific data was in LLM pretraining."
     },
     {
       "title": "Stop Uploading Test Data in Plain Text: Practical Strategies for Mitigating Data Contamination by Evaluation Benchmarks",
       "authors": [
         "Alon Jacovi",
         "Avi Caciularu",
         "Omer Goldman",
         "Yoav Goldberg"
       ],
       "year": 2023,
       "relevance": "Proposes practical mitigation strategies against benchmark contamination."
     },
     {
       "title": "DyVal: Graph-informed Dynamic Evaluation of Large Language Models",
       "authors": [
         "Kaijie Zhu",
         "Jiaao Chen",
         "Jindong Wang",
         "Neil Zhenqiang Gong",
         "Diyi Yang",
         "Xing Xie"
       ],
       "year": 2023,
       "relevance": "Proposes dynamic evaluation as a solution to benchmark contamination."
     },
     {
       "title": "Don't Make Your LLM an Evaluation Benchmark Cheater",
       "authors": [
         "Kun Zhou",
         "Yutao Zhu",
         "Zhipeng Chen",
         "Wentong Chen",
         "Wayne Xin Zhao",
         "Xu Chen",
         "Yankai Lin",
         "Ji-Rong Wen",
         "Jiawei Han"
       ],
       "year": 2023,
       "relevance": "Investigates how LLMs can game evaluation benchmarks through contamination."
     },
     {
       "title": "Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models",
       "authors": [
         "Martin Riddell",
         "Ansong Ni",
         "Arman Cohan"
       ],
       "year": 2024,
       "relevance": "Extends contamination investigation to code generation benchmarks at both surface and semantic levels."
     },
     {
       "title": "Data contamination: From memorization to exploitation",
       "authors": [
         "Inbal Magar",
         "Roy Schwartz"
       ],
       "year": 2022,
       "relevance": "Foundational work investigating the relationship between pretraining memorization and downstream task exploitation."
     },
     {
       "title": "What's In My Big Data?",
       "authors": [
         "Yanai Elazar",
         "Akshita Bhagia",
         "Ian Magnusson"
       ],
       "year": 2023,
       "relevance": "Provides analysis of contamination in open pretraining corpora including C4 and The Pile."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": [
         "Dan Hendrycks",
         "Collin Burns",
         "Steven Basart",
         "Andy Zou",
         "Mantas Mazeika",
         "Dawn Song",
         "Jacob Steinhardt"
       ],
       "year": 2021,
       "relevance": "MMLU benchmark — the primary subject of contamination investigation in this paper."
     },
     {
       "title": "Pretraining on the Test Set Is All You Need",
       "authors": [
         "Rylan Schaeffer"
       ],
       "year": 2023,
       "relevance": "Demonstrates that pretraining contamination can artificially inflate benchmark performance."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The TS-Guessing method could help benchmark designers check for contamination, but requires significant adaptation and no code is released."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that ChatGPT can guess 52% of missing wrong MMLU options is genuinely surprising and suggests widely-cited benchmark scores may be inflated."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about trustworthiness of LLM evaluations but doesn't demonstrate direct safety risks or attacks."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly challenges the validity of MMLU scores for ChatGPT and GPT-4, implying OpenAI's flagship benchmarks may be contaminated."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, scripts, or reproducible artifacts are released despite proposing a detection methodology."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Directly investigates ChatGPT and GPT-4 on MMLU, one of the most widely discussed benchmarks in the LLM space."
     }
   }
 }