ai-research-survey

scan.json (29613B)

---

 {
   "paper": {
     "title": "Probing Language Models for Pre-training Data Detection",
     "authors": [
       "Zhenhua Liu",
       "Tong Zhu",
       "Chuanyuan Tan",
       "Haonan Lu",
       "Bing Liu",
       "Wenliang Chen"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2406.01333",
     "doi": "10.48550/arXiv.2406.01333"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes using linear probes on LLM internal activations for pre-training data detection, achieving state-of-the-art AUC on WikiMIA (69.8) and ArxivMIA (60.3). The new ArxivMIA benchmark, comprising arxiv abstracts from CS and Math, proves substantially more challenging than WikiMIA across all detection methods. The probing approach benefits from larger models and is data-efficient (optimal at ~200 training examples), but requires domain-specific training data and additional compute for proxy model fine-tuning compared to baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in a footnote: 'Our code and dataset are available at https://github.com/zhliu0106/probing-lm-data' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The ArxivMIA benchmark dataset is released at the same GitHub repository. They also use the publicly available WikiMIA benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '2 NVIDIA A100 (40GB) GPUs' (Section 5.4) but provides no requirements.txt, Dockerfile, or detailed library version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository may contain them, but the paper itself does not include a 'Reproducing Results' section or detailed run commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, 4, 5, and 6 report only point estimates (AUC values, TPR) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'our method consistently outperforms all baselines' (Section 6.1) based solely on comparing numerical AUC/TPR values without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 show absolute AUC and TPR values for both baselines and the proposed method, allowing readers to assess the magnitude of improvement (e.g., 69.8 vs 65.5 on Pythia WikiMIA)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for benchmark sizes (776 for WikiMIA, 2000 for ArxivMIA) or training data sizes. The ablation on training data number (Figure 3) explores different sizes but doesn't justify the final choices."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All reported results appear to be single-run numbers. No standard deviation, variance, or spread measures are provided anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: Loss Attack, Neighbor Attack, Min-K% Prob, Zlib Compression, Lowercased Text, and Smaller Model (Section 5.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Min-K% Prob (Shi et al., 2023) and Neighbor Attack (Mattern et al., 2023), which are recent and represent the state of the art in pre-training data detection."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are conducted: model size (Figure 2), training data number (Figure 3), prompt template selection (Table 8), real vs synthetic data (Tables 2-3), full fine-tuning vs LoRA (Table 5), and cross-domain evaluation (Table 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: AUC (Table 2) and TPR at 5% FPR (Table 3), as stated in Section 5.2."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a binary classification task (member vs non-member) with known ground truth labels. Human evaluation is irrelevant — automated metrics fully capture detection performance."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "'We split each dataset into a validation set and a test set in a ratio of 1:4. The validation set is used to select the best hyperparameters, and the test set is used to evaluate the performance' (Section 5.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (WikiMIA, ArxivMIA), by model (Pythia, OPT, TinyLLaMA, OpenLLaMA), and by ArxivMIA subcategory (CS vs Math) in Tables 2 and 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses poor performance on ArxivMIA-Math ('values are only above 50'). Section 6.4 acknowledges 'the overall detection efficacy is unsatisfactory' at low duplication counts. Cross-domain performance degradation is discussed in Section 6.2."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance decline with >200 training samples (Figure 3), LoRA underperforming full fine-tuning (Table 5, 62.7 vs 74.3), cross-domain AUC drops (Table 6), and near-chance performance on contamination detection challenge (Table 4) are all reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'outperforms all the baselines, and achieves state-of-the-art performance on both WikiMIA and ArxivMIA' which is supported by Tables 2 and 3 showing the highest average AUC and TPR values."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (model size, training data number, prompt template, LoRA vs fine-tuning) use controlled single-variable manipulation to support causal claims about component contributions. The proxy model injection step is a controlled intervention."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Probing Language Models for Pre-training Data Detection' is broad, but experiments are limited to 4 models (up to 13B), 2 benchmarks, and English text only. The Limitations section notes domain-specific data requirements but the abstract and title do not bound the claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why the probing method works. For example, the probe might detect stylistic or distributional differences between member and non-member data rather than actual pre-training memorization signals. No confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures AUC and TPR@FPR for pre-training data detection, and claims are framed at the same granularity — detecting whether text was in pre-training data. No broader proxy gap exists between measurement and framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: Pythia-2.8B, OPT-6.7B, TinyLLaMA-1.1B, OpenLLaMA-13B, Pythia-70M, OPT-350M, OpenLLaMA-3B, Contam-1.4b, RoBERTa-base (Section 5.4). These are specific, versioned open-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The actual prompt template is provided verbatim: 'Here is a statement: [SAMPLE] \\n Is the above statement correct? Answer:' (Section 3.3). All alternative templates tested are also shown in Table 8 (Appendix B)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the best choice is to put all the data to be injected into one batch and train for 2 epochs' and mentions grid search, but does not report the actual learning rates, which layer activations were extracted from, or other hyperparameter values for each model-dataset combination."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a direct pipeline: fine-tune proxy model → extract activations → train linear probe."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents ArxivMIA construction (post-2024 non-member, RedPajama member), Section 4.2 describes training data collection (real vs synthetic), prompt template processing (Section 3.3), and dataset splits (1:4 validation/test, Section 5.1). Appendix A details synthetic data generation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses two specific limitations: generalization requiring domain-specific training data, and computational resource requirements with LoRA comparison."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'the generalizability of the probe classifier, which necessitates domain-specific training data' and the computational overhead with a concrete LoRA comparison showing 62.7 vs 74.3 AUC (Table 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what was NOT tested. The mention of 'future work could extend our methods to larger model scales or apply them to multi-modal models' implicitly suggests untested areas but does not explicitly bound the current scope (e.g., English-only, models up to 13B, specific domains only)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The ArxivMIA dataset and code are released at https://github.com/zhliu0106/probing-lm-data. WikiMIA is a public benchmark. The underlying data can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes ArxivMIA construction: member data from RedPajama arxiv subset, non-member from post-2024 arxiv abstracts. Section 4.2 describes training data collection (real data post-model-release, synthetic via ChatGPT). Table 1 provides counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (WikiMIA) and arxiv/RedPajama (ArxivMIA)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: data collection → member/non-member split → proxy model fine-tuning → activation extraction → probe training → evaluation. Dataset sizes at each stage are given in Table 1 and Section 4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists: 'National Natural Science Foundation of China (Grant No. 62036004, 62376177)', Provincial Key Laboratory for Computer Information Processing Technology, Collaborative Innovation Center, and Priority Academic Program Development."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Soochow University (Institute of Artificial Intelligence) and OPPO AI Center, China."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from the National Natural Science Foundation of China and academic institutions, which have no financial stake in the method's performance. OPPO is an affiliation but the paper does not evaluate OPPO products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state training data cutoff dates for the evaluated models. It references that TinyLLaMA/OpenLLaMA were 'pre-trained on RedPajama' and ArxivMIA uses post-2024 non-member data, but specific cutoff dates for each model are not provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about detecting train/test overlap. The benchmarks are constructed with known member/non-member splits based on temporal cutoffs and training data provenance (Section 4)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "ArxivMIA is designed with contamination awareness: non-member data is post-2024, member data is from RedPajama (known training data). WikiMIA similarly uses temporal splits. The ground truth membership is established by construction."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, wall-clock time, or cost per example is reported. The method requires fine-tuning a proxy model and training a probe, but the computational cost of these steps is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '2 NVIDIA A100 (40GB) GPUs' (Section 5.4) but does not state total GPU hours, training time, or overall computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single runs without seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented without indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we conducted a grid search hyperparameters on a held-out validation set' (Section 5.4) but does not report how many configurations were tried or total compute spent on search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Selection is done on a validation set: 'we set the best learning rate and activation extraction model layer according to the performance of the validation set' (Section 5.4). The validation/test split is 1:4."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 8 methods across 8 model-benchmark combinations without any correction for multiple comparisons or family-wise error rate control."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own method against baselines they also implement, without acknowledging author-evaluation bias or using independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The proposed method requires fine-tuning a proxy model and training a probe classifier, which is substantially more expensive than reference-free baselines (which need only one forward pass). This compute difference is not quantified or discussed as a trade-off."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether AUC on WikiMIA/ArxivMIA actually measures real-world pre-training data detection ability. The construct validity of using temporal splits as ground truth for membership is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splits are explicitly used: ArxivMIA designates 'abstracts published post-2024 as non-member data' while member data comes from RedPajama (Section 4.1). WikiMIA also uses temporal splits by design."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether features other than actual membership (e.g., text style differences between pre-2024 and post-2024 arxiv abstracts, or domain shifts in RedPajama) could leak membership status to the probe."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether member and non-member samples might share structural similarities or whether there are distributional differences between RedPajama-sourced members and post-2024 non-members beyond actual pre-training membership."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper's method itself IS a leakage detection technique (probing for pre-training data membership). Additionally, temporal splits serve as a concrete prevention method for benchmark construction."
       }
     }
   },
   "claims": [
     {
       "claim": "The probing method outperforms all baselines on both WikiMIA and ArxivMIA benchmarks, achieving state-of-the-art AUC values.",
       "evidence": "Table 2 shows the highest AUC for Probe Attack across all model-benchmark combinations (e.g., 69.8 on Pythia WikiMIA vs next-best 65.5). Table 3 shows highest average TPR@5%FPR (11.8 with real data).",
       "supported": "strong"
     },
     {
       "claim": "ArxivMIA is a more challenging benchmark than WikiMIA for pre-training data detection.",
       "evidence": "Table 2 shows consistently lower AUC values on ArxivMIA vs WikiMIA across all methods. Section 6.1 notes 'the average performance across all detection methods is notably lower on ArxivMIA.' Cross-domain results in Table 6 also support this.",
       "supported": "strong"
     },
     {
       "claim": "The probing method benefits from larger model sizes.",
       "evidence": "Figure 2 shows AUC increasing with OpenLLaMA size (3B→7B→13B) for the probing method, while Neighbor Attack shows no significant change. Only 3 data points on one model family.",
       "supported": "moderate"
     },
     {
       "claim": "The method is data-efficient, with optimal performance at around 200 training samples.",
       "evidence": "Figure 3 shows AUC peaking at 200 synthetic training samples on ArxivMIA with TinyLLaMA, with slight decline at 500 and 1000. Tested on only one model and one benchmark.",
       "supported": "moderate"
     },
     {
       "claim": "Both real and synthetic training data are effective for the probing method.",
       "evidence": "Tables 2 and 3 show comparable performance for 'Probe w. Real Data' and 'Probe w. Synthetic Data' across all benchmarks, with real slightly better on WikiMIA and synthetic slightly better on ArxivMIA.",
       "supported": "strong"
     },
     {
       "claim": "Mathematical content is harder for LLMs to memorize and harder for the method to detect than CS content.",
       "evidence": "Table 2 shows ArxivMIA-Math AUC values ~10 points lower than ArxivMIA-CS across all models (e.g., TinyLLaMA: 56.7 vs 64.3 with synthetic data).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across all tables appear to be single-run point estimates with no standard deviation, confidence intervals, or indication of result stability. Given the stochastic nature of fine-tuning the proxy model, results could vary substantially across runs."
     },
     {
       "flag": "Modest improvements without significance testing",
       "detail": "AUC improvements over baselines are often modest (e.g., 69.8 vs 65.5 on Pythia WikiMIA, ~4 points) and could fall within noise without statistical significance tests. The TPR@5%FPR improvements are even smaller."
     },
     {
       "flag": "Unquantified compute overhead",
       "detail": "The method requires fine-tuning a full proxy model and training a probe classifier, which is substantially more expensive than all baselines (which need at most one forward pass). This cost-performance trade-off is never quantified, making practical comparison unfair."
     },
     {
       "flag": "Potential confound between temporal/stylistic signals and membership",
       "detail": "ArxivMIA member data comes from RedPajama (older arxiv papers) while non-member data is post-2024. The probe could be detecting temporal or stylistic shifts in academic writing rather than actual pre-training memorization. This confound is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia", "Yangsibo Huang", "Daogao Liu", "Terra Blevins", "Danqi Chen", "Luke Zettlemoyer"],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "Proposes Min-K% Prob method and WikiMIA benchmark for pre-training data detection, a key baseline and evaluation setting in this paper."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramer", "Eric Wallace", "Matthew Jagielski", "Ariel Herbert-Voss", "Katherine Lee", "Adam Roberts", "Tom Brown", "Dawn Song", "Ulfar Erlingsson"],
       "year": 2021,
       "relevance": "Foundational work on training data extraction from LLMs, demonstrating privacy risks of large language models."
     },
     {
       "title": "Membership inference attacks against language models via neighbourhood comparison",
       "authors": ["Justus Mattern", "Fatemehsadat Mireshghallah", "Zhijing Jin", "Bernhard Schoelkopf", "Mrinmaya Sachan", "Taylor Berg-Kirkpatrick"],
       "year": 2023,
       "relevance": "Proposes the Neighbor Attack for membership inference in language models, a key baseline method."
     },
     {
       "title": "Proving test set contamination in black box language models",
       "authors": ["Yonatan Oren", "Nicole Meister", "Niladri Chatterji", "Faisal Ladhak", "Tatsunori B. Hashimoto"],
       "year": 2023,
       "relevance": "Presents statistical test for benchmark contamination and the contamination detection challenge used for evaluation in Section 6.4."
     },
     {
       "title": "Data contamination quiz: A tool to detect and estimate contamination in large language models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "arxiv_id": "2311.06233",
       "relevance": "Proposes a multiple-choice format for assessing LLM data contamination, directly relevant to benchmark contamination detection."
     },
     {
       "title": "Membership inference attacks from first principles",
       "authors": ["Nicholas Carlini", "Steve Chien", "Milad Nasr", "Shuang Song", "Andreas Terzis", "Florian Tramer"],
       "year": 2022,
       "relevance": "Foundational framework for membership inference attacks in machine learning, establishing evaluation metrics used in this paper."
     },
     {
       "title": "Quantifying privacy risks of masked language models using membership inference attacks",
       "authors": ["Fatemehsadat Mireshghallah", "Kartik Goyal", "Archit Uniyal", "Taylor Berg-Kirkpatrick", "Reza Shokri"],
       "year": 2022,
       "relevance": "Investigates membership inference for masked language models, extending MIA techniques to NLP settings."
     },
     {
       "title": "Pythia: A suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman", "Hailey Schoelkopf", "Quentin Gregory Anthony"],
       "year": 2023,
       "relevance": "Provides the Pythia model suite with known training data (the Pile), used as a target model for evaluation."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["Oscar Sainz", "Jon Ander Campos", "Iker García-Ferrero", "Julen Etxaniz", "Oier Lopez de Lacalle", "Eneko Agirre"],
       "year": 2023,
       "arxiv_id": "2310.18018",
       "relevance": "Highlights the importance of measuring data contamination in LLM benchmark evaluations, motivating the need for detection methods."
     },
     {
       "title": "Membership inference attacks on machine learning: A survey",
       "authors": ["Hongsheng Hu", "Zoran Salcic", "Lichao Sun", "Gillian Dobbie", "Philip S. Yu", "Xuyun Zhang"],
       "year": 2022,
       "relevance": "Comprehensive survey of membership inference attacks in machine learning, providing broader context for the pre-training data detection problem."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful for researchers studying data contamination in LLMs but requires model weights and fine-tuning, limiting immediate practical deployment."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Novel approach of probing internal activations rather than surface features, but builds incrementally on established MIA and probing literatures."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Directly addresses privacy concerns and benchmark contamination risks in LLMs, which are active safety and integrity concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the topical issue of LLM data contamination and benchmark reliability, but presents findings without controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and dataset released on GitHub, allowing researchers to run the detection pipeline on their own models."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Soochow University and OPPO AI Center, which are not widely recognized in the LLM research community."
     }
   }
 }