ai-research-survey

scan.json (32217B)

---

 {
   "paper": {
     "title": "Evaluating Large Language Models for Generalization and Robustness via Data Compression",
     "authors": ["Yucheng Li", "Yunhao Guo", "Frank Guerin", "Chenghua Lin"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.00861",
     "doi": "10.48550/arXiv.2402.00861"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Lossless data compression is proposed as a contamination-resistant evaluation method for LLMs, using temporal splits around model training cutoffs. Across 14 models and 6 data sources spanning 83 months, compression performance degrades noticeably after cutoff dates, with clear divergence patterns between LLaMA and Llama-2. Models struggle to generalize on news and code but maintain performance on arXiv papers, and all models fail to compress raw multimodal byte streams. A 2K context with sliding window consistently outperforms larger static contexts.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the abstract: 'Our data and code can be found at https://github.com/liyucheng09/llm-compressive.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same GitHub repository is stated to contain the data: 'Our data and code can be found at https://github.com/liyucheng09/llm-compressive.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided in the paper. The paper does not specify library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is referenced but the paper itself does not include commands or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 and all other results report only point estimates (e.g., '7.539' compression rate). No confidence intervals, error bars, or ± notation are present anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Mistral-7B achieves the most favorable balance') based solely on comparing raw compression rate numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports compression rate differences with baseline context throughout Table 3 (e.g., '↑.333' for LLaMA-7B on wikitext, meaning the 2023 rate is 0.333 higher than 2017-2022), and states 'LLaMA-65B's compression rate worsens 20% on 2023 Wikipedia data compared to 2017-2022.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 500 Wikipedia articles, 1270 BBC news articles, 395 GitHub files, etc. were chosen. The sample sizes are stated in Table 1 without rationale or power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviation, variance, or spread measures are reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 includes traditional compression baselines: Gzip, PNG, and FLAC. These provide a comparative perspective against the LLM-based compression approach."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 14 LLMs tested are all from 2023 releases. Gzip, PNG, and FLAC are established standard compression algorithms appropriate for comparison. The models represent the state of open-source LLMs at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 systematically varies context size (2K, 4K, 8K, 2K+SW) holding all else constant, measuring each component's contribution to compression performance (Table 5). Section 5.6 analyzes the tokenization component separately."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses compression rate as the primary metric (Table 3), plus bits-per-token (BPT) and bits-per-character (BPC) in Table 6. It also measures time and memory cost in Table 5, and compares to HumanEval pass@1 and MMLU accuracy in Table 4."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant for measuring compression rates — the evaluation is entirely automated and deterministic given the model and data."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly splits data into training period (2017-2022) and testing period (2023) based on model cutoff dates. Section 5.2 states: 'we split the test data into the training period (2017-2022) and the testing period (2023).'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-source breakdowns across all 6 datasets (Wikitext, BBC News, Code, arXiv, Image, Audio) for every model. Additional per-model-size comparisons are also provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 discusses that 'large language models struggle with pure byte streams' on multimodal data, and that models face challenges with 2023 Wikipedia, news, and code. Section 5.2 notes CodeLlama's lower robustness despite better code performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: all models fail on multimodal compression (§5.4), CodeLlama has weaker generalization than base Llama-2 despite better code performance (§5.2), larger vocabularies hurt token-level prediction (§5.6), and larger contexts do not outperform sliding window (§5.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about compression rate reducing after cutoff (Fig 1, §5.1), Mistral and Llama-2 showing good balance (Fig 2b, §5.2), models struggling on news and code (Table 3, §5.4), and context/tokenization impact (Tables 5-6, §5.5-5.6) are all supported by corresponding results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'further training on domain knowledge can lead to better domain capability, it may result in weaker generalization compared to the base model' based on comparing CodeLlama vs Llama-2 (§5.2). This is a causal claim from an observational comparison — these models differ in many ways beyond code training, and no controlled experiment isolates this factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'Large Language Models' broadly, but only open-source models up to 70B are tested. Key closed-source models (GPT-4, Claude, PaLM) are excluded. Findings like 'models struggle to generalize on news and code data' (abstract) may not hold for larger or closed-source models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the post-cutoff performance decline could be driven by topic drift, vocabulary shift, or world-event novelty rather than purely data contamination effects. The arXiv stability finding is attributed to 'consistent writing styles' without considering alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames compression rate as a metric grounded in information theory (§2.2), discusses the theoretical equivalence between compression and prediction, and validates the proxy against established benchmarks (HumanEval, MMLU) in Table 4 and §5.3 to show correlation. The connection between compression and generalization is the paper's core contribution."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 specifies each model with name, release date, size, and context length (e.g., 'Llama-2, 2023-07, 7/13/70B, 4096'). For open-source models, the family name + parameter count identifies the exact checkpoint."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. Models compute token-level likelihoods on raw data for compression — no prompts or instructions are sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports context window size (2K default, with 4K, 8K, and 2K+SW variants in §5.5), sliding window step size (512 tokens), chunk size (equal to context window), and arithmetic coding precision (32-bit). For likelihood computation, temperature/top-p are not applicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method directly computes token likelihoods using LLMs and applies arithmetic coding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents preprocessing for each source: Wikipedia articles monitored monthly, BBC articles from front page only, GitHub filtered to newly added or 50%+ changed files, arXiv with author info/bibliographies/appendices excluded, images converted to 64×128 grayscale patches, audio converted to 16kHz FLAC."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated limitations section. It goes from Results (§5) to Conclusion (§6) to Impact (§7), which only says 'There are many potential societal consequences of our work, none which we feel must be specifically highlighted here.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no consideration of specific threats such as the unknown cutoff dates for most models, the limitation to English-only text evaluation, or the potential for non-temporal confounds."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do not show. No discussion of excluded model types (closed-source), excluded languages (only English text analysis noted briefly in §5.6), or limits on the generalization of the proposed method."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository at https://github.com/liyucheng09/llm-compressive is stated to contain both data and code, allowing independent verification of the compression results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 provides detailed data collection descriptions: 500 Wikipedia articles monitored monthly, 1270 BBC front-page articles per month, 75 GitHub projects with rich commit history, randomly collected arXiv papers across all disciplines, BBC news images, and BBC radio/podcast audio. Time span (Jan 2017 to Nov 2023), sizes, and selection criteria are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are public archives (Wikipedia, BBC, GitHub, arXiv) documented in §4.1."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: collect monthly data from each source → tokenize/convert to byte streams → segment into chunks of context size C → compute likelihood with language model (Eq. 3) → apply arithmetic coding (Algorithms 1-2) → measure compressed size / raw size. Each transformation step is described in §3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. No mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Surrey (UK), Harbin Engineering University (China), University of Manchester (UK). These are academic institutions with no direct financial interest in the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure means the criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 lists training cutoff dates where known: LLaMA ~2020 (estimated from CommonCrawl dump dates), Llama-2 September 2022. The paper acknowledges that 'many of these models have their technical reports released, limited details are shared regarding their pre-training data' (§4.2)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central methodology. Section 5.1 analyzes how model performance correlates with training data collection time, and §5.2 explicitly splits data into training period (2017-2022) and testing period (2023) to measure overlap effects."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Addressing benchmark contamination is the paper's primary motivation. Sections 1 and 2.1 extensively discuss contamination in existing benchmarks (citing 30-80% contamination rates in MMLU and SQuAD). The proposed temporal-split method is designed specifically to avoid contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It evaluates LLM compression capabilities on automated datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available text, code, images, and audio data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports inference time in seconds (e.g., 118s for Mistral-7B at 2K context) and memory usage in megabytes (e.g., 15487MB) for each model across different context sizes."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not mention what GPU hardware was used, total GPU hours, or overall compute cost for the experiments. Table 5 provides per-model relative costs but without hardware specification."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single deterministic runs (compression is deterministic given the model, but initial conditions or batching could vary)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Results are presented without indicating whether they come from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The default 2K context size is used without justification, and the context size analysis in §5.5 explores a few configurations without reporting a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper defaults to 2K context for all main results without justifying this choice. The context size analysis (§5.5) shows 2K+SW is best, but main results (Table 3) use 2K without explanation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose a new evaluation method and validate it by comparing to existing benchmarks (Table 4), but do not acknowledge potential bias in evaluating their own method or discuss limitations of their own approach vs alternatives."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports compression rate alongside computational cost (time and memory) for different context sizes, showing that the 2K+SW approach requires ~4x compute but outperforms larger static contexts. The time/memory/performance tradeoff is explicitly discussed in §5.5."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses whether compression rate measures generalization ability (§2.2), grounding it in Shannon's information theory and citing Delétang et al. (2023) and Rae (2023). Section 5.3 validates the metric against HumanEval and MMLU, showing close rank correlation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The method directly uses models for likelihood computation without any scaffolding layer."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the paper's central concern. The entire methodology splits data by model training cutoff dates, and §5.1 explicitly analyzes how performance correlates with whether data falls within or after the training period."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage is not explicitly discussed. While the compression evaluation setup (raw data without prompts) naturally avoids many forms of feature leakage, the paper does not explicitly address whether the evaluation setup could leak answer information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its test data sources overlap structurally with training data. For example, the 500 monitored Wikipedia articles may have earlier versions in model training data, and the same GitHub repositories could appear in training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The temporal split itself serves as a concrete leakage prevention method. Data is split by model training cutoff dates, with 2023 data used as the testing period since it postdates all models' training. This is a principled prevention strategy documented in §3 and §5.1."
       }
     }
   },
   "claims": [
     {
       "claim": "Models' compression performance over time correlates closely with their training data collection cutoff, with clear divergences after the cutoff date.",
       "evidence": "Figure 1 shows LLaMA's compression rate worsening rapidly after its 2020 cutoff, while Llama-2 maintains performance until its September 2022 cutoff before also declining. Demonstrated on both wikitext and BBC news datasets (§5.1).",
       "supported": "strong"
     },
     {
       "claim": "Models with similar performance on training-period data can demonstrate widely different generalization results on new, unseen data.",
       "evidence": "Table 3 shows models with comparable average rates diverge significantly on 2023 data. LLaMA-65B's compression rate worsens 20% on 2023 Wikipedia vs 2017-2022 (§5.2). Figure 2(b) visualizes the performance-robustness tradeoff space.",
       "supported": "strong"
     },
     {
       "claim": "Models struggle to generalize on wikitext, news, and code data but generalize well on arXiv papers.",
       "evidence": "Table 3 shows all models have worse (higher) compression rates on 2023 wikitext, news, and code data (↑ arrows), but maintain or improve on arXiv (↓ arrows). The paper attributes this to 'consistent writing styles in academic papers' (§5.4).",
       "supported": "moderate"
     },
     {
       "claim": "All models fail to compress multimodal data (images and audio).",
       "evidence": "Table 3 shows compression rates of 150-210% for images and audio, far worse than traditional compressors like PNG (64%) and FLAC (76%). The paper notes 'limited capabilities in dealing with raw byte streams' (§5.4).",
       "supported": "strong"
     },
     {
       "claim": "A 2K context with sliding window consistently outperforms larger static context sizes (4K, 8K).",
       "evidence": "Table 5 shows 2K+SW achieves the lowest compression rates across all tested models (e.g., Mistral-7B: 7.385% vs 7.462% for 8K), at the cost of approximately 4x compute (§5.5).",
       "supported": "moderate"
     },
     {
       "claim": "Compression performance correlates closely with established benchmark rankings (HumanEval, MMLU).",
       "evidence": "Table 4 shows model rankings on the paper's code compression metric closely match HumanEval rankings, and arXiv compression ranks match MMLU rankings (§5.3). However, this is based on rank correlation across only 7 models.",
       "supported": "moderate"
     },
     {
       "claim": "Models with larger vocabularies have more difficulty with token-level prediction despite better pre-compression.",
       "evidence": "Table 6 shows Qwen (152K vocab) achieves higher BPT (2.75) than Llama-2 (32K vocab, BPT 2.31) while having fewer total tokens. BPC comparison is closer, suggesting the token-level difficulty partially offsets the pre-compression benefit (§5.6).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 3-6 and Figures 1-5 are single-point estimates with no confidence intervals, standard deviations, or significance tests. For claims comparing models (e.g., 'Mistral-7B achieves the most favorable balance'), the observed differences may not be statistically meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. Key limitations like unknown cutoff dates for most models, restriction to English text, exclusion of closed-source models, and potential Wikipedia version overlap with training data are not discussed."
     },
     {
       "flag": "Unknown training cutoffs for most models",
       "detail": "Table 2 shows cutoff dates for only LLaMA (~2020, estimated) and Llama-2 (September 2022). The remaining 10 model configurations have no known cutoff, yet 2023 is used as the universal test period. If some models' training data extends into 2023, the temporal split is compromised."
     },
     {
       "flag": "Missing hardware specification",
       "detail": "No GPU type or hardware specification is mentioned. Table 5 reports time and memory but without anchoring to specific hardware, the cost numbers are not reproducible or comparable."
     }
   ],
   "cited_papers": [
     {
       "title": "Language modeling is compression",
       "authors": ["G. Delétang", "A. Ruoss", "P.-A. Duquenne", "E. Catt", "T. Genewein", "C. Mattern", "J. Grau-Moya", "L. K. Wenliang", "M. Aitchison", "L. Orseau"],
       "year": 2023,
       "arxiv_id": "2309.10668",
       "relevance": "Establishes the theoretical equivalence between language modeling and compression, the foundation of this paper's evaluation method."
     },
     {
       "title": "An open source data contamination report for large language models",
       "authors": ["Y. Li", "F. Guerin", "C. Lin"],
       "year": 2023,
       "arxiv_id": "2310.17589",
       "relevance": "Shows benchmark contamination can inflate accuracy by 7-14% on MMLU and C-Eval, motivating the need for contamination-resistant evaluation."
     },
     {
       "title": "Data contamination through the lens of time",
       "authors": ["M. Roberts", "H. Thakur", "C. Herlihy", "C. White", "S. Dooley"],
       "year": 2023,
       "arxiv_id": "2310.10628",
       "relevance": "Reveals significant association between a code problem's presence on GitHub and a model's pass rate, directly relevant to temporal contamination analysis."
     },
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "authors": ["M. Sclar", "Y. Choi", "Y. Tsvetkov", "A. Suhr"],
       "year": 2023,
       "arxiv_id": "2310.11324",
       "relevance": "Demonstrates LLMs' high sensitivity to prompt design, motivating prompt-free evaluation methods like compression-based approaches."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun", "Q. Yuan"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, used as a comparison point for validating the compression-based evaluation method."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["D. Hendrycks", "C. Burns", "S. Basart", "A. Zou", "M. Mazeika", "D. Song", "J. Steinhardt"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "Introduces MMLU, a major LLM evaluation benchmark that is shown to have 30-80% contamination rates and is used for validation comparison."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder", "M. Subbiah"],
       "year": 2020,
       "relevance": "GPT-3 paper reporting that Wikipedia-based benchmarks are almost entirely contained in training data, demonstrating the contamination problem at scale."
     },
     {
       "title": "Stop uploading test data in plain text: Practical strategies for mitigating data contamination by evaluation benchmarks",
       "authors": ["A. Jacovi", "A. Caciularu", "O. Goldman", "Y. Goldberg"],
       "year": 2023,
       "arxiv_id": "2305.10160",
       "relevance": "Proposes strategies to mitigate benchmark contamination, directly relevant to the survey's focus on evaluation methodology quality."
     },
     {
       "title": "LLMZip: Lossless text compression using large language models",
       "authors": ["C. S. K. Valmeekam", "K. Narayanan", "D. Kalathil", "J.-F. Chamberland", "S. Shakkottai"],
       "year": 2023,
       "arxiv_id": "2306.04050",
       "relevance": "Demonstrates that LLMs can achieve state-of-the-art lossless compression via arithmetic coding, directly relevant to compression-based evaluation."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Y. Chang", "X. Wang", "J. Wang", "Y. Wu", "L. Yang", "K. Zhu"],
       "year": 2023,
       "relevance": "Comprehensive survey on LLM evaluation methods, providing context for the challenges in current evaluation approaches that this paper addresses."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["O. Sainz", "J. A. Campos", "I. García-Ferrero", "J. Etxaniz", "O. L. de Lacalle", "E. Agirre"],
       "year": 2023,
       "arxiv_id": "2310.18018",
       "relevance": "Argues for per-benchmark contamination measurement, highlighting the contamination crisis in LLM evaluation that this paper aims to address."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Researchers could adopt compression-based evaluation as a complement to benchmarks, but practitioners cannot directly apply this in product development."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that performance clearly degrades after training cutoff confirms data contamination concerns rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised by this evaluation methodology paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the 'benchmarks are contaminated' narrative and shows some models generalize poorly, but stops short of making provocative claims about specific companies."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data are released on GitHub, allowing reproduction of compression evaluations on new models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (LLaMA, Mistral, CodeLlama) but authors are from universities rather than major AI labs."
     }
   }
 }