ai-research-survey

scan-v4.json (34010B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Evaluating Large Language Models for Generalization and Robustness via Data Compression",
     "authors": [
       "Yucheng Li",
       "Yunhao Guo",
       "Frank Guerin",
       "Chenghua Lin"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.00861",
     "doi": "10.48550/arXiv.2402.00861"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about compression rate reducing after cutoff (Fig 1, §5.1), Mistral and Llama-2 showing good balance (Fig 2b, §5.2), models struggling on news and code (Table 3, §5.4), and context/tokenization impact (Tables 5-6, §5.5-5.6) are all supported by corresponding results.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'further training on domain knowledge can lead to better domain capability, it may result in weaker generalization compared to the base model' based on comparing CodeLlama vs Llama-2 (§5.2). This is a causal claim from an observational comparison — these models differ in many ways beyond code training, and no controlled experiment isolates this factor.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'Large Language Models' broadly, but only open-source models up to 70B are tested. Key closed-source models (GPT-4, Claude, PaLM) are excluded. Findings like 'models struggle to generalize on news and code data' (abstract) may not hold for larger or closed-source models.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the post-cutoff performance decline could be driven by topic drift, vocabulary shift, or world-event novelty rather than purely data contamination effects. The arXiv stability finding is attributed to 'consistent writing styles' without considering alternatives.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames compression rate as a metric grounded in information theory (§2.2), discusses the theoretical equivalence between compression and prediction, and validates the proxy against established benchmarks (HumanEval, MMLU) in Table 4 and §5.3 to show correlation. The connection between compression and generalization is the paper's core contribution.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated limitations section. It goes from Results (§5) to Conclusion (§6) to Impact (§7), which only says 'There are many potential societal consequences of our work, none which we feel must be specifically highlighted here.'",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no consideration of specific threats such as the unknown cutoff dates for most models, the limitation to English-only text evaluation, or the potential for non-temporal confounds.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do not show. No discussion of excluded model types (closed-source), excluded languages (only English text analysis noted briefly in §5.6), or limits on the generalization of the proposed method.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. No mention of grants, sponsors, or funding agencies.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Surrey (UK), Harbin Engineering University (China), University of Manchester (UK). These are academic institutions with no direct financial interest in the evaluated models.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure means the criterion is not satisfied.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines 'generalization' (compression performance on post-cutoff testing period) and 'robustness' (gap between training and testing period compression rates) in the introduction and operationalizes them consistently throughout.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it contributes a lossless compression-based evaluation framework that avoids data contamination and prompt sensitivity, with a specific temporal split methodology and 83-month dataset.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides substantive background engaging with prior benchmark evaluation approaches, compression-language model equivalence (Deletang et al.), and contamination literature, explaining how this work differs by using temporal splits.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper states in the abstract: 'Our data and code can be found at https://github.com/liyucheng09/llm-compressive.' A working GitHub URL is provided.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The same GitHub repository is stated to contain the data: 'Our data and code can be found at https://github.com/liyucheng09/llm-compressive.'",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided in the paper. The paper does not specify library versions or dependencies.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is referenced but the paper itself does not include commands or a 'Reproducing Results' section.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 3 and all other results report only point estimates (e.g., '7.539' compression rate). No confidence intervals, error bars, or ± notation are present anywhere in the paper.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes comparative claims (e.g., 'Mistral-7B achieves the most favorable balance') based solely on comparing raw compression rate numbers without any statistical significance tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports compression rate differences with baseline context throughout Table 3 (e.g., '↑.333' for LLaMA-7B on wikitext, meaning the 2023 rate is 0.333 higher than 2017-2022), and states 'LLaMA-65B's compression rate worsens 20% on 2023 Wikipedia data compared to 2017-2022.'",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification is provided for why 500 Wikipedia articles, 1270 BBC news articles, 395 GitHub files, etc. were chosen. The sample sizes are stated in Table 1 without rationale or power analysis.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be from single runs. No standard deviation, variance, or spread measures are reported across experimental runs.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 includes traditional compression baselines: Gzip, PNG, and FLAC. These provide a comparative perspective against the LLM-based compression approach.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The 14 LLMs tested are all from 2023 releases. Gzip, PNG, and FLAC are established standard compression algorithms appropriate for comparison. The models represent the state of open-source LLMs at the time.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 systematically varies context size (2K, 4K, 8K, 2K+SW) holding all else constant, measuring each component's contribution to compression performance (Table 5). Section 5.6 analyzes the tokenization component separately.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses compression rate as the primary metric (Table 3), plus bits-per-token (BPT) and bits-per-character (BPC) in Table 6. It also measures time and memory cost in Table 5, and compares to HumanEval pass@1 and MMLU accuracy in Table 4.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation is irrelevant for measuring compression rates — the evaluation is entirely automated and deterministic given the model and data.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly splits data into training period (2017-2022) and testing period (2023) based on model cutoff dates. Section 5.2 states: 'we split the test data into the training period (2017-2022) and the testing period (2023).'",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 provides per-source breakdowns across all 6 datasets (Wikitext, BBC News, Code, arXiv, Image, Audio) for every model. Additional per-model-size comparisons are also provided.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.4 discusses that 'large language models struggle with pure byte streams' on multimodal data, and that models face challenges with 2023 Wikipedia, news, and code. Section 5.2 notes CodeLlama's lower robustness despite better code performance.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Several negative findings are reported: all models fail on multimodal compression (§5.4), CodeLlama has weaker generalization than base Llama-2 despite better code performance (§5.2), larger vocabularies hurt token-level prediction (§5.6), and larger contexts do not outperform sliding window (§5.5).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 specifies each model with name, release date, size, and context length (e.g., 'Llama-2, 2023-07, 7/13/70B, 4096'). For open-source models, the family name + parameter count identifies the exact checkpoint.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "The paper does not use prompting. Models compute token-level likelihoods on raw data for compression — no prompts or instructions are sent to the models.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports context window size (2K default, with 4K, 8K, and 2K+SW variants in §5.5), sliding window step size (512 tokens), chunk size (equal to context window), and arithmetic coding precision (32-bit). For likelihood computation, temperature/top-p are not applicable.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The method directly computes token likelihoods using LLMs and applies arithmetic coding.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 documents preprocessing for each source: Wikipedia articles monitored monthly, BBC articles from front page only, GitHub filtered to newly added or 50%+ changed files, arXiv with author info/bibliographies/appendices excluded, images converted to 64×128 grayscale patches, audio converted to 16kHz FLAC.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The GitHub repository at https://github.com/liyucheng09/llm-compressive is stated to contain both data and code, allowing independent verification of the compression results.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 provides detailed data collection descriptions: 500 Wikipedia articles monitored monthly, 1270 BBC front-page articles per month, 75 GitHub projects with rich commit history, randomly collected arXiv papers across all disciplines, BBC news images, and BBC radio/podcast audio. Time span (Jan 2017 to Nov 2023), sizes, and selection criteria are specified.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved. Data sources are public archives (Wikipedia, BBC, GitHub, arXiv) documented in §4.1.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline is documented: collect monthly data from each source → tokenize/convert to byte streams → segment into chunks of context size C → compute likelihood with language model (Eq. 3) → apply arithmetic coding (Algorithms 1-2) → measure compressed size / raw size. Each transformation step is described in §3.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 lists training cutoff dates where known: LLaMA ~2020 (estimated from CommonCrawl dump dates), Llama-2 September 2022. The paper acknowledges that 'many of these models have their technical reports released, limited details are shared regarding their pre-training data' (§4.2).",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "This is the paper's central methodology. Section 5.1 analyzes how model performance correlates with training data collection time, and §5.2 explicitly splits data into training period (2017-2022) and testing period (2023) to measure overlap effects.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Addressing benchmark contamination is the paper's primary motivation. Sections 1 and 2.1 extensively discuss contamination in existing benchmarks (citing 30-80% contamination rates in MMLU and SQuAD). The proposed temporal-split method is designed specifically to avoid contamination.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved in this study. It evaluates LLM compression capabilities on automated datasets.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study uses publicly available text, code, images, and audio data.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 reports inference time in seconds (e.g., 118s for Mistral-7B at 2K context) and memory usage in megabytes (e.g., 15487MB) for each model across different context sizes.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget is stated. The paper does not mention what GPU hardware was used, total GPU hours, or overall compute cost for the experiments. Table 5 provides per-model relative costs but without hardware specification.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single deterministic runs (compression is deterministic given the model, but initial conditions or batching could vary).",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is not explicitly stated. Results are presented without indicating whether they come from single or multiple runs.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget is reported. The default 2K context size is used without justification, and the context size analysis in §5.5 explores a few configurations without reporting a search budget.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper defaults to 2K context for all main results without justifying this choice. The context size analysis (§5.5) shows 2K+SW is best, but main results (Table 3) use 2K without explanation.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": false,
           "answer": false,
           "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors propose a new evaluation method and validate it by comparing to existing benchmarks (Table 4), but do not acknowledge potential bias in evaluating their own method or discuss limitations of their own approach vs alternatives.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 reports compression rate alongside computational cost (time and memory) for different context sizes, showing that the 2K+SW approach requires ~4x compute but outperforms larger static contexts. The time/memory/performance tradeoff is explicitly discussed in §5.5.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": true,
           "justification": "The paper extensively discusses whether compression rate measures generalization ability (§2.2), grounding it in Shannon's information theory and citing Delétang et al. (2023) and Rae (2023). Section 5.3 validates the metric against HumanEval and MMLU, showing close rank correlation.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is involved. The method directly uses models for likelihood computation without any scaffolding layer.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Temporal leakage is the paper's central concern. The entire methodology splits data by model training cutoff dates, and §5.1 explicitly analyzes how performance correlates with whether data falls within or after the training period.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Feature leakage is not explicitly discussed. While the compression evaluation setup (raw data without prompts) naturally avoids many forms of feature leakage, the paper does not explicitly address whether the evaluation setup could leak answer information.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether its test data sources overlap structurally with training data. For example, the 500 monitored Wikipedia articles may have earlier versions in model training data, and the same GitHub repositories could appear in training corpora.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": true,
           "justification": "The temporal split itself serves as a concrete leakage prevention method. Data is split by model training cutoff dates, with 2023 data used as the testing period since it postdates all models' training. This is a principled prevention strategy documented in §3 and §5.1.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Model compression performance correlates closely with training data cutoff dates, with clear divergence after the cutoff.",
       "evidence": "Figure 1 shows LLaMA and Llama-2 performing comparably before LLaMA's 2020 cutoff, then clearly diverging; Table 3 shows performance gap (Rate23 − Rate17-22) aligned with known cutoffs.",
       "supported": "strong"
     },
     {
       "claim": "Models with similar in-distribution performance can demonstrate widely different generalization on unseen post-cutoff data.",
       "evidence": "Table 3 and Figure 2(b) show, e.g., LLaMA-65B's compression rate worsening 20% on 2023 Wikipedia while Mistral-7B shows only 0.115 degradation despite comparable absolute performance.",
       "supported": "strong"
     },
     {
       "claim": "Compression rate rankings correlate with established benchmark scores (HumanEval, MMLU).",
       "evidence": "Table 4 shows ordinal rank agreement between compression rates on code/arXiv and HumanEval/MMLU scores for 7 models (Spearman-style match but no formal correlation computed).",
       "supported": "moderate"
     },
     {
       "claim": "Models generalize well on arXiv data but struggle on Wikipedia, news, and code.",
       "evidence": "Table 3 shows arXiv results with small or negative (improving) Rate23 − Rate17-22 gaps for most models, while news and code show consistent positive gaps.",
       "supported": "moderate"
     },
     {
       "claim": "Larger context sizes improve compression with diminishing returns; 2K+sliding window outperforms static larger contexts.",
       "evidence": "Table 5 shows compression rate improvements from 2K→4K→8K with diminishing gains, and 2K+SW consistently achieves the best rate at 4× the compute cost.",
       "supported": "strong"
     },
     {
       "claim": "All models fail to compress multimodal (image and audio) byte streams effectively.",
       "evidence": "Table 3 shows all LLMs with compression rates of 146–212% on images and audio, far worse than specialized codecs (PNG 64.9%, FLAC 76.3%).",
       "supported": "strong"
     },
     {
       "claim": "Mistral-7B achieves the best generalization-robustness balance among models under 7B parameters.",
       "evidence": "Figure 2(b) places Mistral-7B in the top-right quadrant (strong generalization, high robustness); Table 3 shows lowest BBC news compression rate and small performance gap.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "The paper proposes lossless data compression as a contamination-free, prompt-insensitive LLM evaluation framework, testing 14 models on 6 data sources across 83 months (2017–2023). Compression performance reliably diverges after training cutoffs, empirically confirming that standard benchmarks may conflate memorization with generalization. Domain findings show models generalize well on arXiv (consistent academic style) but degrade on Wikipedia, news, and code (time-sensitive content); all models universally fail on raw multimodal byte streams. Mistral-7B and Llama-2-70B achieve the best generalization-robustness tradeoffs in their size classes, while sliding window contexts outperform static large contexts at the cost of ~4× compute.",
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All model comparisons are made without significance tests or confidence intervals; conclusions about 'best' models or domain-specific findings are based on unverified point estimates."
     },
     {
       "flag": "Uncertain training cutoffs",
       "detail": "Most models (InternLM, CodeLlama, Baichuan2, Mistral, Qwen, ChatGLM3, Yi) do not report exact training cutoffs; using 2023 as the test period assumes these models haven't seen 2023 data, which is unverified."
     },
     {
       "flag": "Ordinal benchmark correlation only",
       "detail": "The correlation between compression rates and HumanEval/MMLU scores (Table 4) is presented as rank-matching without a formal correlation coefficient or statistical measure, overstating the strength of the relationship."
     },
     {
       "flag": "No variance across runs",
       "detail": "All results are single-pass point estimates; no repeated runs or variance measures are reported, making it impossible to assess result stability."
     },
     {
       "flag": "Alternative explanations not ruled out",
       "detail": "The performance gap after cutoff could reflect distribution shift in writing styles or topic drift rather than knowledge memorization; this confound is not discussed or controlled."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Modeling is Compression",
       "relevance": "Core theoretical foundation establishing that LLM log-likelihood minimization is equivalent to lossless compression, directly enabling this paper's evaluation framework."
     },
     {
       "title": "An Open Source Data Contamination Report for Large Language Models",
       "relevance": "Prior work by same author group quantifying contamination rates in MMLU and C-Eval, motivating the need for contamination-free evaluation."
     },
     {
       "title": "Data Contamination Through the Lens of Time",
       "relevance": "Demonstrates association between GitHub code presence and LLM pass rates, directly motivating the temporal split approach."
     },
     {
       "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design",
       "relevance": "Evidence that prompt sensitivity inflates or distorts benchmark results, one of three problems motivating this compression-based alternative."
     },
     {
       "title": "Stop Uploading Test Data in Plain Text: Practical Strategies for Mitigating Data Contamination by Evaluation Benchmarks",
       "relevance": "Documents the benchmark contamination problem this paper aims to solve."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Key benchmark used for correlation validation and cited as an example of contamination-vulnerable evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Second benchmark used for correlation validation; code evaluation benchmark whose rankings are compared against compression-based code evaluation."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "relevance": "Primary model used to demonstrate training cutoff / generalization correlation; only model with publicly documented September 2022 cutoff date."
     },
     {
       "title": "A Survey on Evaluation of Large Language Models",
       "relevance": "Survey of existing evaluation methods that frames the landscape of benchmark-based evaluation this paper critiques."
     },
     {
       "title": "LatestEval: Addressing Data Contamination in Language Model Evaluation through Dynamic and Time-Sensitive Test Construction",
       "relevance": "Related prior work by same authors on contamination-aware evaluation, showing the lineage of this approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Researchers could adopt compression-based evaluation as a complement to benchmarks, but practitioners cannot directly apply this in product development."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that performance clearly degrades after training cutoff confirms data contamination concerns rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised by this evaluation methodology paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the 'benchmarks are contaminated' narrative and shows some models generalize poorly, but stops short of making provocative claims about specific companies."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data are released on GitHub, allowing reproduction of compression evaluations on new models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (LLaMA, Mistral, CodeLlama) but authors are from universities rather than major AI labs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39257837",
         "title": "Tiny Titans: Can Smaller LLMs Punch Above Their Weight?",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39257837"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 0
   }
 }