ai-research-survey

scan.json (24697B)

---

 {
   "paper": {
     "title": "Information Capacity: Evaluating the Efficiency of Large Language Models via Text Compression",
     "authors": ["Cheng Yuan", "Jiawei Shao", "Xuelong Li"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.08066",
     "doi": "10.48550/arXiv.2511.08066"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces 'information capacity,' a metric evaluating LLM efficiency via text compression relative to computational complexity. Evaluating 52 open-source models across 5 datasets, the authors find consistent information capacity within model series, strong linguistic biases across models, and identify tokenizer efficiency, pretraining data, and MoE architecture as three dominant factors. The metric enables single-reference performance prediction across model scales with errors under ±8%, outperforming power-law-based approaches.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/TeleAI-AI-Flow/InformationCapacity"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "HuggingFace dataset link provided: https://huggingface.co/datasets/TeleAI-AI-Flow/InformationCapacity"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A code repository is linked but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates throughout. No confidence intervals or error bars are provided for information capacity measurements."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about model rankings and correlations but uses no statistical significance tests. Pearson correlation coefficients are reported without p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported contextually throughout. For example, Table 3 shows IC gains (e.g., ↑0.0104), Table 6 shows IC reductions from temperature changes, and Figure 7 shows percentage estimation errors. Differences are presented with enough baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (e.g., 200,000 samples for mixed text) but no justification is given for why these sizes were chosen or whether they are sufficient for stable measurements."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Information capacity values appear to be single-run measurements."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares information capacity against the power law method from Kaplan et al. (2020) for performance prediction (Section 4.4, Figure 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons include the densing law (Xiao et al., 2025) discussed in Sections 5.2-5.3, and the power law (Kaplan et al., 2020). These represent the relevant prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents ablation studies on post-training (4.3.1), test sample length (4.3.2), and softmax temperature (4.3.3), examining their impact on information capacity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses information capacity as the primary metric but also reports NLL, FLOPs, text size per token, and correlates with benchmark scores (MMLU, LiveCodeBench, C-Eval)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for evaluating a compression-based efficiency metric for LLMs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Five separate evaluation datasets are used (Mixed text, FinePDFs-en, Ch-FineWeb-Edu, FineWeb-Edu, NextCoder). These are publicly available datasets used for evaluation, not used for any training or tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Table 2), per model series, and across multiple dimensions (tokenizer efficiency in Figure 3, pretraining data in Table 3, MoE in Table 4, temperature in Table 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where information capacity is limited: post-training degrades it (Section 4.3.1), linguistic biases cause poor performance on non-native-language corpora (Section 4.1), and the offset approach's limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results include: post-training degrades information capacity (Section 4.3.1), temperature changes from T=1 always reduce IC (Table 6), and that the formula without offset (Eq. 4) is inadequate (Figure 2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about consistent IC within model series (Figure 1, Table 2), linguistic biases (Table 2 rankings), three major factors (Sections 4.2.1-4.2.3), and performance prediction (Section 4.4) are all supported by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made about tokenizer efficiency, pretraining data, and MoE architecture affecting IC. The ablation studies (Section 4.3) provide controlled manipulations. The TinyLlama pretraining data experiment (Table 3) uses the same model with different data volumes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to provide 'a unified metric of LLM efficiency' but tests only on text compression across open-source models. The title and abstract frame results broadly as evaluating LLM 'efficiency' without bounding to the tested setting of text compression performance."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the consistent information capacity within model series. The offset parameter b is introduced to force consistency but no theoretical justification is given for why it should be constant."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures text compression performance and frames it as 'efficiency' and 'intelligence.' While the correlation between compression and intelligence is cited, the gap between compression performance and actual model capability/efficiency is not substantively discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided throughout (e.g., 'Qwen3-0.6B-Base', 'Llama-3.2-1B', 'TinyLlama-1.1B'). For the 52 models evaluated, size and variant are specified."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. Models are used as probability estimators on input text for compression, not prompted for generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifies: temperature T=1, sequence length L=1024, logits promoted from bfloat16 to float32. The offset values for each dataset are given in Table 1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated as probability estimators."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes preprocessing: text samples are truncated to L=1024 tokens, the first token is excluded, ineffective logits are truncated before softmax, and samples shorter than the threshold are filtered out. Table 1 provides dataset details."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion section (Section 5) discusses some design choices but does not substantively address limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address concerns like the arbitrary offset parameter, the choice of sequence length, or the stability of rankings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what IC does not measure. It acknowledges that post-training models cannot be fairly evaluated (Section 4.3.1) and that base models of some series are unavailable, but does not bound the scope of the metric's applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation datasets are publicly available (HuggingFace link provided, and references to FinePDFs-en, Ch-FineWeb-Edu, FineWeb-Edu, NextCoder). Code is also released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the five datasets with sources, sample counts, minimum/average lengths (Table 1). The Mixed text corpus composition (books, webpages, code, papers) is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets/benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from text to IC is documented: tokenization → truncation to L=1024 → logit extraction → NLL computation → FLOPs calculation (Eq. 6-8) → IC computation (Eq. 5). Filtering criteria (minimum length) are stated in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: 'Institute of Artificial Intelligence (TeleAI), China Telecom.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Authors are from China Telecom (TeleAI), which develops AI products. Whether this creates a conflict is unclear, but the lack of disclosure is a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate models on capability benchmarks. It measures compression performance (NLL) on text corpora, which is not a benchmark of model knowledge that could be contaminated."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper measures text compression, not benchmark task performance. Overlap with pretraining data would affect NLL but is a feature of the compression task, not a contamination issue in the traditional sense."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates compression performance, not knowledge-based benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite FLOPs being central to the metric, the paper does not report the actual computational cost (GPU hours, wall-clock time) of running the evaluation across 52 models and 5 datasets."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated for running the experiments. The paper evaluates 52 models on hundreds of thousands of samples but does not report the hardware or time required."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The NLL computation is deterministic given a model and text, so seed sensitivity is less critical, but this is not discussed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "No statement about number of runs. NLL computation is deterministic for a given input, so single runs may be appropriate, but this is not explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The offset parameter b is a key hyperparameter (Table 1 shows different values per dataset), but no search budget or selection methodology for b is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The offset values in Table 1 differ per dataset (-24, -27, -18.7, -27, -27) but the selection process is not justified. How these values were determined is not explained."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose information capacity as a new metric and evaluate it, but do not acknowledge the bias of evaluating their own metric against alternatives."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The metric itself is about compute vs. performance. The evaluation method has negligible compute differences across models (forward pass only)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 discusses the correlation between IC and benchmark scores (MMLU, LiveCodeBench, C-Eval) and acknowledges that the correlation depends on dataset-benchmark alignment. Section 5.1 discusses what IC captures."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are used directly as probability estimators."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets (FineWeb-Edu, etc.) may overlap with model pretraining data. The paper does not discuss whether models were trained on these evaluation corpora, which would inflate compression performance."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Feature leakage is not a concern here — the task is straightforward NLL computation on text."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets may overlap with pretraining corpora. For example, FineWeb-Edu is a well-known pretraining dataset that some evaluated models may have been trained on. This is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used to check whether evaluation data appeared in model pretraining corpora."
       }
     }
   },
   "claims": [
     {
       "claim": "A series of models with varying sizes exhibits consistent information capacity.",
       "evidence": "Figure 1 and Table 2 show IC values for model families (e.g., Qwen3, Llama-3) remaining roughly constant across sizes within each series.",
       "supported": "moderate"
     },
     {
       "claim": "Tokenizer efficiency is the dominant factor in information capacity, with Pearson correlation coefficients consistently exceeding 0.98.",
       "evidence": "Figure 3 shows linear correlations between text size per token and IC across 4 datasets, with r values from 0.981 to 0.998.",
       "supported": "strong"
     },
     {
       "claim": "Information capacity enables accurate cross-scale NLL prediction from a single reference model with errors bounded within ±8%.",
       "evidence": "Figures 6 and 7 show prediction errors: Qwen3 within ±3% (Figure 6a), Qwen2.5 within ±7.73% (Figure 7a), compared to power law errors exceeding 25% (Figure 7b).",
       "supported": "strong"
     },
     {
       "claim": "Post-training degrades information capacity.",
       "evidence": "Figure 4 and Section 4.3.1 show IC drops for all tested models after post-training (e.g., Qwen3-Base vs Qwen3, Llama-3 vs Llama-3-Instruct).",
       "supported": "strong"
     },
     {
       "claim": "MoE architecture enhances information capacity by reducing FLOPs while maintaining prediction accuracy.",
       "evidence": "Table 4 shows MoE models (Qwen1.5-MoE, Qwen2-MoE) achieving higher IC than dense variants with similar activated parameters.",
       "supported": "strong"
     },
     {
       "claim": "Strong linguistic biases exist in mainstream LLMs, with rankings varying significantly across languages.",
       "evidence": "Table 2 shows Llama-3 ranks 11th on Chinese (Ch-FineWeb-Edu) but 3rd on English PDFs (FinePDFs-en). Gemma-3 similarly underperforms on Chinese.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Arbitrary offset parameter",
       "detail": "The offset b in Eq. 5 is introduced to force consistency of IC across model sizes, with different values per dataset (Table 1), but no principled derivation or selection methodology is provided. This parameter could be tuned to make any metric appear consistent."
     },
     {
       "flag": "Potential pretraining data overlap with evaluation corpora",
       "detail": "Evaluation datasets like FineWeb-Edu are commonly used as pretraining data. Models trained on FineWeb-Edu would achieve lower NLL and higher IC on that dataset, confounding the metric. This is not discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations discussion, which is unusual for a methods paper introducing a new metric."
     },
     {
       "flag": "Company evaluating metric that could benefit its products",
       "detail": "Authors are from China Telecom (TeleAI). The AI Flow framework referenced as a use case (Shao and Li, 2025) is co-authored by the same group. The metric's application to device-edge-cloud infrastructure aligns with their corporate interests."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper that information capacity aims to improve upon for performance prediction."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "Chinchilla scaling laws for compute-optimal training, a key baseline for model efficiency analysis."
     },
     {
       "title": "Densing law of LLMs",
       "authors": ["Chaojun Xiao", "Jie Cai"],
       "year": 2025,
       "relevance": "Direct competitor metric for LLM efficiency that normalizes inference complexity via equivalent parameter count."
     },
     {
       "title": "Language modeling is compression",
       "authors": ["Gregoire Deletang", "Anian Ruoss"],
       "year": 2024,
       "relevance": "Establishes the theoretical link between language modeling and compression that motivates information capacity."
     },
     {
       "title": "Compression represents intelligence linearly",
       "authors": ["Yuzhen Huang", "Jinghan Zhang"],
       "year": 2024,
       "relevance": "Empirically validates the correlation between compression and intelligence that information capacity builds upon."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling parameters for reasoning",
       "authors": ["Charlie Victor Snell", "Jaehoon Lee"],
       "year": 2025,
       "relevance": "Test-time scaling work that motivates the need for inference efficiency metrics."
     },
     {
       "title": "Beyond chinchilla-optimal: accounting for inference in language model scaling laws",
       "authors": ["Nikhil Sardana", "Jacob Portes"],
       "year": 2024,
       "relevance": "Extends scaling laws to account for inference costs, directly relevant to LLM efficiency evaluation."
     },
     {
       "title": "Attention is all you need",
       "authors": ["Ashish Vaswani", "Noam Shazeer"],
       "year": 2017,
       "relevance": "Foundational transformer architecture paper underlying all evaluated models."
     },
     {
       "title": "DeepSeek-V2: A strong, economical, and efficient mixture-of-experts language model",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2405.04434",
       "relevance": "Introduces MLA attention mechanism evaluated in the paper's FLOPs calculations."
     }
   ]
 }