ai-research-survey

scan.json (27270B)

---

 {
   "paper": {
     "title": "Relative Scaling Laws for LLMs",
     "authors": ["William Held", "David Hall", "Percy Liang", "Diyi Yang"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.24626",
     "doi": "10.48550/arXiv.2510.24626"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states experimental code is in the Marin Github, analysis/plotting code in a separate project repository, and all 255 model checkpoints are released on HuggingFace."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (CommonPile, DCLM Baseline, Nemotron-CC) and releases all 255 model checkpoints on HuggingFace. Evaluation benchmarks (MMLU, ICE, Anthropic risk evals) are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper references external code repositories but does not specify dependency versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code repositories and model checkpoints are referenced, the paper itself does not provide step-by-step reproduction instructions. The reader must navigate external repositories."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The scaling law plots show fitted curves but no confidence intervals or error bars on the individual data points or fitted parameters."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 states: 'we only interpret the slope if the sign is significant at P < 0.05 by a bootstrap significance test.' Pearson correlations in Section 4.2 include p-values (P=0.002, P=0.001)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout: percentage relative error reductions (e.g., '−29% Humanities at 10^18 FLOPs narrowing to within 5% at 10^20'), Pearson R correlations (0.82–0.84), and slopes like '0.3–0.4% relative error slope improvement per ten-fold increase in speaker population.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 85 models per dataset (255 total) or the specific FLOP range was chosen. No power analysis for detecting the relative scaling effects."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation across runs is reported. Each model appears to be a single training run. No discussion of run-to-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Traditional (absolute) scaling laws serve as the baseline framework. The paper compares relative scaling against absolute scaling throughout, and validates against held-out models (OLMo 2, Llama 3, Qwen 3) in Figure 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Held-out comparison models include OLMo 2, Llama 3, and Qwen 3 — all contemporary models. Prior work compared includes recent papers (Hoffmann 2022, Besiroglu 2024, Porian 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Figure 6 isolate model-size scaling vs. token scaling, showing that model-size scaling drives relative performance shifts while data scaling leaves relative performance unchanged. The paper also compares across 3 training datasets as an ablation over data distribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses loss (BPB), accuracy, R² fit quality, Pearson correlation, and risk likelihood as metrics across different case studies."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this scaling law analysis — the claims are about statistical relationships between compute and model performance metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 explicitly validates scaling law predictions against held-out models (OLMo 2, Llama 3, Qwen 3) not used for fitting. Evaluation benchmarks (MMLU, ICE, Anthropic risk evals) are separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by MMLU sub-domain (STEM, Humanities, Social Sciences, Misc.), by regional English variety (USA, Canada, Nigeria, Singapore, Sri Lanka + 5 others), and by AI risk cluster (5 categories). This is the core of the paper."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses cases where scaling fails: regional English varieties that diverge with scale (Sri Lanka, Nigeria), and AI risk categories where scaling has no effect (Scheming, Incorrigibility)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative results: scaling does NOT close gaps for all English varieties (divergence for Sri Lanka and Nigeria), scaling does NOT reduce all risk categories (Scheming and Incorrigibility are flat or declining). The Limitations section acknowledges theoretical grounding is missing."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about MMLU convergence, dialect divergence by population size, and AI risk splitting are all supported by Figures 4, 5, and 7 respectively with data across three training corpora."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses controlled IsoFLOP training to isolate the effect of compute. Figure 6 further isolates model-size vs. data scaling as separate causes. The causal identification strategy (matched compute budgets, three independent corpora) is appropriate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'The three case studies we present are necessarily selective, so they should be viewed as a first attempt rather than a full coverage of the application space.' The paper also notes the connection between loss and utility is 'not well studied.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses that intercepts show no correlation with prevalence (only slopes do), considers the role of training data distribution, and Section 4.3 discusses whether adversarial risks require 'additional pressures' beyond pretraining. The Limitations section discusses the gap between loss and utility."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the proxy gap: Section 3.2 establishes that loss is used as a proxy for accuracy, and the Limitations section states 'the connection between raw language modeling loss, as studied in the case study on linguistic variation, and broad utility for downstream users is not well studied.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Qwen 3 architecture for training, with exact model sizes ranging by width (512–4096). Held-out models are named (OLMo 2, Llama 3, Qwen 3). All 255 models are released on HuggingFace."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D (Figure A.4) provides the exact prompt formats used for MMLU evaluation, including Default MCQ, Continuation-Form, and Modified MCQ with the actual template and scored completions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides comprehensive hyperparameter details: learning rate formula, β1=0.95, β2=0.95, ε=10^-15, weight decay=0.1, gradient clipping at norm 1.0, warmup 5%, linear decay 20%, batch size formula, and attention head configuration."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains and evaluates Transformer models directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the three training datasets and their design philosophies. For ICE (Section 4.2), the paper specifies restriction to the written component and describes the corpus structure (~1M words per variety, 500 texts of ~2,000 words). Appendix E documents the clustering of AI risk behaviors."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Conclusion) contains a dedicated 'Limitations' paragraph discussing theoretical grounding gaps, selective case studies, and the loss-utility connection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific threats: 'Our analyses are primarily empirical and do not yet provide the kind of theoretical grounding suggested by prior work,' 'three case studies we present are necessarily selective,' and 'the connection between raw language modeling loss...and broad utility for downstream users is not well studied.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: case studies 'should be viewed as a first attempt rather than a full coverage,' identifies what was NOT tested (targeted data augmentation, multimodal models, post-training effects), and notes the loss-utility gap."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All 255 model checkpoints are released on HuggingFace. Experimental logs are viewable on the Marin data browser (footnote 1). Training datasets are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the three training datasets and their characteristics. Section 4.2 describes ICE corpus (collection methodology, sampling of 500 texts per variety). Appendix E documents AI risk behavior clustering from Perez et al. (2023)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks and corpora (MMLU, ICE, Anthropic risk evals)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: IsoFLOP training → compute-optimal selection (Figure 2 left) → evaluation on benchmarks → scaling law fitting → relative scaling analysis. Appendix A details the training configuration pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses: Google TPU Research Cloud, Stanford HAI–GCP Grant (Marin Project), and Open Philanthropy funding for WH and DY."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stanford University, OpenAthena, Georgia Institute of Technology. No authors are affiliated with the companies whose models are evaluated (OLMo/AI2, Meta/Llama, Alibaba/Qwen)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding from Google TPU Research Cloud (compute), Stanford HAI, and Open Philanthropy. Open Philanthropy has interest in AI safety but the paper's findings on AI risk are mixed (some risks increase, some don't), suggesting no outcome pressure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper trains its own models on CommonPile, DCLM Baseline, and Nemotron-CC but does not state the cutoff dates for these training datasets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU questions, ICE text, or Anthropic risk evaluation questions appear in the training corpora (CommonPile, DCLM, Nemotron-CC)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU was published in 2021 and ICE much earlier. All three training corpora include web data that could contain benchmark content. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or evaluation time reported for the 255 models across the benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The compute budget is the core experimental variable: 255 models trained under IsoFLOP budgets from 10^18 to 10^20 FLOPs. Google TPU Research Cloud is acknowledged as the compute provider."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity. Each of the 255 models appears to be a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state whether results are from single runs or averaged over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 mentions hyperparameters are set via heuristic reparameterizations from a tuned configuration (Wen et al., 2025), but the search budget for the original tuning is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The compute-optimal model selection is explicit: for each FLOP budget, model size and token count are swept, and the compute-optimal point is selected (Figure 2, left panel). The selection criterion is clearly stated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many relative scaling comparisons are made across domains, regions, and risk categories, but no multiple comparison correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors train all 255 models and evaluate them on their proposed framework. No discussion of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance as a function of compute budget is the entire paper — all figures plot performance metrics against FLOPs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses MMLU evaluation protocol issues (prompt format effects on R², surface form competition). The Limitations section questions whether loss maps to downstream utility. Appendix C validates across multiple benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — models are evaluated directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether MMLU, ICE, or Anthropic risk evaluation content exists in the CommonPile, DCLM, or Nemotron-CC training corpora."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context or whether the modified MCQ prompt format introduces feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and evaluation benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied despite training models on web-scale corpora and evaluating on public benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "Academic domains on MMLU converge toward parity as compute increases, regardless of training data distribution.",
       "evidence": "Figure 4 shows convergence across all three training corpora. For CommonPile, gaps narrow from −29%/−16%/−19% at 10^18 FLOPs to within 5% at 10^20 FLOPs. Similar patterns for DCLM and Nemotron.",
       "supported": "strong"
     },
     {
       "claim": "Regional English varieties show mixed relative scaling trajectories correlated with online English-speaking population size.",
       "evidence": "Figure 5 shows Canada converging, Singapore flat, Sri Lanka and Nigeria diverging. Pearson R = 0.82–0.84 (p < 0.005) between relative scaling slope and estimated online English-speaking population across all 10 ICE corpora and 3 training datasets.",
       "supported": "strong"
     },
     {
       "claim": "AI risk behaviors split into two groups: capability/influence-related risks increase with scale while adversarial risks (Scheming, Incorrigibility) do not.",
       "evidence": "Figure 7 shows Self-Improvement, Influence, and Self-Replication scaling with compute while Scheming and Incorrigibility are flat or declining across all three training corpora.",
       "supported": "moderate"
     },
     {
       "claim": "Model-size scaling drives relative performance shifts, while data scaling leaves relative performance unchanged.",
       "evidence": "Figure 6 shows that scaling model size at fixed tokens produces relative shifts similar to compute-optimal scaling, but scaling tokens at fixed model size yields nearly parallel lines.",
       "supported": "moderate"
     },
     {
       "claim": "A modified MCQ prompt format achieves both high accuracy and predictable loss scaling for MMLU.",
       "evidence": "Figure 3 shows modified MCQ achieves R²=0.61 and max 81.3% accuracy, compared to CF (R²=0.68, max 57.7%) and standard MCQ (R²=0.28, max 82.0%).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Relative scaling laws reveal that scaling compute has non-uniform effects across subpopulations. MMLU knowledge domains converge with scale regardless of training data, but regional English varieties diverge for smaller-population dialects (Pearson R=0.82–0.84 between population size and scaling slope). AI risk behaviors split: capability-related risks increase with scale while adversarial risks (scheming, incorrigibility) do not emerge during pretraining. Model size, not data volume, drives these relative shifts.",
   "red_flags": [
     {
       "flag": "No contamination analysis for benchmark evaluation",
       "detail": "255 models are trained on web-scale corpora (CommonPile, DCLM, Nemotron-CC) and evaluated on public benchmarks (MMLU published 2021, ICE much earlier). No analysis of whether benchmark content appears in training data. Since relative scaling laws measure performance gaps, differential contamination across subdomains could confound the relative scaling conclusions."
     },
     {
       "flag": "Single training runs without variance reporting",
       "detail": "Each of the 255 models appears to be a single training run with no seed sensitivity analysis. Given the paper's claim that relative scaling slopes are meaningful at P < 0.05, the absence of run-to-run variance is a concern — the bootstrap test on slopes may not capture training stochasticity."
     },
     {
       "flag": "AI risk evaluation limitations",
       "detail": "The Anthropic AI risk evaluations (Perez et al., 2023) are model-written evaluations measuring binary yes/no tendencies in base models near 50% likelihood. The absolute risk likelihoods range from ~49% to ~53% — very close to chance. Small shifts in this regime may not reflect meaningful behavioral differences, yet the paper draws conclusions about risk trajectory."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling law paper establishing power-law relationships between compute, data, parameters, and model performance."
     },
     {
       "title": "Training Compute-Optimal Large Language Models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws establishing compute-optimal model size and data size tradeoffs."
     },
     {
       "title": "Inverse Scaling: When Bigger Isn't Better",
       "authors": ["Ian R. McKenzie"],
       "year": 2023,
       "relevance": "Documents cases where larger models perform worse, directly relevant to understanding non-uniform scaling effects."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Challenges emergent abilities claims as metric artifacts, directly relevant to evaluation methodology for scaling studies."
     },
     {
       "title": "Discovering Language Model Behaviors with Model-Written Evaluations",
       "authors": ["Ethan Perez"],
       "year": 2023,
       "relevance": "Source of the Anthropic AI risk evaluations used in Case Study 3 for measuring risk behavior scaling."
     },
     {
       "title": "Alignment Faking in Large Language Models",
       "authors": ["Ryan Greenblatt"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Directly relevant to AI safety evaluation — studies deceptive alignment behaviors that relate to the scheming risk cluster examined in this paper."
     },
     {
       "title": "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling",
       "authors": ["Stella Biderman"],
       "year": 2023,
       "relevance": "Open scaling suite for scaling law evaluation, analogous resource to what this paper provides."
     },
     {
       "title": "Language Models Scale Reliably with Over-Training and on Downstream Tasks",
       "authors": ["Samir Yitzhak Gadre"],
       "year": 2024,
       "arxiv_id": "2403.08540",
       "relevance": "Studies downstream scaling reliability, directly relevant to understanding when scaling laws predict task performance."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["Percy Liang"],
       "year": 2023,
       "relevance": "HELM benchmark emphasizing multi-domain evaluation and robustness across distributions."
     },
     {
       "title": "Paloma: A Benchmark for Evaluating Language Model Fit",
       "authors": ["Ian Magnusson"],
       "year": 2024,
       "arxiv_id": "2312.10523",
       "relevance": "Benchmark for evaluating model fit across heterogeneous domains, directly relevant to measuring distributional robustness."
     },
     {
       "title": "Chinchilla Scaling: A Replication Attempt",
       "authors": ["Tamay Besiroglu"],
       "year": 2024,
       "arxiv_id": "2404.10102",
       "relevance": "Replication study of Chinchilla scaling laws highlighting reproducibility challenges in scaling research."
     },
     {
       "title": "Scaling Laws Are Unreliable for Downstream Tasks: A Reality Check",
       "authors": ["Nicholas Lourie"],
       "year": 2025,
       "arxiv_id": "2507.00885",
       "relevance": "Challenges reliability of scaling laws for downstream predictions, directly relevant to the evaluation methodology concerns in this paper."
     }
   ]
 }