ai-research-survey

scan.json (21043B)

---

 {
   "paper": {
     "title": "Scaling Laws for Neural Language Models",
     "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B. Brown", "Benjamin Chess", "Rewon Child", "Scott Gray", "Alec Radford", "Jeffrey Wu", "Dario Amodei"],
     "year": 2020,
     "venue": "arXiv",
     "arxiv_id": "2001.08361"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["observational", "theoretical"],
   "key_findings": "Language model cross-entropy loss follows power-law relationships with model size, dataset size, and compute budget, spanning seven orders of magnitude. Performance depends strongly on scale but weakly on architectural details like depth vs. width. Optimally compute-efficient training involves training very large models on relatively modest data and stopping well before convergence, with optimal model size scaling as N ∝ C^0.73. The paper derives a unified framework predicting overfit onset, optimal compute allocation, and learning curve shapes.",
   "claims": [
     {
       "claim": "Performance scales as a power law with model size N, dataset size D, and compute C, with trends spanning more than seven orders of magnitude.",
       "evidence": "Figure 1 and Section 3 show power-law fits across 6+ orders of magnitude in N (768 to 1.5B parameters), 2+ orders in D (22M to 23B tokens), and 8 orders in compute. Exponents: αN ≈ 0.076, αD ≈ 0.095, αC^min ≈ 0.050.",
       "supported": "strong"
     },
     {
       "claim": "Performance depends very weakly on architectural hyperparameters (depth, width, attention heads) when total non-embedding parameter count is fixed.",
       "evidence": "Figure 5 shows loss varies only a few percent across a wide range of shapes at fixed N. Aspect ratio can vary by 40x with only ~3% impact. Section 3.1.",
       "supported": "strong"
     },
     {
       "claim": "Optimally compute-efficient training allocates most additional compute to larger models rather than more training steps, with N ∝ C^0.73 and S ∝ C^0.03.",
       "evidence": "Figure 14 and Section 6.1 show empirical fits of N(C_min) ∝ C^0.73 and S_min ∝ C^0.03. Theoretical predictions from L(N,S) match within a few percent (Section 6.2).",
       "supported": "strong"
     },
     {
       "claim": "Dataset size need only grow sublinearly with model size (D ∝ N^0.74) to avoid overfitting.",
       "evidence": "Equation 1.5 and Figure 9 show the combined L(N,D) fit with αN/αD ≈ 0.74. Equation 4.4 provides the practical threshold D ≳ 5000 × N^0.74.",
       "supported": "strong"
     },
     {
       "claim": "Transfer to other data distributions improves in parallel with training distribution performance, with a roughly constant offset in loss.",
       "evidence": "Figure 8 shows test loss on Books, Wikipedia, Common Crawl all follow the same power-law slope, offset from WebText2. Right panel shows generalization depends only on training loss, not training phase.",
       "supported": "strong"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or URL is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "WebText2 is not publicly released. The paper describes how it was constructed but does not provide a download link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details beyond 'memory constraints' for largest models are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The paper describes methodology but not step-by-step reproduction procedures."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported on the power-law fits or individual data points. The paper mentions run-to-run variation is ~0.05 in loss (Appendix D.6) but does not include error bars on main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims of power-law scaling are supported by visual fits, not formal statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as power-law exponents with clear context: e.g., 'doubling the number of parameters yields a loss smaller by a factor 2^−αN = 0.95' (Section 1.2). Exponents αN ≈ 0.076, αD ≈ 0.095, αC ≈ 0.050 quantify the magnitude of scaling effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of model sizes, dataset sizes, or training runs chosen. The range is described but the number of data points for fitting power laws is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Run-to-run variation is mentioned as ~0.05 in Appendix D.6 but no systematic variance reporting (std dev, multiple seeds) is provided for the main results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "LSTMs and Universal Transformers are included as baselines for comparison (Section 3.2.1, Figure 7, Figure 17)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LSTMs and Universal Transformers (2018) were contemporary and relevant architecture comparisons at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Systematic ablation of architectural parameters: depth, width, attention heads, feed-forward dimension are varied independently at fixed N (Section 3.1, Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only cross-entropy loss as its metric throughout. No other metrics (e.g., perplexity in explicit form, downstream task performance) are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to claims about scaling laws for loss metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "A held-out test set of 6.6 × 10^8 tokens from WebText2 is reserved (Section 2.3). Results are reported on this test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Performance is broken down by: data distribution (Figure 8: WebText2, Books, Wikipedia, Common Crawl), token position in context (Figure 20-21), model architecture type (Figure 7, 17), and dataset size (Figure 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where fits break down: very small datasets (~2×10^7 tokens, Section 4.2), 1-layer models (Figure 13), extreme depth-to-width ratios (Figure 6), and the predicted eventual contradiction between scaling laws (Section 6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative/null results reported: architectural shape has minimal effect (Figure 5), LSTMs cannot match Transformer performance on later tokens (Figure 7), recurrent Transformers perform slightly worse per FLOP (Figure 17), convergence is compute-inefficient."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (power-law scaling, weak architecture dependence, overfitting governed by N/D ratio, optimal compute allocation favoring large models stopped before convergence) are supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to frame results as empirical observations and fits rather than causal claims. Ablation-style experiments (varying one factor at a time) support the limited causal claims about architecture independence. Language is appropriately hedged: 'we observe', 'we find', 'we conjecture'."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to Transformer language models trained on WebText2. Section 8 explicitly notes: 'we do not know which of our results depend on the structure of natural language data, and which are universal.' The paper conjectures but does not claim generalization to other domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses the contradiction between scaling laws and what it implies. Appendix C lists specific caveats including confounding from context length, potential hyperparameter sensitivity, and limitations of the compute estimate. The paper discusses multiple functional forms (logarithmic vs power-law, Figure 23)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that it studies cross-entropy loss and does not claim this directly translates to task performance. Section 8 notes: 'it will be important to investigate whether continued improvement on the loss translates into improvement on relevant language tasks. Smooth quantitative change can mask major qualitative improvements.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own models from scratch rather than evaluating pre-existing model versions. Architecture details (layers, dimensions) are fully specified."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. Models are trained on autoregressive language modeling."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters are reported: Adam optimizer, 2.5×10^5 steps, batch size 512 sequences of 1024 tokens, learning rate schedule with 3000-step warmup + cosine decay (Section 2.2). Learning rate formula in Appendix D.6. Adafactor for largest models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is described: Reddit outbound links with ≥3 karma, text extracted with Newspaper3k, reversible BPE tokenizer with nvocab=50257 (Section 2.3). Dataset size: 20.3M documents, 96GB text, 2.29×10^10 tokens."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C ('Caveats') provides a dedicated section listing specific limitations. Section 8 (Discussion) also discusses limitations and open questions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C lists specific threats: no theoretical understanding of scaling laws, uncertainty in Bcrit extrapolation, poor fits for smallest datasets, compute estimate excludes context-dependent terms, potential neglected hyperparameters, learning rate sensitivity to target loss."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: results are for Transformer language models on WebText2. Section 8 states results may not apply to other domains. The paper notes 'we do not know which of our results depend on the structure of natural language data, and which are universal.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the WebText2 dataset nor the raw experimental results (individual training runs, loss curves) are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: Reddit outbound links through October 2018, ≥3 karma threshold, Newspaper3k extraction (Section 2.3). Model training procedures are documented in Section 2.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is web-scraped text."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw web text to tokens is documented: Reddit links → karma filter → Newspaper3k extraction → BPE tokenization → train/test split (Section 2.3). Model parameter counts and compute estimates are derived in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The work was done at OpenAI but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list OpenAI as their affiliation (one co-affiliated with Johns Hopkins). Affiliations are prominently displayed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a direct commercial interest in scaling results — larger models being better supports their business strategy. The funder (OpenAI) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own models from scratch on a known dataset. It does not evaluate a pre-trained model's capability on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — models are trained on WebText2 and evaluated on its held-out test set. No pre-trained model benchmark evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not use external benchmarks to evaluate pre-trained model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported, though the paper extensively discusses compute in PF-days for training."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compute is extensively reported in PF-days throughout the paper. Individual training runs span from ~10^-8 to ~10^3 PF-days (Figures 1, 13, 14). The compute estimation formula C ≈ 6NBS is derived in Section 2.1."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Company evaluating its own scaling paradigm",
       "detail": "All authors are OpenAI employees. The conclusion that 'larger models will continue to perform better' directly supports OpenAI's commercial strategy of building increasingly large models. While the empirical methodology appears sound, the conflict is not disclosed."
     },
     {
       "flag": "No error bars or uncertainty quantification on fits",
       "detail": "Power-law exponents are reported as point estimates (e.g., αN ≈ 0.076) without confidence intervals or goodness-of-fit statistics. Run-to-run variation of ~0.05 is mentioned in an appendix but not propagated to the main results. The claimed precision of exponents like '0.076' may not be warranted."
     },
     {
       "flag": "Private dataset",
       "detail": "WebText2 is not publicly available, making independent verification of the core results impossible. The original WebText was also not released. Results cannot be reproduced on the same data."
     }
   ],
   "cited_papers": [
     {
       "title": "Attention is all you need",
       "authors": ["Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit", "Llion Jones", "Aidan N Gomez", "Łukasz Kaiser", "Illia Polosukhin"],
       "year": 2017,
       "relevance": "Foundational Transformer architecture paper that this scaling study is built upon."
     },
     {
       "title": "Language models are unsupervised multitask learners",
       "authors": ["Alec Radford", "Jeff Wu", "Rewon Child", "David Luan", "Dario Amodei", "Ilya Sutskever"],
       "year": 2019,
       "relevance": "GPT-2 paper; provides the WebText dataset and Transformer implementation used in this study."
     },
     {
       "title": "BERT: Pre-training of deep bidirectional transformers for language understanding",
       "authors": ["Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova"],
       "year": 2018,
       "arxiv_id": "1810.04805",
       "relevance": "Major language model pretraining approach; context for understanding the scaling landscape."
     },
     {
       "title": "An empirical model of large-batch training",
       "authors": ["Sam McCandlish", "Jared Kaplan", "Dario Amodei"],
       "year": 2018,
       "arxiv_id": "1812.06162",
       "relevance": "Critical batch size theory used extensively in this paper's analysis of compute-efficient training."
     },
     {
       "title": "Deep learning scaling is predictable, empirically",
       "authors": ["Joel Hestness", "Sharan Narang", "Newsha Ardalani", "Gregory Diamos"],
       "year": 2017,
       "arxiv_id": "1712.00409",
       "relevance": "Prior work on scaling relationships between model size and data size; found super-linear scaling (contrasting this paper's sub-linear finding)."
     },
     {
       "title": "EfficientNet: Rethinking model scaling for convolutional neural networks",
       "authors": ["Mingxing Tan", "Quoc V. Le"],
       "year": 2019,
       "relevance": "Parallel scaling work in vision models; advocates different depth/width scaling than found here for language."
     },
     {
       "title": "A constructive prediction of the generalization error across scales",
       "authors": ["Jonathan S. Rosenfeld", "Amir Rosenfeld", "Yonatan Belinkov", "Nir Shavit"],
       "year": 2019,
       "arxiv_id": "1909.12673",
       "relevance": "Concurrent work making similar predictions for loss dependence on model and dataset size."
     },
     {
       "title": "Improving language understanding by generative pre-training",
       "authors": ["Alec Radford", "Karthik Narasimhan", "Tim Salimans", "Ilya Sutskever"],
       "year": 2018,
       "relevance": "GPT-1 paper; establishes the decoder-only Transformer approach used in this scaling study."
     }
   ]
 }