ai-research-survey

scan.json (27882B)

---

 {
   "paper": {
     "title": "Training Compute-Optimal Large Language Models",
     "authors": [
       "Jordan Hoffmann",
       "Sebastian Borgeaud",
       "Arthur Mensch",
       "Elena Buchatskaya",
       "Trevor Cai",
       "Eliza Rutherford",
       "Diego de Las Casas",
       "Lisa Anne Hendricks",
       "Johannes Welbl",
       "Aidan Clark",
       "Tom Hennigan",
       "Eric Noland",
       "Katie Millican",
       "George van den Driessche",
       "Bogdan Damoc",
       "Aurelia Guy",
       "Simon Osindero",
       "Karen Simonyan",
       "Erich Elsen",
       "Jack W. Rae",
       "Oriol Vinyals",
       "Laurent Sifre"
     ],
     "year": 2022,
     "venue": "NeurIPS 2022",
     "arxiv_id": "2203.15556"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository or link is provided anywhere in the paper. The paper describes training procedures in detail but does not release any code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training data (MassiveText) is not publicly released. The model card states 'We will not make this model available publicly.' Standard evaluation benchmarks (MMLU, The Pile, etc.) are public, but the authors' own dataset and training curves data are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on TPUv3/TPUv4 with JAX and Haiku but provides no requirements file, library versions, or detailed environment specifications beyond these high-level mentions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While many hyperparameters and architectural details are given (Table 4, Appendix D, F), there are no README-style instructions or scripts for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports 10th and 90th percentiles via bootstrapping for the scaling exponents (e.g., a=0.50 (0.488, 0.502), b=0.50 (0.501, 0.512) for Approach 1), providing uncertainty estimates for the key scaling law parameters."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Chinchilla 'significantly outperforms' Gopher and other models on many tasks but provides no statistical significance tests. Comparisons are based on point estimates of accuracy/loss without any formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context throughout: e.g., Chinchilla achieves 67.6% on MMLU vs 60.0% for Gopher (7.6% improvement, Table 6), 10.7% average improvement on BIG-bench (65.1% vs 54.4%), and specific per-task comparisons with baselines in all evaluation tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper trains over 400 models for the scaling analysis but does not justify why this number was chosen or discuss whether it is sufficient. The number of IsoFLOP profiles (9) or model sizes is not justified by any formal analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation across runs is reported. All benchmark results (MMLU, BIG-bench, question answering, etc.) appear to be from single runs. The bootstrapping in Table 2 addresses parameter estimation uncertainty but not run-to-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: Gopher (280B), GPT-3 (175B), Jurassic-1 (178B), MT-NLG 530B (530B), along with supervised SOTA and human baselines on multiple tasks (Tables 6-9)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines were contemporary at time of publication: Gopher (2021), GPT-3 (2020), Jurassic-1 (2021), MT-NLG 530B (2022). These represented the largest and most capable dense LLMs available."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G ablates the training differences between Chinchilla and Gopher: Adam vs AdamW optimizer (Figure A7), high-precision optimizer state, and their combination (Figure A6). These isolate the contribution of each training change."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses numerous evaluation metrics: perplexity, bits-per-byte on The Pile (20 subsets), accuracy on MMLU (57 tasks), accuracy on BIG-bench (62 tasks), reading comprehension accuracy, closed-book QA accuracy, common sense benchmarks, gender bias metrics, and toxicity scores."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates scaling laws for language model training and compares model performance on automated benchmarks. Human evaluation of model outputs is not relevant to the core claims about compute-optimal training."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard held-out test/dev sets for each benchmark (e.g., Natural Questions dev set, TriviaQA test/dev sets as stated in Table 9, MMLU 5-shot evaluation). The paper also notes train/test leakage concerns for language modelling benchmarks (Section 4.2.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: per-subset Pile evaluation (Figure 5, Table A5), per-task MMLU results (Figure 6, Table A6), per-task BIG-bench results (Figure 7, Table A7), and per-benchmark common sense results (Table 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly identifies tasks where Chinchilla underperforms Gopher: 4 MMLU tasks (college_mathematics, econometrics, moral_scenarios, formal_logic) in Section 4.2.2 and 4 BIG-bench tasks (crash_blossom, dark_humor_detection, mathematical_induction, logical_args) in Section 4.2.4. Gender bias disparities are also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that toxicity levels are largely unchanged between Chinchilla and Gopher despite improved performance (Section 4.2.7). It also reports uneven gender bias improvements and notes that compute-optimal training does not uniformly improve all capabilities."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) current LLMs are under-trained, (2) model size and tokens should scale equally, (3) Chinchilla outperforms Gopher/GPT-3/Jurassic-1/MT-NLG on downstream tasks, (4) 67.5% MMLU accuracy are all supported by results in Sections 3-4. The actual MMLU result is 67.6% (Table 6), slightly higher than the 67.5% stated in the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core causal claim is that the scaling law relationship determines optimal compute allocation. This is supported by three independent estimation approaches (Section 3) and validated by the Chinchilla training run. The ablation in Appendix G isolates optimizer changes from the scaling hypothesis. The paper appropriately notes confounds between Chinchilla and Gopher (slightly different data distribution, optimizer, tokenizer)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper states its limitations clearly: 'we only have two comparable training runs at large scale (Chinchilla and Gopher), and we do not have additional tests at intermediate scales' (Section 5). It notes the power-law assumption may not hold at very large scales, and acknowledges the analysis is on less than one epoch of data. The title 'Training Compute-Optimal Large Language Models' is reasonably bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses that the concavity in the FLOP-loss frontier may mean even smaller models are optimal. The paper also acknowledges in Section 4.2.1 that train/test leakage may artificially enhance language modelling results since Chinchilla saw 4x more data. Appendix G addresses whether performance gains come from optimizer changes rather than scaling."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "All models are trained from scratch by the authors, with exact architectures specified in Table 4 (layers, heads, key/value size, d_model, learning rate, batch size). Model sizes are precisely stated (70B for Chinchilla, 280B for Gopher). This is not an API-based study where version snapshots matter."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates on various benchmarks using few-shot prompting but does not provide the actual prompt text used for evaluations. It references the evaluation setup in Rae et al. (2021) but does not reproduce the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters are reported: learning rates (Table 4, Appendix D.1), batch sizes (Table 4), optimizer choice (AdamW), learning rate schedule (cosine with 10x decay), precision (bfloat16 forward/backward, float32 optimizer state), and architecture details. Appendix D provides additional details for the scaling runs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard language model training and evaluation study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table A1 documents the MassiveText dataset composition including disk size, document counts, sampling proportions, and epochs for 1.4T tokens. The tokenizer change (SentencePiece without NFKC normalization) is documented in Section 4.1. The paper references Rae et al. (2021) for the full MassiveText data pipeline details."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion & Conclusion) contains substantive discussion of limitations spanning two paragraphs, covering limited large-scale validation runs, power-law assumption concerns, single-epoch limitation, and ethical concerns about large dataset collection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: (1) only two comparable large-scale runs (Chinchilla vs Gopher), (2) concavity in the FLOP-loss frontier suggesting power-law may not hold at large scales (Appendix E), (3) potential train/test leakage from 4x more training data (Section 4.2.1), (4) confounding factors between Chinchilla and Gopher beyond just scaling (different optimizer, tokenizer, data distribution)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: the analysis covers only dense transformer models (not MoE), only single-epoch training regimes, and only autoregressive language models. Section 5 states 'we assume that the efficient computational frontier can be described by a power-law relationship' and notes this may not hold. The model card states the model is not publicly available and is for research only."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the raw training data (MassiveText) nor the individual training curves from the 400+ model runs are released. Only aggregated results and fitted scaling law parameters are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The training data composition is described in Table A1 (MassiveText subsets: MassiveWeb, Books, C4, News, GitHub, Wikipedia with sizes and proportions). The paper references Rae et al. (2021) for full data collection details. The scaling experiment setup (model sizes, training horizons, FLOP budgets) is described in Section 3 and Appendix D."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study trains and evaluates language models on existing datasets and benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from training runs to scaling law estimates is well documented: Section 3.1 describes how training curves are smoothed and interpolated, how the envelope of minimal loss per FLOP is extracted, and how power laws are fitted. Section 3.3 and Appendix D.2 detail the parametric fitting procedure including the Huber loss, L-BFGS optimization, and initialization grid."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding statement appears in the paper. The copyright line states '© 2023 DeepMind. All rights reserved' but no funding sources or grants are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are identified as DeepMind researchers via the corresponding author email addresses ({jordanhoffmann|sborgeaud|amensch|sifre}@deepmind.com) and the DeepMind copyright notice."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepMind (a Google/Alphabet subsidiary) funded and conducted this research. DeepMind has a direct financial interest in demonstrating effective approaches to LLM training, and the results validate their research direction (showing that their 280B Gopher was over-sized and that a 70B model trained optimally performs better)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. DeepMind employees may hold equity in Alphabet, which is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff date for MassiveText. It references the dataset from Rae et al. (2021) but does not specify when the data was collected or what temporal range it covers."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 explicitly discusses train/test overlap: 'Some caution is needed when comparing Chinchilla with Gopher on these language modelling benchmarks as Chinchilla is trained on 4× more data than Gopher and thus train/test set leakage may artificially enhance the results.' The paper shifts emphasis to benchmarks where leakage is less of a concern (MMLU, BIG-bench)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges contamination risk for language modelling benchmarks (Section 4.2.1) and explicitly notes that MMLU and BIG-bench are less susceptible to leakage. Section 5 notes 'Larger datasets will require extra care to ensure train-test set overlap is properly accounted for, both in the language modelling loss but also with downstream tasks.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper qualitatively notes that Chinchilla's smaller size 'reduces inference cost considerably' (Section 4) but does not quantify inference cost, latency, or tokens per second for any model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The compute budget is central to the paper. The Gopher compute budget is stated as 5.76×10^23 FLOPs (Table 3), and Chinchilla uses the same budget. Appendix F details the exact FLOP calculation methodology. Table 3 provides compute budgets for various model sizes. Training was done on TPUv3/TPUv4 (Section 4.1)."
       }
     }
   },
   "claims": [
     {
       "claim": "Current large language models are significantly under-trained: for compute-optimal training, model size and number of training tokens should be scaled equally (for every doubling of model size, training tokens should also be doubled).",
       "evidence": "Three independent approaches (Sections 3.1-3.3) all yield scaling exponents near a=0.5, b=0.5 (Table 2), compared to Kaplan et al. (2020) which predicted a=0.73, b=0.27. Over 400 model training runs support this finding. Bootstrap confidence intervals are provided.",
       "supported": "strong"
     },
     {
       "claim": "Chinchilla (70B parameters, 1.4T tokens) uniformly and significantly outperforms Gopher (280B) on a large range of downstream evaluation tasks despite using the same compute budget.",
       "evidence": "Chinchilla outperforms Gopher on all 20 Pile subsets (Figure 5), 51/57 MMLU tasks (Figure 6), 58/62 BIG-bench tasks (Figure 7), all reading comprehension tasks (Table 7), all question answering tasks (Table 9), and 4/5 common sense benchmarks (Table 8). The average MMLU improvement is 7.6% (67.6% vs 60.0%).",
       "supported": "strong"
     },
     {
       "claim": "Chinchilla achieves state-of-the-art average accuracy of 67.6% on the MMLU benchmark, exceeding the June 2023 expert forecast of 63.4%.",
       "evidence": "Table 6 shows Chinchilla 5-shot accuracy of 67.6% vs Gopher 60.0%, GPT-3 43.9%, and the June 2023 forecast of 63.4% from Steinhardt (2021). Per-task breakdown in Table A6 and Figure 6.",
       "supported": "strong"
     },
     {
       "claim": "Chinchilla also significantly outperforms GPT-3 (175B) and Megatron-Turing NLG (530B) on most evaluated tasks.",
       "evidence": "Tables 7-9 show Chinchilla outperforming GPT-3 on LAMBADA, RACE, Natural Questions, and TriviaQA. Table 8 shows Chinchilla outperforming MT-NLG 530B on 4/5 common sense benchmarks despite being 7.5x smaller.",
       "supported": "strong"
     },
     {
       "claim": "The scaling results are consistent across different datasets (MassiveText, C4, GitHub code).",
       "evidence": "Appendix C and Table A2 show IsoFLOP analysis on C4 (a=0.50, b=0.50) and GitHub (a=0.53, b=0.47), both very similar to the MassiveText results, confirming equal scaling of parameters and tokens.",
       "supported": "strong"
     },
     {
       "claim": "Toxicity levels in unconditional text generation are largely independent of model quality, and Chinchilla's improved loss does not increase toxicity.",
       "evidence": "Section 4.2.7 reports nearly identical toxicity scores: Gopher mean 0.081 (median 0.064) vs Chinchilla mean 0.087 (median 0.066), with 95th percentile scores of 0.230 vs 0.238 based on 25,000 unprompted samples evaluated with PerspectiveAPI.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "The paper establishes that for compute-optimal training of large language models, model size and number of training tokens should be scaled in approximately equal proportions, contradicting the prior recommendation from Kaplan et al. (2020) that favored scaling model size much faster than data. Three independent estimation approaches using over 400 training runs all converge on this finding. The practical validation, Chinchilla (70B parameters trained on 1.4T tokens), outperforms the 4x larger Gopher (280B) and other larger models on nearly every benchmark tested, achieving 67.6% on MMLU — a 7.6% improvement over Gopher.",
   "red_flags": [
     {
       "flag": "Company evaluating its own models",
       "detail": "All authors are DeepMind researchers comparing their new model (Chinchilla) against their previous model (Gopher). While the paper also compares against external models (GPT-3, Jurassic-1, MT-NLG), the primary comparison is internal. No conflicts of interest statement is provided."
     },
     {
       "flag": "Confounding factors in Chinchilla vs Gopher comparison",
       "detail": "Chinchilla differs from Gopher in multiple ways beyond the scaling hypothesis: different optimizer (AdamW vs Adam), different tokenizer (no NFKC normalization), different data subset distribution, and higher precision optimizer state. While Appendix G attempts to ablate optimizer effects, the final Chinchilla comparison bundles all changes together. The paper acknowledges this but the main results conflate these factors."
     },
     {
       "flag": "No run-to-run variance reported",
       "detail": "All benchmark results appear to be from single runs with no reported variance or standard deviation. Given the stochastic nature of language model training and evaluation, the lack of repeated runs makes it impossible to assess the stability of the reported improvements."
     },
     {
       "flag": "Training data not released",
       "detail": "MassiveText is proprietary to DeepMind and not publicly available. The 400+ training runs underlying the scaling law analysis cannot be independently verified or reproduced. The raw loss curves are not released."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["J. Kaplan", "S. McCandlish", "T. Henighan", "T. B. Brown", "B. Chess", "R. Child", "S. Gray", "A. Radford", "J. Wu", "D. Amodei"],
       "year": 2020,
       "relevance": "The primary work this paper extends and corrects — established the original scaling laws that recommended scaling model size faster than data, which Chinchilla refutes."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper — one of the key baselines in the evaluation and an example of a model trained with the suboptimal scaling recommended by Kaplan et al."
     },
     {
       "title": "Scaling language models: Methods, analysis & insights from training Gopher",
       "authors": ["J. Rae", "S. Borgeaud", "T. Cai"],
       "year": 2021,
       "relevance": "The Gopher paper — the primary baseline model and training setup that Chinchilla builds upon and improves over."
     },
     {
       "title": "Using Deepspeed and Megatron to Train Megatron-turing NLG 530b, A Large-Scale Generative Language Model",
       "authors": ["S. Smith", "M. Patwary", "B. Norick"],
       "year": 2022,
       "relevance": "MT-NLG 530B — the largest dense model at time of publication, a key baseline that Chinchilla (70B) outperforms on most tasks despite being 7.5x smaller."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["D. Hendrycks", "C. Burns", "S. Basart", "A. Zou", "M. Mazeika", "D. Song", "J. Steinhardt"],
       "year": 2020,
       "relevance": "The MMLU benchmark used as a primary evaluation metric — Chinchilla achieves SOTA 67.6% on this benchmark."
     },
     {
       "title": "Unified scaling laws for routed language models",
       "authors": ["A. Clark", "D. de Las Casas", "A. Guy"],
       "year": 2022,
       "relevance": "Examines scaling laws for Mixture of Expert models, complementing Chinchilla's analysis of dense models."
     },
     {
       "title": "Improving language models by retrieving from trillions of tokens",
       "authors": ["S. Borgeaud", "A. Mensch", "J. Hoffmann"],
       "year": 2021,
       "relevance": "RETRO paper — shows that retrieval augmentation effectively increases training data seen, supporting the hypothesis that data quantity is crucial for model performance."
     },
     {
       "title": "On the dangers of stochastic parrots: Can language models be too big?",
       "authors": ["E. M. Bender", "T. Gebru", "A. McMillan-Major", "S. Shmitchell"],
       "year": 2021,
       "relevance": "Discusses risks of large language models including environmental cost and societal biases, directly relevant to the efficiency argument that compute-optimal models reduce waste."
     },
     {
       "title": "Ethical and social risks of harm from language models",
       "authors": ["L. Weidinger", "J. Mellor", "M. Rauh"],
       "year": 2021,
       "relevance": "Framework for evaluating LLM risks; Chinchilla's bias and toxicity evaluations build on this work."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["S. Lin", "J. Hilton", "O. Evans"],
       "year": 2021,
       "relevance": "Chinchilla shows large improvements on TruthfulQA (14.1% in 0-shot) over Gopher, contradicting the original finding that larger models perform worse on this benchmark."
     },
     {
       "title": "GLaM: Efficient scaling of language models with mixture-of-experts",
       "authors": ["N. Du", "Y. Huang", "A. M. Dai"],
       "year": 2021,
       "relevance": "1.2T parameter MoE model — represents the alternative approach of sparse scaling that complements Chinchilla's dense model efficiency findings."
     }
   ]
 }