ai-research-survey

scan.json (21225B)

---

 {
   "paper": {
     "title": "Evaluating the Robustness of Chinchilla Compute-Optimal Scaling",
     "authors": ["Rylan Schaeffer", "Noam Levi", "Andreas Kirsch", "Theo Guenais", "Brando Miranda"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.23963",
     "doi": "10.48550/arXiv.2509.23963"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive URL is provided in the paper. The paper mentions using 'Besiroglu et al. (2024)'s Chinchilla fitting code' but does not release their own perturbation analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Chinchilla's publicly available Table A9 data (model parameters and architectural hyperparameters), which is fully reproduced in Appendix B (Table 2). All perturbation parameters are specified with exact sweep ranges."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methods are described at a mathematical level but there are no runnable scripts or README."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Error bars from 4000 bootstrapped samples are reported for scaling law parameters (Fig. 2, Fig. 4). 80% confidence intervals are shown for compute-optimal tokens-per-parameter ratios (Fig. 2, Fig. 5)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests are used to assess whether differences between parameter interpretations or perturbation effects are statistically significant. Comparisons rely on visual inspection of overlapping error bars."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports specific effect sizes: relative errors between parameter interpretations up to 15.2%, slopes of tokens-per-parameter ratio (-0.572, -1.049, -1.248 per decade), R² > 0.999 for power-law fit of ˆα under systematic bias."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The analysis uses 50 models from Chinchilla's Table A9. No justification is given for whether this is sufficient for the sensitivity analyses performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors from 4000 bootstrapped samples are reported throughout (Figs. 2, 4). For log-normal noise perturbation, variance across random noise realizations is shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The unperturbed Chinchilla results (reported model parameters, no perturbation) serve as the baseline against which all perturbations are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper engages with contemporary replication/re-evaluation studies: Besiroglu et al. (2024), Porian et al. (2024), Pearce & Song (2024), all recent work scrutinizing Chinchilla."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The four perturbation types (multiplicative, additive, systematic bias, log-normal noise) function as an ablation/sensitivity analysis, systematically varying one factor at a time."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated on multiple metrics: five scaling law parameters (Ê, Â, α̂, B̂, β̂) and the compute-optimal tokens-per-parameter ratio."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a mathematical/analytical study of scaling laws. Human evaluation is not relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a prediction task. The paper refits scaling laws to existing data under perturbations; there is no train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per perturbation type (Sections 3.1-3.4), per scaling parameter (Fig. 4), and per perturbation strength (Figs. 3-5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies cases where robustness breaks down: additive perturbations and systematic bias can 'qualitatively change the compute-optimal scaling strategy' (Section 3.2, 3.3). NaN results for extreme multiplicative constants are also noted."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that additive and systematic bias perturbations DO affect the trend of the optimal ratio (making it less constant), which is a negative finding for robustness. NaN results at extremes are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that key results are robust to parameter interpretation and structured perturbations, which is supported by Figs. 2, 4, 5. The abstract also notes sensitivity to additive/systematic errors, which is shown in Sections 3.2-3.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about how perturbations affect scaling law fits. These are justified by controlled single-variable manipulation (each perturbation type varied independently) and mathematical derivations in Appendix C."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to the specific Chinchilla dataset and scaling law formulation. The Future Directions section explicitly notes that extending to 'more recent scaling results with additional considerations' is future work."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative interpretations: the 'best fit formula' with factor 5 instead of 4 in attention parameters (Eqn. 3), and connects findings to Porian et al. and Pearce & Song's explanations for discrepancies between Chinchilla and Kaplan scaling laws."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It measures scaling law fit parameters and tokens-per-parameter ratios under perturbations, and claims robustness of these specific quantities — no broader framing gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use any LLM for experiments. It refits scaling laws using mathematical optimization on existing data."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a mathematical/analytical study."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "All perturbation sweep parameters are specified: multiplicative cm in logspace(-3, 3, num=11), additive ca in specified ranges, systematic bias s in logspace(-0.5, 0.5, 11), noise σ from 1e-2 to 1e2. Bootstrap uses 4000 samples."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data source is clearly documented: Chinchilla's Table A9 with 50 models. Three interpretations of model parameters are derived from architectural hyperparameters using Equations 1 and 3. The full table is reproduced in Appendix B."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion (Section 5) briefly mentions future directions but does not discuss limitations of the current analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address, e.g., whether the four perturbation types are exhaustive, whether the bootstrap methodology is appropriate for this data structure, or whether using Besiroglu et al.'s fitting code introduces assumptions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Future Directions paragraph explicitly states scope boundaries: 'One obvious next step is to evaluate the robustness of more recent scaling results with additional considerations such as inference constraints, data constraints and overtraining.' This acknowledges the analysis is limited to the original Chinchilla formulation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The raw data (Chinchilla's Table A9) is fully reproduced in Appendix B (Table 2), including all 50 models' architectural hyperparameters and three interpretations of model parameters."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data comes from a single, clearly identified source: Hoffmann et al. (2022)'s Table A9. No new data was collected."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is from a published paper's model specifications."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is transparent: take Table A9 data → compute three parameter interpretations using Eqns. 1 and 3 → apply perturbations using Eqns. 6-9 → refit using Besiroglu et al.'s code. Each step is documented with equations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. Authors are from Stanford University and EPFL but no grants or sponsorship are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Stanford University and EPFL. The paper evaluates Chinchilla (DeepMind), and no authors are affiliated with DeepMind."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. It refits scaling laws to published data."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluation on benchmarks is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluation on benchmarks is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/analytical paper. No inference costs are relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/analytical paper refitting scaling laws. Compute is negligible."
       }
     }
   },
   "claims": [
     {
       "claim": "Three interpretations of Chinchilla's model parameters are possible, with relative differences as high as 15.2%.",
       "evidence": "Table 1 and Fig. 1 show that the standard formula disagrees with reported parameters for 50/50 models, with average relative error of 7.4% and max 15.2%. The 'best fit' formula reduces discrepancies to 6/50 models.",
       "supported": "strong"
     },
     {
       "claim": "Key Chinchilla results (scaling law parameters and ~20:1 tokens-per-parameter ratio) are robust to all three parameter interpretations.",
       "evidence": "Fig. 2 shows the five fit parameters do not meaningfully differ across interpretations, and the compute-optimal ratio remains ~20 tokens per parameter. Standard formula yields the flattest slope (-0.572 per decade).",
       "supported": "strong"
     },
     {
       "claim": "Multiplicative perturbations shift the tokens-per-parameter ratio but preserve the flat trend with compute budget.",
       "evidence": "Fig. 4 Row 1 and Fig. 5 Top Left show ˆA scales as A·cm^α while ˆα remains stable. Theoretical derivation in Appendix C.2.1 confirms this.",
       "supported": "strong"
     },
     {
       "claim": "Additive perturbations and systematic bias can qualitatively change the compute-optimal scaling strategy.",
       "evidence": "Fig. 5 Top Right and Bottom Left show the tokens-per-parameter ratio becomes less constant, with positive/negative slopes depending on perturbation direction. Mathematical derivations in Appendix C.2.2-C.2.3 explain the mechanism.",
       "supported": "strong"
     },
     {
       "claim": "Overall, Chinchilla's key results withstand sizable perturbations, providing renewed confidence in compute-optimal scaling.",
       "evidence": "Across all four perturbation types, the 20:1 ratio and scaling law form are preserved for moderate perturbations. Only extreme additive/systematic perturbations alter the qualitative picture.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper discovers that three possible interpretations of Chinchilla's model parameters exist with discrepancies up to 15.2%, but none meaningfully affect the scaling law estimates or the ~20:1 compute-optimal tokens-per-parameter ratio. Through four structured perturbation analyses (multiplicative, additive, systematic bias, log-normal noise), they show that multiplicative errors and random noise preserve robustness while additive constants and systematic biases can alter the trend of the optimal ratio with compute budget. The work provides mathematical derivations explaining each perturbation's effects and concludes that Chinchilla's compute-optimal prescription remains a durable guide.",
   "red_flags": [
     {
       "flag": "Confirmatory framing",
       "detail": "The paper's stated goal is to provide 'renewed confidence' in Chinchilla, and the framing consistently emphasizes robustness. The finding that additive and systematic perturbations DO qualitatively change results is somewhat downplayed relative to the robustness narrative. The abstract says 'the answer is yes' to whether practitioners can rely on Chinchilla, but the sensitivity to additive perturbations (which correspond to the realistic scenario of including/excluding embedding parameters) suggests the answer is more nuanced."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations: whether the four perturbation types cover the space of realistic errors, whether the 50 Chinchilla models provide sufficient statistical power, or whether the bootstrap methodology is appropriate given the structured (non-i.i.d.) nature of the model configurations."
     }
   ],
   "cited_papers": [
     {
       "title": "An empirical analysis of compute-optimal large language model training",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "The foundational Chinchilla paper whose robustness is the subject of this study."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Earlier scaling law work whose discrepancies with Chinchilla motivated this robustness analysis."
     },
     {
       "title": "Chinchilla scaling: A replication attempt",
       "authors": ["Tamay Besiroglu", "Ege Erdil", "Matthew Barnett", "Josh You"],
       "year": 2024,
       "arxiv_id": "2404.10102",
       "relevance": "Key replication study that provided the fitting code used in this paper and identified inconsistencies in Chinchilla's three approaches."
     },
     {
       "title": "Resolving discrepancies in compute-optimal scaling of language models",
       "authors": ["Tomer Porian", "Mitchell Wortsman", "Jenia Jitsev"],
       "year": 2024,
       "relevance": "Identified three factors resolving Chinchilla-Kaplan discrepancies; this paper's additive perturbation analysis relates to their finding about head parameters."
     },
     {
       "title": "Reconciling Kaplan and Chinchilla scaling laws",
       "authors": ["Tim Pearce", "Jinyeop Song"],
       "year": 2024,
       "relevance": "Found embedding parameter inclusion explains much of the Chinchilla-Kaplan discrepancy, directly related to this paper's additive perturbation analysis."
     },
     {
       "title": "Beyond chinchilla-optimal: Accounting for inference in language model scaling laws",
       "authors": ["Nikhil Sardana", "Jacob Portes", "Sasha Doubov", "Jonathan Frankle"],
       "year": 2024,
       "relevance": "Extends Chinchilla scaling to account for inference costs, identified as a future direction in this paper."
     },
     {
       "title": "Scaling data-constrained language models",
       "authors": ["Niklas Muennighoff"],
       "year": 2023,
       "relevance": "Studied scaling under data constraints, another future direction identified in this paper."
     },
     {
       "title": "Language models scale reliably with over-training and on downstream tasks",
       "authors": ["Samir Yitzhak Gadre"],
       "year": 2024,
       "arxiv_id": "2403.08540",
       "relevance": "Studied scaling with overtraining, mentioned as a future extension of robustness analysis."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Prior work by the same authors on scaling phenomena, relevant to understanding LLM capability scaling."
     },
     {
       "title": "Scaling laws for precision",
       "authors": ["Tanishq Kumar"],
       "year": 2024,
       "arxiv_id": "2411.04330",
       "relevance": "Studies how quantization precision interacts with scaling laws, extending the Chinchilla framework."
     }
   ]
 }