ai-research-survey

scan.json (25776B)

---

 {
   "paper": {
     "title": "Broken Neural Scaling Laws",
     "authors": ["Ethan Caballero", "Kshitij Gupta", "Irina Rish", "David Krueger"],
     "year": 2022,
     "venue": "ICLR 2023",
     "arxiv_id": "2210.14891"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at github.com/ethancaballero/broken_neural_scaling_laws' providing a direct repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available scaling laws benchmark dataset of Alabdulmohsin et al. (2022) for its main vision and language experiments, and the 4-digit addition task uses the authors' own code to generate data. Additional experimental data points were obtained from published papers (GPT-3, etc.)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions SciPy for curve fitting and minGPT for the arithmetic experiments, but does not provide a reproducible environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the code repository is referenced and hyperparameters for the 4-digit addition task are given in Table 5 and Section A.5, there are no step-by-step reproduction instructions in the paper itself. The paper does not include a 'Reproducing Results' section or README-level guidance for replicating the main curve-fitting experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "All extrapolation results in Tables 3 and 4 are reported as RMSLE ± root standard log error, providing uncertainty quantification for each result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims BNSL outperforms other functional forms (M1-M4) based on comparing RMSLE values in Tables 2-4, but no statistical significance tests (e.g., paired t-tests, Wilcoxon) are used to determine if the differences are statistically significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the percentage of tasks where each functional form is best (Table 2: BNSL best on 69.44% of vision tasks and 75% of language tasks), and provides raw RMSLE values allowing comparison of the magnitude of improvement across methods."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a large set of tasks from the Alabdulmohsin et al. (2022) benchmark and many additional tasks, but does not justify why this particular set of tasks or the number of data points per task is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports root standard log error alongside RMSLE in all tables (Tables 3, 4, 6). For the 4-digit addition experiments, each point is 'mean of greater than 1000 seeds at that dataset size' (Section 5.5, Figure 5 caption)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares BNSL against four baseline functional forms: M1 (power law), M2 (power law + constant), M3 (shifted power law + constant), and M4 (Alabdulmohsin et al. 2022's form). Results are presented in Tables 2, 3, 4, and 6."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include M4 from Alabdulmohsin et al. (2022), which was the most recent scaling law functional form at the time. M3 from Zhai et al. (2021) and Bansal et al. (2022) are also contemporary. The comparison was done using the benchmark from Alabdulmohsin et al. (2022)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include a formal ablation study of the BNSL functional form (e.g., systematically varying the number of breaks, the sharpness parameters, or other components). Section 6 discusses the impact of break sharpness on predictability but this is more of an analysis than a controlled ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary metric used throughout is RMSLE (Root Mean Squared Logarithmic Error) for evaluating extrapolation quality. While the paper evaluates across many tasks with different performance metrics (error rate, cross-entropy, FID, etc.), the comparison between functional forms uses only RMSLE."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a curve-fitting methodology paper. Human evaluation is not relevant to the claims about which functional form best extrapolates scaling behavior."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a clear train/test split for extrapolation evaluation: 'black points are points used for fitting a functional form, green (gray if color blind) points are held-out points used for evaluating extrapolation' (Section 5). This held-out evaluation is used consistently across all experiments."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 provide per-task breakdowns for every individual task/model combination. Table 2 provides per-domain summaries. The appendix provides individual plots for each task."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses the limits of BNSL's predictability, noting that extrapolation fails when fitting points are before a sufficiently sharp break. Figure 5 right shows the noiseless simulation revealing the fundamental limitation. Figures in Appendix A.36 (e.g., Caltech101, BIG-Bench) acknowledge 'unsatisfactory extrapolations' for certain tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly reports the fundamental limitation that BNSL cannot extrapolate past sharp breaks it hasn't observed. The captions for Figures 37 and 36 acknowledge subsets with 'unsatisfactory extrapolations'. The paper also notes BNSL is not always the best (Table 2 shows it is best on 69-75% of tasks, not all)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims BNSL 'accurately models and extrapolates the scaling behaviors' across a diverse set of tasks. Tables 2-4 and extensive appendix results support this. The abstract's claim about non-monotonic behaviors (double descent) is supported by Figure 4, and inflection points by Figure 5. The claim about being 'considerably more accurate' is supported by the 69-75% best-rate in Table 2."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It proposes a functional form and empirically evaluates its curve-fitting and extrapolation accuracy compared to alternatives. The language is descriptive ('accurately models', 'yields extrapolations') rather than causal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly bounds the generalization by identifying limitations: the need for data points near or past breaks for accurate extrapolation, the inability to predict unobserved sharp breaks, and the dependence on noise/number of seeds. The paper is careful to state results per-domain and per-task."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses why extrapolation may fail (insufficient points near breaks, noise masking breaks). The captions for Figures 36-37 offer alternative explanations for unsatisfactory extrapolations (fitting points near or before a break, not enough points for the fitter to distinguish breaks from noise). The ethics statement discusses whether BNSL truly reflects the underlying scaling behavior or is simply a flexible curve fit."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "For their own experiments (4-digit addition), exact architecture details are given in Table 5. For other experiments, they reference specific published results (e.g., GPT-3 Table H.1, specific figures from cited papers) with exact model specifications (e.g., '2.62e+8 Param', '1.07e+9 Param', specific BiT/ViT/MiX model variants)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting in its methodology. It fits mathematical functional forms to empirical scaling data using curve-fitting libraries (SciPy)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides a comprehensive hyperparameter table for the 4-digit addition experiments (learning rate, weight decay, dropout, optimizer settings, etc.). Section A.6 describes the curve-fitting procedure including the use of scipy.optimize.brute and scipy.optimize.curve_fit."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The methodology involves curve fitting with SciPy, not agentic AI systems."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.6 describes the fitting procedure (grid search initialization, non-linear least squares). For the 4-digit addition task, Section A.5 specifies dataset sizes (144-1008), how data was generated, and all experimental settings. For the benchmark data, the source is explicitly cited (Alabdulmohsin et al. 2022 benchmark, with specific figures/tables from other papers)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('The Limit of the Predictability of Scaling Behavior') is a substantial discussion of the fundamental limitations of the approach. The Ethics Statement also discusses a limitation about the need for sufficient data points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides specific threats: (1) sharp breaks beyond observed data cannot be predicted, (2) points before sufficiently sharp breaks can be useless for extrapolation, (3) noise across seeds limits accuracy. These are specific to this study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states three implications that bound the scope: what happens when breaks are sufficiently sharp, when additional breaks occur beyond observed scales, and when fitting data is beyond a sharp break. The ethics statement notes the compute-access limitation for gathering sufficient data points."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The experimental data comes from publicly available sources: the scaling laws benchmark of Alabdulmohsin et al. (2022), published papers (GPT-3 Table H.1, etc.), and the code repository github.com/ethancaballero/broken_neural_scaling_laws. Data for the 4-digit addition task is generated by the provided code."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "For each experiment, the paper states exactly where the data was obtained: specific figures and tables from cited papers (e.g., 'Experimental data obtained from Figure 1 of Hernandez et al. (2021)'). For the 4-digit addition task, Section A.5 describes the experimental setup in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data consists of empirical scaling measurements from neural network training runs, sourced from published papers and public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.6 documents the full pipeline: collecting scaling data from sources, fitting BNSL via scipy.optimize.brute grid search followed by scipy.optimize.curve_fit least squares, using a numerically stable MSLE variant. The train/test split for extrapolation evaluation is clearly described in Section 5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments section thanks various individuals for feedback and assistance but does not disclose any funding sources, grants, or financial support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Mila + McGill University (Caballero), Mila + University of Montreal (Gupta, Rish), and University of Cambridge (Krueger). These are academic institutions with no obvious conflict regarding the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so the independence of any funder cannot be assessed. The absence of funding disclosure does not confirm independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It fits mathematical functional forms to existing empirical scaling data from published papers. The only novel model training is for the 4-digit addition task, which is a from-scratch training experiment, not evaluation of a pre-trained model."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper's methodology is curve fitting, not model evaluation on benchmarks. There is no concern about train/test overlap in the context of fitting scaling laws to published experimental data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not applicable here. The paper fits functional forms to scaling data points, not evaluating a language model on a knowledge benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or wall-clock time is reported for running the BNSL curve fitting procedure. Section A.5 mentions 'each run took less than 2 hours' for the 4-digit addition training experiments, but not for the curve-fitting methodology itself."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section A.5 states each 4-digit addition experiment 'was run on a single V100 GPU and each run took less than 2 hours', but the total computational budget across all experiments and curve-fitting runs is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "BNSL yields extrapolations with the lowest RMSLE for 69.44% of downstream image classification tasks compared to four alternative functional forms.",
       "evidence": "Table 2 and Table 3 in Section 5.1, comparing BNSL to M1, M2, M3, M4 on the scaling laws benchmark of Alabdulmohsin et al. (2022).",
       "supported": "strong"
     },
     {
       "claim": "BNSL yields extrapolations with the lowest RMSLE for 75% of language tasks compared to four alternative functional forms.",
       "evidence": "Table 2 and Table 4 in Section 5.2, comparing BNSL to M1, M2, M3, M4 on language tasks from the BIG-Bench benchmark and language modeling tasks.",
       "supported": "strong"
     },
     {
       "claim": "Previously proposed scaling law functional forms (M1, M2, M3) are mathematically incapable of expressing non-monotonic behavior or inflection points on linear-linear plots.",
       "evidence": "Section 4 provides mathematical proofs via first and second derivative analysis in Table 1, showing these forms are strictly monotonic and lack inflection points over the relevant domain.",
       "supported": "strong"
     },
     {
       "claim": "BNSL accurately models and extrapolates non-monotonic scaling behaviors such as double descent.",
       "evidence": "Figure 4 in Section 5.4 shows BNSL fitting and extrapolating double descent behavior for transformers trained on neural machine translation, using data from Nakkiran et al. (2021).",
       "supported": "strong"
     },
     {
       "claim": "BNSL accurately extrapolates to scales over an order of magnitude larger than the maximum of the fitting points.",
       "evidence": "Figure 2 and Section A.8/A.15 show extrapolation to scales >10x larger for both vision and language tasks. Section A.35 shows extrapolation to scales >100,000x larger using GPT-4 data.",
       "supported": "moderate"
     },
     {
       "claim": "There is a fundamental limit to the predictability of scaling behavior: when breaks are sufficiently sharp, data points near or beyond the break are needed for accurate extrapolation.",
       "evidence": "Section 6 demonstrates this using the 4-digit addition task (Figure 5), showing that fitting requires points with dataset size >= 720 (real data) or >= 415 (noiseless simulation, near the break point).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper proposes a smoothly broken power law functional form (BNSL) that generalizes standard power laws by introducing smooth transitions ('breaks') between power law segments. BNSL achieves the best extrapolation accuracy (by RMSLE) on 69-75% of tasks from a large scaling laws benchmark spanning vision, language, reinforcement learning, and many other domains. Unlike prior functional forms (M1-M4), BNSL can mathematically express non-monotonic behaviors (double descent) and inflection points, and the paper demonstrates accurate modeling and extrapolation of these phenomena. The paper also identifies a fundamental predictability limit: extrapolation past sufficiently sharp, unobserved breaks is impossible regardless of the number of seeds.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims BNSL is 'considerably more accurate' than alternatives based on comparing RMSLE values across tasks, but never applies statistical significance tests (e.g., paired Wilcoxon or bootstrap tests across tasks) to determine if the differences are statistically reliable."
     },
     {
       "flag": "Overfitting risk from flexible functional form",
       "detail": "BNSL has more free parameters than the baseline forms (M1-M4) due to additional break parameters. While the held-out extrapolation evaluation mitigates this concern, the paper does not systematically analyze whether BNSL's advantage comes from genuine modeling ability versus having more parameters (e.g., via information criteria like BIC/AIC or cross-validation)."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B. Brown"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational work on neural scaling laws that BNSL generalizes; directly relevant to understanding how LLM performance scales with compute, data, and parameters."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper whose published scaling data (Table H.1) is used as experimental data for BNSL extrapolation evaluation on downstream language tasks."
     },
     {
       "title": "Revisiting Neural Scaling Laws in Language and Vision",
       "authors": ["Ibrahim Mansour I Alabdulmohsin", "Behnam Neyshabur", "Xiaohua Zhai"],
       "year": 2022,
       "arxiv_id": "2209.06640",
       "relevance": "Proposed M4 functional form and the scaling laws benchmark dataset used extensively in this paper's evaluation."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Discusses 'emergent' abilities of LLMs that BNSL aims to model and predict; directly relevant to AI capability forecasting."
     },
     {
       "title": "Beyond the Imitation Game: Quantifying and Extrapolating the Capabilities of Language Models",
       "authors": ["Aarohi Srivastava", "Abhinav Rastogi"],
       "year": 2022,
       "arxiv_id": "2206.04615",
       "relevance": "BIG-Bench benchmark used for evaluation of language task scaling; key benchmark for measuring LLM capabilities."
     },
     {
       "title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback",
       "authors": ["Yuntao Bai", "Andy Jones"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "RLHF training data used to demonstrate BNSL extrapolation of AI alignment scaling behavior; directly relevant to AI safety scaling."
     },
     {
       "title": "Deep Double Descent: Where Bigger Models and More Data Hurt",
       "authors": ["Preetum Nakkiran", "Gal Kaplun", "Yamini Bansal"],
       "year": 2021,
       "relevance": "Source of experimental data for the double descent phenomenon that BNSL can model but prior scaling laws cannot."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Source of scaling data showing BNSL can extrapolate downstream performance to scales >100,000x larger than fitting data; relevant to LLM capability prediction."
     },
     {
       "title": "Scaling Laws for Transfer",
       "authors": ["Danny Hernandez", "Jared Kaplan", "Tom Henighan", "Sam McCandlish"],
       "year": 2021,
       "arxiv_id": "2102.01293",
       "relevance": "Prior work that described a smoothly broken power law in passing (for transfer learning scaling); BNSL extends this to multiple breaks and diverse settings."
     },
     {
       "title": "Grokking: Generalization Beyond Overfitting on Small Algorithmic Datasets",
       "authors": ["Alethea Power", "Yuri Burda", "Harri Edwards"],
       "year": 2022,
       "arxiv_id": "2201.02177",
       "relevance": "Introduced the modular arithmetic task exhibiting delayed sharp transitions in scaling behavior that BNSL aims to capture."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Related to emergent reasoning capabilities in LLMs whose scaling behavior BNSL aims to predict."
     }
   ]
 }