ai-research-survey

scan.json (24857B)

---

 {
   "paper": {
     "title": "Pretraining Scaling Laws for Generative Evaluations of Language Models",
     "authors": ["Rylan Schaeffer", "Noam Levi", "Brando Miranda", "Sanmi Koyejo"],
     "year": 2025,
     "venue": "ICLR 2026",
     "arxiv_id": "2509.24012",
     "doi": "10.48550/arXiv.2509.24012"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper proposes three pretraining scaling laws for pass-at-k on generative evaluations (GSM8K, MATH) using (1) compute, (2) parameters+tokens, and (3) gold reference likelihoods. The number of attempts k acts as a hyperparameter that modulates scaling behavior, eliminating irreducible error and steepening power laws. The gold reference likelihood law is uniquely stable across ~5 orders of magnitude of compute, compared to ~1.5-2.5 for the other laws. The compute scaling law is proven to be the compute-optimal envelope of the parameters+tokens law.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available Pythia model family, GSM8K, and MATH benchmarks. The sampling data (~500M GSM8K, ~400M MATH samples) is not explicitly released, but the underlying benchmarks and models are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM for sampling (Appendix C) but provides no environment specification, requirements file, or dependency versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the approach but not how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported on the scaling law fits or predictions. Figures show point estimates of fitted curves without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares three scaling laws' predictive performance but does not apply statistical significance tests to determine whether differences are meaningful."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes: e.g., compute exponent α(k) rising from 0.121 to 0.375, parameter stability within ~1.5-2.5 vs ~5 orders of magnitude, and relative prediction errors (Figs 3, 5, 7)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 justifies using 128 problems from each benchmark and explains the compute-aware sampling strategy (Appendix B), noting 2^14 minimum samples per problem to ensure resolution for k up to 10^4."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. Results appear to be from single fits without reporting stability across different random subsets of problems."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The three scaling laws serve as mutual baselines, compared against each other for fit quality and predictive performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The scaling law formulations are grounded in the most relevant prior work: GPT-4 Technical Report (2024), Hoffmann et al. (2022), and Schaeffer et al. (2024b)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies k across five orders of magnitude and examines how each scaling law parameter changes, effectively ablating the role of the attempt hyperparameter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates fit quality, backtesting relative error, parameter stability across compute horizons, and compares across two benchmarks (GSM8K, MATH)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for a scaling law study on automated benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The backtesting methodology (Sec 3.2) explicitly holds out the most expensive model (Pythia 12B at 300B tokens) as the prediction target, fitting only on cheaper models."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by k value (1, 10, 100, 1000, 10000) and by benchmark (GSM8K vs MATH in Section 7 and Appendix F)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where scaling laws fail: unreliable extrapolation beyond ~2 orders of magnitude (Sec 3.2), the parameters+tokens law's large error on the most expensive checkpoint (Sec 4.1), and instability of MATH parameters (Sec 7)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the parameters+tokens law despite better in-range fit does not improve extrapolation (Sec 4.2), and that gold reference law worsens for large k (Sec 5.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: k modulates scaling behavior (Figs 1-2), parameter stability differences (Figs 3,5,7), comparable predictive performance (Figs 3,5,7), and the theoretical envelope connection (Sec 6, Appendix E)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily mathematical (the compute law IS the envelope of the N+D law, proved in Sec 6/Appendix E) or descriptive (k shapes scaling behavior). The theoretical derivation provides adequate justification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Sec 8) explicitly states the primary limitation is 'its empirical focus on a single model family (Pythia)' and that they 'focus exclusively on pass-at-k metrics.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses how benchmark hardness affects scaling behavior. Section 8 discusses that other sampling algorithms and temperatures could affect results. The paper acknowledges it's unclear what determines stability differences across benchmarks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes pass-at-k from accuracy (Appendix A, three explicit differences listed) and frames its claims specifically in terms of pass-at-k, not broader 'model capability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pythia model family (Biderman et al., 2023), specifying 8 models from 14M to 12B parameters, non-deduplicated variants, with specific checkpoint counts."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models via generative sampling on math problems, not through prompting with system instructions. The problems themselves are from public benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature τ=1.0 is stated (Sec 2). Sampling strategy (temperature-only, no top-p/top-k) is specified. Minimum 2^14 samples per problem, maximum 2^15 (Appendix B)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates raw model sampling."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents: randomly selected 128 problems from each benchmark, excluded overtrained checkpoints per Godey et al. (2024), compute approximated as C≈6ND, and sampling strategy detailed in Appendix B."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Discussion) contains a dedicated 'Limitations' paragraph discussing single model family and single metric focus."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: single model family constraint (Sec 8), using intermediate checkpoints in lieu of fully trained models (Sec 2), potential impact of the 128-problem subset (Sec 2), and excluded overtrained checkpoints (Sec 2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: 'focusing on a particular setting: benchmarks with verifiable binary rewards, with multiple attempts per problem, with performance scored using the pass-at-k metric' (Sec 1). Limitations in Sec 8 state focus on 'a single model family' and 'pass-at-k metrics.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The ~500M GSM8K and ~400M MATH samples are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Appendix B describe the sampling procedure in detail: model family, number of samples per problem (min 2^14, max 2^15, early stopping at 10 successes), vLLM with prefix caching (Appendix C)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks (GSM8K, MATH) and a public model family (Pythia)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: select 128 problems → sample from each Pythia checkpoint → apply unbiased pass@k estimator (Eq. 1-2) → fit scaling laws → backtest. Appendix B shows exact sample counts per model per problem."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists: Stanford Data Science, OpenAI Superalignment Fast Grant (RS), EPFL AI4science/AI Center (NL), Schmidt Sciences, Stanford EDGE Scholar Fellowship (BM), NSF, MacArthur Foundation, Stanford HAI, OpenAI, Google (SK)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stanford (RS, BM, SK) and EPFL (NL). No product of these institutions is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates the open-source Pythia model family, not products of any funder. While OpenAI and Google fund some authors, the evaluated models are from EleutherAI, making the funders independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Pythia was trained on The Pile (Gao et al., 2020), and the paper specifies up to 300B tokens. The Pile's composition is public, establishing an implicit cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GSM8K (2021) or MATH (2021) problems appeared in The Pile training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K and MATH were published in 2021. The Pile was assembled around the same time. No discussion of whether benchmark solutions leaked into training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed cost estimates: A100-seconds per 128 samples by model size (Tables 1-2), total lower bound of 81K A100-hours for GSM8K, and financial cost >$112K at current cloud pricing."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states the total compute: lower bound of 81K A100-hours for GSM8K, with per-model breakdowns in Tables 1-2. Total samples: ~500M for GSM8K, ~400M for MATH."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No sensitivity analysis across random seeds is reported. The 128-problem subset is randomly selected but no analysis of how different subsets would affect results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B details exact sample counts per model per problem (Figs 9-10), with minimum 2^14 and maximum 2^15 samples per problem."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The scaling law fits use standard nonlinear regression; no hyperparameter search is involved in the methodology."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The three functional forms are motivated by prior work (GPT-4 report, Hoffmann et al., Schaeffer et al.) and all three are reported rather than selecting only the best."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose all three scaling laws and evaluate them without acknowledging self-comparison bias or having independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about performance as a function of compute budget. Figures 1, 4, 6 and the backtesting analysis explicitly show performance vs compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GSM8K and MATH without discussing whether pass-at-k on these specific benchmarks validly measures the 'generative capability' claimed in the title."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated via direct sampling."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K and MATH were published in 2021, around the same time as The Pile. No discussion of whether benchmark solutions appeared in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup introduces information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test data share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The number of attempts per problem k acts as a hyperparameter that modulates scaling law parameters: irreducible error E0(k) vanishes exponentially, compute exponent α(k) increases from ~0.121 to ~0.375 as k goes from 1 to 10^4.",
       "evidence": "Figures 1-2 and Section 3.1 show fitted scaling law parameters as functions of k across GSM8K.",
       "supported": "strong"
     },
     {
       "claim": "The gold reference likelihood scaling law has uniquely stable parameters, converging across ~5 orders of magnitude of compute, compared to ~1.5-2.5 for the other two laws.",
       "evidence": "Figure 7 (other three panels) shows parameter stability across backtesting horizons, compared to Figures 3 and 5 for the other laws.",
       "supported": "strong"
     },
     {
       "claim": "All three scaling laws perform comparably in predicting the most expensive model's performance, though the compute law is slightly worse for small k and the gold reference law slightly worse for large k.",
       "evidence": "Top-left panels of Figures 3, 5, and 7 show relative prediction errors across k values.",
       "supported": "strong"
     },
     {
       "claim": "The compute scaling law is the compute-optimal envelope of the parameters+tokens scaling law, with α(k) = (1/β(k) + 1/γ(k))^{-1}.",
       "evidence": "Section 6 and Appendix E provide the full mathematical derivation with proof.",
       "supported": "strong"
     },
     {
       "claim": "MATH retains significant irreducible error even at k=10^4 (E0≈0.45), quantifying that it is a harder benchmark than GSM8K where E0 vanishes by k≈100.",
       "evidence": "Section 7 and Appendix F (Figure 14) compare GSM8K and MATH scaling behaviors.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model family",
       "detail": "All empirical results are from Pythia (14M-12B parameters). The authors acknowledge this but it severely limits generalizability — scaling laws may not hold for differently architected or differently trained model families."
     },
     {
       "flag": "No contamination analysis",
       "detail": "GSM8K and MATH were published around the same time The Pile was assembled. If benchmark solutions leaked into training data, the scaling law fits could be distorted, particularly for larger models that memorize more."
     },
     {
       "flag": "Small benchmark subset",
       "detail": "Only 128 problems from each benchmark were used (out of 8,500 GSM8K and 12,500 MATH). The random subset could affect scaling law parameters, and no sensitivity analysis is performed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced pass-at-k metric and the unbiased estimator used in this paper; foundational for code generation evaluation."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws for compute-optimal training; the parameters+tokens law in this paper extends Chinchilla to generative evaluation."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational neural scaling laws paper; this work extends scaling laws from pretraining loss to generative benchmarks."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenged emergent abilities claims by showing they arise from metric choice; directly relevant to understanding benchmark evaluation methodology."
     },
     {
       "title": "Language models scale reliably with over-training and on downstream tasks",
       "authors": ["Samir Yitzhak Gadre"],
       "year": 2024,
       "arxiv_id": "2403.08540",
       "relevance": "Studied scaling on 46 downstream tasks (none generative); this paper fills the generative evaluation gap."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Studied repeated sampling as inference-time compute scaling strategy; directly related to the pass-at-k scaling studied here."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Inference-time compute scaling; complementary to pretraining scaling laws for generative tasks."
     },
     {
       "title": "Codemonkeys: Scaling test-time compute for software engineering",
       "authors": ["Ryan Ehrlich", "Bradley Brown"],
       "year": 2025,
       "arxiv_id": "2501.14723",
       "relevance": "Applied repeated sampling scaling to software engineering tasks; extends pass-at-k to practical code generation."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2303.08774",
       "relevance": "Pioneered scaling law prediction for generative evaluations (HumanEval); the compute law in this paper adopts their functional form."
     },
     {
       "title": "Predicting emergent abilities with infinite resolution evaluation",
       "authors": ["Shengding Hu", "Xin Liu"],
       "year": 2024,
       "arxiv_id": "2310.03262",
       "relevance": "Infinite resolution sampling for emergent abilities; philosophically related approach to understanding scaling via finer-grained evaluation."
     },
     {
       "title": "Pythia: A suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman"],
       "year": 2023,
       "relevance": "The model family used for all experiments in this paper; enables dense checkpoint analysis across model sizes."
     }
   ]
 }