ai-research-survey

scan.json (24993B)

---

 {
   "paper": {
     "title": "Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for Problem-Solving with Language Models",
     "authors": [
       "Yangzhen Wu",
       "Zhiqing Sun",
       "Shanda Li",
       "Sean Welleck",
       "Yiming Yang"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2408.00724"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Scaling inference compute with smaller models and advanced inference strategies can outperform larger models at the same compute budget. The proposed REBASE tree search algorithm achieves Pareto-optimal cost-performance tradeoffs, with Llemma-7B + REBASE outperforming Llemma-34B with standard methods using 2x less FLOPs. Theoretical analysis shows sampling-based voting converges exponentially but saturates at a limit determined by the model's output distribution, motivating more sophisticated inference algorithms.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A project page URL is provided (https://thu-wyz.github.io/inference-scaling/) in the author block, which serves as a release page for the work."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (MATH, GSM8K, MBPP, MetaMath, Math-Shepherd) are publicly available datasets. The paper references them with citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. Table 2 lists fine-tuning hyperparameters but not software dependencies or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Hyperparameters are listed in Appendix C but no runnable commands or scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'Each configuration is run multiple times to calculate the mean and variance, which mitigates effects from randomness.' Figures show shaded regions indicating variance."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'REBASE outperforms sampling' are based on visual comparison of curves and point estimates without p-values or formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context throughout: '2× less FLOPs', '7 times less compute' (Table 1), specific accuracy numbers with baselines (e.g., 45.5% vs 46.8% MATH500 accuracy)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why MATH500 (500 problems) or GSM8K test set sizes were used. No power analysis or discussion of whether sample sizes are adequate for the conclusions drawn."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'Each configuration is run multiple times to calculate the mean and variance.' Figures display variance bands."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive baselines: greedy search, majority voting, best-of-n, weighted majority voting, and MCTS. Both sampling-based and tree-search variants are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MCTS variants from 2023-2024 are included (Zhang et al. 2023, Zhou et al. 2024, Liu et al. 2024). Self-consistency/majority voting (Wang et al. 2023) is contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of REBASE components (e.g., effect of balance temperature Tb, reward model quality, expansion budget formula). The MATH-easy vs MATH-hard analysis is a breakdown, not an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy/error rate is used as the evaluation metric throughout. While multiple benchmarks are tested (MATH, GSM8K, MBPP), the metric is always accuracy or pass rate."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant — the benchmarks have ground-truth mathematical answers that can be verified automatically."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MATH500 test set and GSM8K test set are standard held-out splits. Models are fine-tuned on MetaMath training data, evaluated on separate test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 breaks results down by MATH difficulty level (easy: levels 1-2, hard: levels 3-5). Results are shown per-model and per-strategy throughout."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "MCTS failure mode is discussed: 'MCTS underperforms the sampling-based methods at each compute budget, likely due to its costly rollouts' and its production of 'many unfinished solutions.' Saturation behavior is analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "MCTS is shown to underperform simpler sampling methods — a significant negative result given MCTS's popularity. Saturation of sampling methods is documented as a limitation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — smaller models outperform larger ones at fixed compute, REBASE is Pareto-optimal, Llemma-7B outperforms Llemma-34B — are all supported by Figures 4-7 and Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are supported by controlled single-variable manipulation: varying model size while fixing strategy, or varying strategy while fixing model. The inference strategy is the only variable changed in each comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title claims 'Inference Scaling Laws' and 'Problem-Solving with Language Models' broadly, but experiments are primarily on mathematical reasoning benchmarks (MATH, GSM8K) with one small code generation experiment (MBPP, Appendix D.2). No natural language reasoning, commonsense, or other problem types tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. For example, the reward model quality could explain much of REBASE's advantage (it uses the same Llemma-34B reward model for both 7B and 34B policy models), but this confound is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims match measurement granularity. The paper measures accuracy on math benchmarks and claims compute-optimal inference for math problem-solving. No overreach to broader 'reasoning' or 'intelligence' framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names and sizes are provided: Pythia-410M/1.4B/2.8B/6.9B/12B, Llemma-7B, Llemma-34B, Mistral-7B, Llama3-8B-Instruct. These are specific open-source model checkpoints."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses fine-tuned models that take math problems as direct input. No custom prompting or prompt engineering is involved — inputs are benchmark questions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides fine-tuning hyperparameters (LR, batch size, epochs, max seq length, dtype). Appendix C reports inference parameters: temperature 1.0, max tokens 1024, REBASE Tb=0.1, MCTS C=1 and expansion configurations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The inference strategies (sampling, tree search) are the methods being studied, not scaffolding around an agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions 'We preprocess the MetaMath Dataset to make the solutions in a stepwise format' but provides no detail on how this preprocessing was done."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion mentions some caveats about future work but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do not show. The paper does not bound its conclusions to math or acknowledge that inference scaling laws may differ for other task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Generated solutions, reward model scores, and per-problem results are not released. Only aggregated accuracy numbers and FLOPs are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: MATH500 test set, GSM8K test set, MetaMath for fine-tuning, Math-Shepherd for reward model training. All with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: fine-tune policy models on MetaMath → fine-tune reward model on Math-Shepherd → generate solutions via various strategies → aggregate via voting/selection → evaluate accuracy."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'Zhiqing Sun acknowledges the support of the Google Fellowship. Sean Welleck thanks NSF SCALE (NSF DMS 2134012) and Convergent Research.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors' affiliations are listed: Tsinghua University and Carnegie Mellon University. No company products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Google Fellowship and NSF — neither has a direct financial interest in the specific inference scaling law findings or REBASE algorithm."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the base models (Pythia, Llemma, Mistral, Llama3). The pretraining data composition is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MATH or GSM8K problems appear in the pretraining data of any model used. MetaMath is derived from these benchmarks' training splits, but overlap with test sets is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and MATH (2021) predate all models used, creating contamination risk. This is not discussed. MetaMath is derived from these datasets, adding further risk of leakage."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference FLOPs per question are reported extensively throughout — this is the core axis of analysis. Table 1 provides specific FLOPs values for each configuration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total computational budget (GPU hours, hardware, wall-clock time for all experiments) is not stated. Only per-question inference FLOPs are reported. Fine-tuning compute is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'Each configuration is run multiple times to calculate the mean and variance, which mitigates effects from randomness.' Variance is shown in figures."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'run multiple times' but never states the exact number of runs per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The REBASE balance temperature Tb=0.1 and MCTS C=1 appear tuned but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "REBASE uses Tb=0.1 without justification for this specific value. MCTS expansion configurations (4/8/16 children) are stated but not justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose REBASE and compare it against their own implementations of baselines (sampling, MCTS). No acknowledgment of self-comparison bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central contribution. All main results (Figures 1, 4-7) plot performance as a function of inference FLOPs, enabling direct compute-matched comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Brief mention that math 'allows us to accurately evaluate problem solving ability' (Section 4.1) but no substantive discussion of whether MATH/GSM8K actually measure the claimed capabilities or their limitations as benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same reward model (Llemma-34B) is used across all inference strategies and model sizes, controlling for the reward model confound. Each comparison varies only one factor (model size OR strategy)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH (2021) and GSM8K (2021) predate the training of models like Llama3 (2024). No discussion of whether benchmark solutions appeared in pretraining data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The reward model is trained on Math-Shepherd which is derived from the same problem domain."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MetaMath training data is derived from MATH and GSM8K training splits. No verification that train and test problems are truly independent or that near-duplicates don't exist."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Scaling inference compute with smaller models can be more computationally efficient than scaling model parameters.",
       "evidence": "Figure 1 shows Pythia models on GSM8K where smaller models (410M-2.8B) outperform larger ones at lower compute budgets. Figures 4-5 show Llemma-7B achieves comparable accuracy to Llemma-34B with 2x less FLOPs.",
       "supported": "strong"
     },
     {
       "claim": "REBASE is Pareto-optimal across all tested compute budgets, outperforming sampling and MCTS methods.",
       "evidence": "Figures 4-7 show REBASE dominates at all compute budgets across MATH, GSM8K, and multiple models (Llemma-7B/34B, Mistral-7B). Table 1 shows REBASE achieves better accuracy with 7x less compute than sampling.",
       "supported": "strong"
     },
     {
       "claim": "Sampling-based voting converges exponentially to a limit determined by the model's output distribution, with diminishing returns.",
       "evidence": "Theorems 1 and 2 prove convergence bounds with formal proofs in Appendix A. Empirical saturation is visible in Figures 1, 6, and 7.",
       "supported": "strong"
     },
     {
       "claim": "MCTS has unfavorable cost-performance tradeoffs compared to simpler sampling methods.",
       "evidence": "Figure 4 shows MCTS underperforms sampling-based methods at each compute budget on MATH for both 7B and 34B models. Attributed to costly rollouts producing unfinished solutions.",
       "supported": "strong"
     },
     {
       "claim": "REBASE yields greater gains on hard problems than easy problems.",
       "evidence": "Figure 8 shows comparable performance on MATH-easy (levels 1-2) but significant advantage on MATH-hard (levels 3-5) for both Llemma-7B and 34B.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Same reward model used for all comparisons",
       "detail": "All experiments use a Llemma-34B reward model. REBASE relies more heavily on the reward model than sampling-based methods. The reward model quality confound is not discussed — REBASE's advantage may partly stem from having a strong reward model rather than the search algorithm itself."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section and does not discuss threats to validity, scope boundaries, or conditions under which the findings might not hold."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "MATH and GSM8K benchmarks (2021) predate all models used. MetaMath training data is derived from these benchmarks. No contamination analysis is performed."
     },
     {
       "flag": "Generalization beyond math unclear",
       "detail": "Title claims general 'Inference Scaling Laws' and 'Problem-Solving' but experiments are overwhelmingly on mathematical reasoning. The MBPP code generation experiment (Appendix D.2) is a small addition without the full scaling analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws for training compute — the training-side analogue to this paper's inference scaling laws."
     },
     {
       "title": "Scaling test-time compute optimally can be more effective than scaling LLM parameters",
       "authors": ["Charlie Victor Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2025,
       "relevance": "Concurrent work on test-time compute scaling with complementary inference strategies."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Foundational work on majority voting for LLM reasoning that this paper analyzes theoretically."
     },
     {
       "title": "Beyond chinchilla-optimal: Accounting for inference in language model scaling laws",
       "authors": ["Nikhil Sardana", "Jacob Portes", "Sasha Doubov", "Jonathan Frankle"],
       "year": 2024,
       "relevance": "Studies scaling laws accounting for inference cost but only considers fixed inference algorithms."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Seminal training scaling laws paper whose FLOPs formula is used for inference compute estimation."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Chain-of-thought prompting as an inference-time technique that increases compute for better reasoning."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yuri Burda"],
       "year": 2024,
       "relevance": "Process reward modeling for step-by-step verification — the PRM approach used in REBASE."
     },
     {
       "title": "Math-shepherd: Verify and reinforce LLMs step-by-step without human annotations",
       "authors": ["Peiyi Wang", "Lei Li", "Zhihong Shao"],
       "year": 2024,
       "relevance": "Synthetic process reward modeling dataset used to train the reward model in this paper's experiments."
     },
     {
       "title": "From decoding to meta-generation: Inference-time algorithms for large language models",
       "authors": ["Sean Welleck", "Amanda Bertsch", "Matthew Finlayson"],
       "year": 2024,
       "relevance": "Survey of inference-time algorithms for LLMs covering decoding strategies studied in this paper."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Tree search approach for LLM reasoning that combines search with LLMs."
     }
   ]
 }