ai-research-survey

scan.json (24694B)

---

 {
   "paper": {
     "title": "Scaling Test-Time Compute Without Verification or RL is Suboptimal",
     "authors": ["Amrith Setlur", "Nived Rajaraman", "Sergey Levine", "Aviral Kumar"],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2502.12118",
     "doi": "10.48550/arXiv.2502.12118"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper proves that verifier-based (VB) methods for scaling test-time compute outperform verifier-free (VF) methods by a factor of √H when the base LLM is heterogeneous and anti-concentrated. Empirically, VB best-of-N search scales compute by 8× and data by 6× over VF SFT on MATH with 3B/8B Llama models. The s1 model trained with verifier-free distillation is outperformed by simple best-of-N search with a trained verifier. The theoretical separation grows super-linearly when both data and compute are scaled.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: MATH benchmark and MATH500 evaluation set. The didactic planted subsequence problem is fully specified in the paper and appendix."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup listing library versions is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or scripts are provided. Experimental details are in the appendix but no runnable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Figures 5, 7, and 8 show point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims VB outperforms VF based on comparing accuracy numbers (e.g., '8× scaling') without statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context: '8× compute scaling', '6× data efficiency', accuracy gaps of 9%, 17%, 20% (Figure 7c), and specific accuracy differences between VB and VF across budgets."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why specific data budgets (2^12 to 2^16) or evaluation set sizes were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or spread measures are reported for the experimental results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares VB (BoN, RL) against VF (SFT) methods, and also compares against the base policy and expert policy (Figures 5, 7, 8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include the s1 model (2025), SFT following Snell et al. (2024) and Qu et al. (2024) approaches, which are recent and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates over: data budget scaling (Figure 5b, 7b), compute budget scaling (Figure 5a, 7a), base policy heterogeneity (Figure 6, 9), and combined scaling (Figure 5c, 7c)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses test-time efficiency (normalized J_r), accuracy on MATH500, accuracy on AIME24, and verifier accuracy as metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to the claims — the paper evaluates mathematical reasoning with ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training on MATH training set, evaluation on MATH500 benchmark and AIME24 (Appendix D). The didactic setup also separates train and test prompts."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by heterogeneity buckets (low/medium/high, Figure 9), easy/hard problem sets (Figure 10), and separate MATH500/AIME24 results (Figure 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses when VF outperforms VB: low heterogeneity settings (Figure 6), and identifies verifier accuracy decline as a source of RL performance degradation (Appendix C, Figure 12)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows VF SFT outperforming VB RL when heterogeneity is low. Figure 9 shows VF SFT outperforming BoN on low-heterogeneity problem buckets. The paper explicitly discusses when its thesis does not hold."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about √H separation, VB outperforming VF, and empirical corroboration on 3/8/32B models are supported by theorems (5.1, 5.8) and Figures 5, 7, 8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about verification enabling better scaling are supported by both formal proofs (Theorems 5.4, 5.7, 5.8) and controlled experiments varying one variable at a time (heterogeneity, data budget, compute budget)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its empirical findings to specific models (Llama-3.1/3.2 3B/8B, s1, GPT2-xl), specific benchmarks (MATH, AIME, planted subsequence), and acknowledges theoretical conditions (heterogeneity, anti-concentration) required for the separation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses verifier accuracy as an alternative explanation for performance gaps (Figure 12), discusses when heterogeneity is low and VF outperforms VB (Figure 6), and discusses reward hacking as a failure mode of VB methods."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines what it measures (bi-level reward capturing both correctness and token efficiency) and discusses in Remark 5.10 whether improvements come from solving more problems vs. solving existing problems faster."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are named: Llama-3.1-8B-Instruct, Llama-3.2-3B-Instruct, GPT2-xl, and the s1 model with reference to its paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense — it finetunes models with RL/SFT and evaluates math problem solving. The SFT data construction is described procedurally."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C and D report: batch size 64, learning rates (1e-4 for RL, 2e-4 for SFT), KL penalty weight 0.2, Adam optimizer, 20k iterations for RL, 10000 iterations for MATH, weight decay 0.01, temperature 1.0 for sampling."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct model finetuning and inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The SFT data construction is documented: rejection sampling from base policy, stitching incorrect + correct traces (Appendix D, with examples). Verifier training data collection is described: sampling from base LLM, 0/1 annotation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Discussion, Limitations, and Future Work' includes substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 identifies specific limitations: the analysis only separates VB vs VF (not sparse vs dense verification), bi-level rewards may not capture all reward types, and scaling to >32k contexts is not studied. Also discusses verifier accuracy as a practical concern."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what is NOT covered: dense/process-based rewards, generative rewards, scaling to very long contexts (>32k). The theoretical conditions (heterogeneity, anti-concentration) bound when results apply."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (training logs, generated traces, verifier predictions) is released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: rejection sampling from base policy for SFT data, sampling from base policy with 0/1 annotation for verifier training, binary search for bi-level location in didactic setup (Appendix C, D)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks (MATH) and model-generated traces."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: base LLM → sample traces → annotate with rewards → train verifier/SFT → evaluate. SFT trace construction (stitching incorrect + correct responses) is detailed with examples in Appendix D."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists: Office of Naval Research (N00014-24-12206), JP Morgan AI PhD Fellowship, NSF Grants IIS-1901252 and CCF-2211209, Google Cloud TRC program, Lambda labs."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed: Carnegie Mellon University and UC Berkeley. No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (ONR, NSF, JP Morgan fellowship) do not have a direct financial stake in whether VB or VF methods perform better. Google Cloud and Lambda provided compute resources, not research direction."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates Llama-3.1/3.2 models finetuned on MATH and tested on MATH500/AIME24 but does not state the training data cutoff for the base models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MATH problems could appear in Llama pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH was published in 2021; Llama models trained after this. No discussion of contamination risk for MATH500 or AIME24 benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Token budgets are varied (2^9 to 2^13) but no API costs, wall-clock times, or per-example costs are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgements mention Google Cloud and Lambda labs for compute but no GPU hours, total training time, or compute budget is quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No reporting of results across multiple random seeds. Single-run results appear to be reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated for any experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported but no search budget or number of configurations tried is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the reported hyperparameters were selected or whether they were tuned on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both VB and VF baselines themselves without acknowledging potential bias in their implementations. They do compare against the externally-trained s1 model, partially mitigating this."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is explicitly plotted as a function of compute budget (test-time tokens H) in Figures 5a, 7a, 8. Compute-matched evaluation is emphasized."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines and formalizes what 'effective test-time compute scaling' means (Definition 4.2, bi-level reward Property 4.1), explicitly discussing what the evaluation measures and why."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the paper compares training methods (SFT vs RL/BoN) directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. MATH (2021) and AIME24 benchmarks could have been seen during Llama pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test sets beyond using separate MATH splits."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Verifier-based methods scale test-time compute by √H better than any verifier-free method when the base LLM is heterogeneous and anti-concentrated.",
       "evidence": "Theorems 5.1, 5.4, 5.7, 5.8 with formal proofs in Appendix A. Lower bound Ω(H/√n) for VF vs upper bound O(H/n) for VB.",
       "supported": "strong"
     },
     {
       "claim": "VB best-of-N scales compute by 8× and data efficiency by 6× over VF SFT on MATH with Llama 3B/8B models.",
       "evidence": "Figure 7(a)(b): at fixed n=2^14, BoN achieves at 2^10 tokens what SFT needs 2^13 tokens for; at fixed H=2^12, BoN at 2^13 data matches SFT at 2^16.",
       "supported": "moderate"
     },
     {
       "claim": "The gap between VB and VF grows super-linearly when both data and compute are scaled jointly.",
       "evidence": "Figure 7(c) shows accuracy gap growing from 9% to 17% to 20% as log H increases; Figure 5(c) shows similar trend in didactic setup.",
       "supported": "moderate"
     },
     {
       "claim": "The s1 model trained with verifier-free distillation is outperformed by simple best-of-N search with a trained verifier.",
       "evidence": "Figure 8: BoN outperforms budget-forced s1 on both MATH500 and AIME24 across compute budgets.",
       "supported": "moderate"
     },
     {
       "claim": "Common pre-trained LLMs (Llama-3.1-8B) exhibit heterogeneous and anti-concentrated reward distributions.",
       "evidence": "Figure 9 shows distribution of σ_x skewed toward higher values; Figure 10 shows ~0.25 anti-concentration coefficient on both easy and hard MATH problems.",
       "supported": "moderate"
     },
     {
       "claim": "When base policy heterogeneity is low, VF methods can outperform VB methods.",
       "evidence": "Figure 6: at low σ_b, SFT outperforms RL. Figure 9: VF SFT beats BoN on low-heterogeneity problem buckets.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All experimental results (Figures 5-10) appear to be single-run results with no error bars, confidence intervals, or variance across seeds. Given that RL training can be highly variable (as Henderson et al. 2018 demonstrated), the reported gaps could be within noise."
     },
     {
       "flag": "No contamination analysis",
       "detail": "MATH benchmark was published in 2021. Llama models likely trained on data including MATH solutions. No contamination analysis is performed, which could inflate baseline accuracy and affect the VB/VF comparison."
     },
     {
       "flag": "Authors implement their own baselines",
       "detail": "Both VB and VF methods are implemented by the authors. The SFT baseline follows Snell et al. and Qu et al. but is not their original code. The s1 comparison (Figure 8) partially mitigates this by comparing against an externally trained model."
     }
   ],
   "cited_papers": [
     {
       "title": "SFT memorizes, RL generalizes: A comparative study of foundation model post-training",
       "authors": ["Tianzhe Chu", "Yuexiang Zhai", "Jihan Yang"],
       "year": 2025,
       "arxiv_id": "2501.17161",
       "relevance": "Systematic comparison of SFT vs RL post-training, finding RL generalizes better — directly supports this paper's thesis."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major RL-trained reasoning model using 0/1 outcome rewards, exemplifying VB approach at scale."
     },
     {
       "title": "s1: Simple test-time scaling",
       "authors": ["Niklas Muennighoff", "Zitong Yang", "Weijia Shi"],
       "year": 2025,
       "arxiv_id": "2501.19393",
       "relevance": "Key VF baseline: distills Gemini Thinking traces via supervised learning, directly compared against in Figure 8."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on optimal test-time compute scaling that this paper builds upon theoretically and empirically."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduced verifier-based approach for math reasoning that is the basis for the best-of-N method used in this paper."
     },
     {
       "title": "Is behavior cloning all you need? Understanding horizon in imitation learning",
       "authors": ["Dylan J Foster", "Adam Block", "Dipendra Misra"],
       "year": 2024,
       "arxiv_id": "2407.15007",
       "relevance": "Provides theoretical lower bounds for supervised finetuning that this paper extends to multi-prompt settings."
     },
     {
       "title": "RL on incorrect synthetic data scales the efficiency of LLM math reasoning by eight-fold",
       "authors": ["Amrith Setlur", "Saurabh Garg"],
       "year": 2024,
       "arxiv_id": "2406.14532",
       "relevance": "Prior work by same authors on using RL with synthetic data for math reasoning, foundational to this paper's approach."
     },
     {
       "title": "Generative verifiers: Reward modeling as next-token prediction",
       "authors": ["Lunjun Zhang", "Arian Hosseini"],
       "year": 2024,
       "arxiv_id": "2408.15240",
       "relevance": "Proposes generative verification approach that falls under this paper's VB category."
     },
     {
       "title": "STaR: Bootstrapping reasoning with reasoning",
       "authors": ["Eric Zelikman", "Yuhuai Wu", "Jesse Mu", "Noah Goodman"],
       "year": 2022,
       "relevance": "Influential self-training approach for reasoning that uses rejection sampling — a VF method in this paper's taxonomy."
     },
     {
       "title": "Training language models to self-correct via reinforcement learning",
       "authors": ["Aviral Kumar", "Vincent Zhuang", "Rishabh Agarwal"],
       "year": 2024,
       "arxiv_id": "2409.12917",
       "relevance": "RL-based self-correction approach that motivates the sequential revision SFT baseline used in this paper."
     },
     {
       "title": "Rewarding progress: Scaling automated process verifiers for LLM reasoning",
       "authors": ["Amrith Setlur", "Chirag Nagpal", "Adam Fisch"],
       "year": 2024,
       "arxiv_id": "2410.08146",
       "relevance": "Process verification for LLM reasoning, extends the verification paradigm studied in this paper."
     },
     {
       "title": "DeepScaleR: Surpassing o1-preview with a 1.5B model by scaling RL",
       "authors": ["Michael Luo", "Sijun Tan"],
       "year": 2025,
       "relevance": "Demonstrates VB RL scaling with outcome rewards on small models, validating this paper's thesis in practice."
     }
   ]
 }