ai-research-survey

scan.json (26319B)

---

 {
   "paper": {
     "title": "Can 1B LLM Surpass 405B LLM? Rethinking Compute-Optimal Test-Time Scaling",
     "authors": [
       "Runze Liu",
       "Junqi Gao",
       "Jian Zhao",
       "Kaiyan Zhang",
       "Xiu Li",
       "Biqing Qi",
       "Wanli Ouyang",
       "Bowen Zhou"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.06703"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references an open-source codebase OpenR (https://github.com/openreasoner/openr) used for experiments and provides a website at https://ryanliu112.github.io/compute-optimal-tts. The OpenR framework is explicitly cited as their codebase."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: MATH-500 (Lightman et al., 2024) and AIME24 (AI-MO, 2024, available on HuggingFace). All PRMs used are open-source and publicly available with HuggingFace links provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions using OpenR but does not specify library versions or system requirements."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper mentions using OpenR and specifies some hyperparameters (temperature, token limits, beam width) but does not provide a README or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. All tables (Tables 3-6) and figures report only point estimates without uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite numerous comparative claims (e.g., '3B LLM surpasses 405B LLM'). All comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports percentage performance gains (e.g., '154.6% improvement over CoT') and efficiency gains (e.g., '>256.0x'). Table 4 reports FLOPS comparisons with specific magnitudes. The paper provides baseline context throughout (e.g., 'from 26.0% CoT to 66.2% compute-optimal TTS')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why MATH-500 (500 problems) and AIME24 (30 problems) are sufficient. AIME24 has only 30 problems, which is very small for the granular difficulty-level analyses performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experimental runs. It is unclear whether results are from single runs or averaged across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: CoT (zero-shot), majority voting, and comparisons against GPT-4o, o1-preview, o1-mini, o1, DeepSeek-R1, QwQ-32B-Preview, and various long-CoT methods (rStar-Math, Eurus-2, SimpleRL, Satori)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include state-of-the-art models as of early 2025: DeepSeek-R1, o1, GPT-4o, QwQ-32B-Preview, and recent long-CoT methods like SimpleRL and Satori. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically ablates across multiple dimensions: different policy model sizes (0.5B to 72B), different PRMs (7 different PRMs), different TTS methods (BoN, beam search, DVTS), different scoring/voting methods (Table 2), and different difficulty levels (Sections 4.2-4.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy on MATH-500 and AIME24, Pass@k, FLOPS comparisons (Table 4), efficiency gains, and performance gains as distinct metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper on mathematical reasoning. Human evaluation of model outputs is not relevant since correctness is determined by matching ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MATH-500 is a standard test subset of MATH, and AIME24 is a separate competition dataset. The compute-optimal strategy is evaluated on these test sets, not on data used for tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (easy, medium, hard) in Figures 8-9, by policy model size (0.5B to 72B) in Figures 7 and 10-11, and by PRM in Figures 4-5. Per-model and per-task breakdowns are provided in Tables 3 and 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed failure case analysis of PRMs, identifying four categories: Over-Criticism (Figure 13), Error Neglect (Figures 14-15), Error Localization Bias (Figure 16), and Scoring Bias (Figures 17-18). Section 4.4 discusses PRM biases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: TTS performance drops on harder tasks like AIME24 (Table 6), TTS is less effective than distillation from strong reasoning models (Section 5.3), PRMs fail to generalize across policy models (Section 4.2), and TTS gains diminish with larger policy models (Table 5, Qwen2.5-32B shows only 0.8x efficiency gain)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'a 1B LLM can exceed a 405B LLM on MATH-500' are supported by Table 3 (Llama-3.2-1B-Inst. at N=512 achieves 72.2% vs. Llama-3.1-405B-Inst. at 71.4%). Claims about 0.5B outperforming GPT-4o, 3B surpassing 405B, and 7B beating o1 are all supported in Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are mostly about component contributions (e.g., 'the reward significantly affects the generation process'). These are supported by controlled ablations varying one factor at a time: varying PRM while holding policy model fixed, varying policy model size while holding PRM fixed, etc. The ablation design is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Can 1B LLM Surpass 405B LLM?' implies general capability, but results are only on mathematical reasoning tasks (MATH-500 and AIME24). The limitations section acknowledges this ('Extending TTS to more tasks such as coding and chemistry tasks') but the title and abstract do not bound the claim to math."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the strong results of small models with TTS could be an artifact of PRM overfitting to MATH-style problems, or whether the compute-optimal strategy is overfit to these specific benchmarks. No threats-to-validity or confound discussion is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: Llama-3.2-1B-Instruct, Llama-3.2-3B-Instruct, Llama-3.1-8B-Instruct, Qwen2.5-0.5B-Instruct through Qwen2.5-72B-Instruct, DeepSeek-R1-Distill-Qwen-1.5B, DeepSeek-R1-Distill-Qwen-7B. PRM versions are also specified (e.g., Math-Shepherd-PRM-7B, RLHFlow-PRM-Mistral-8B). However, for proprietary models (GPT-4o, o1, o1-mini, o1-preview), no snapshot dates or API versions are given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Tables 7 and 8) provides the full system prompts used for Llama 3 series and Qwen2.5 series models. These are complete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies: compute budgets {4, 16, 64, 256}, beam width 4 for search methods, temperature 0.0 for CoT and 0.7 for other methods, max new tokens 8192 for CoT/BoN, 2048 per step and 8192 total for search methods."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the three TTS methods (Best-of-N, Beam Search, DVTS) in detail with mathematical formulations. Section 4.1 describes scoring methods (PRM-Min, PRM-Last, PRM-Avg) and voting methods (Majority Vote, PRM-Max, PRM-Vote). Figure 2 provides a visual comparison of the methods."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents how problem difficulty is categorized (easy: 50-100%, medium: 10-50%, hard: 0-10% Pass@1 accuracy). Section 4.1 describes the step division format ('\\n\\n'). The footnote explains that Qwen2.5-32B-Instruct is used to extract answers from Llama-3.2-1B outputs that don't contain \\boxed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' subsection is present at the end of Section 7 (Conclusion & Discussion), identifying two specific limitations: results are limited to mathematical tasks, and more effective compute-optimal methods could be explored."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions only two generic directions ('extending to more tasks' and 'more effective methods'). No specific threats to the validity of the current results are discussed, such as the small size of AIME24 (30 problems), potential PRM overfitting to math domains, or the fact that compute-optimal strategy selection uses oracle labels."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions extending to other tasks but does not explicitly state what the results do NOT show. There is no statement like 'these results do not demonstrate that TTS is effective for non-mathematical reasoning' or 'the compute-optimal strategy was selected with oracle difficulty labels and may not transfer to real-world settings.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental outputs (model generations, PRM scores, per-problem results) are not made available. Only aggregated accuracy numbers are reported in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well-described: MATH-500 is a specific 500-problem subset from MATH following prior work, AIME24 is from a HuggingFace dataset. PRM training data sources are documented in Section 4.1 and Table 1. The experimental framework (OpenR) is cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are standard public benchmarks (MATH-500, AIME24)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: problems are taken from standard benchmarks, policy models generate responses using specified TTS methods, PRMs score steps/responses, voting/scoring methods select final answers, and answers are extracted (with special handling for Llama-3.2-1B via Qwen2.5-32B as noted in footnote 5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors are from Shanghai AI Laboratory and Tsinghua University but no grants or funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai AI Laboratory, Tsinghua University, Harbin Institute of Technology, and BUPT. The paper notes 'Work done during an internship at Shanghai AI Laboratory.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors are affiliated with Shanghai AI Laboratory, which develops AI systems, creating a potential interest in demonstrating that smaller models can compete with larger ones. Without funding disclosure, this question cannot be answered affirmatively."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (Llama, Qwen, DeepSeek, GPT-4o, o1). This is relevant because MATH and MATH-500 have been publicly available since 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MATH-500 is a subset of MATH (published 2021), and all models tested were trained after 2021. The paper does not address whether these models may have seen MATH problems during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH was published in 2021 and MATH-500 is derived from it. All models used were trained after this date and could have been trained on MATH problems. AIME24 problems may have appeared online before some models' training cutoffs. None of this is addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports inference FLOPS for key comparisons. Table 5 reports efficiency gains (e.g., '>256.0x' more efficient than majority voting). Token counts are discussed in Section 4.4 and shown in the case studies (e.g., Figure 12 shows 890 vs 2419 tokens)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports pre-training FLOPS, inference FLOPS, and total FLOPS for key model comparisons. The compute budgets N={4, 16, 64, 256} are stated in Section 4.1. The paper explicitly compares total computational cost between small and large models."
       }
     }
   },
   "claims": [
     {
       "claim": "A 1B LLM can surpass a 405B LLM on MATH-500 using compute-optimal test-time scaling",
       "evidence": "Table 3 shows Llama-3.2-1B-Instruct with N=512 achieves 72.2% on MATH-500 vs. Llama-3.1-405B-Instruct at 71.4%. Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "A 3B LLM outperforms a 405B LLM on both MATH-500 and AIME24",
       "evidence": "Table 3 shows Llama-3.2-3B-Instruct achieves 75.6% on MATH-500 (vs 71.4%) and 30.0% on AIME24 (vs 23.3%). Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-R1-Distill-Qwen-7B with compute-optimal TTS beats o1 and DeepSeek-R1",
       "evidence": "Table 3 shows DeepSeek-R1-Distill-Qwen-7B achieves 95.2% on MATH-500 (vs o1's 94.8%) and 83.3% on AIME24 (vs o1's 79.2% and DeepSeek-R1's 79.8%). Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "Compute-optimal TTS can be 256x more efficient than majority voting",
       "evidence": "Table 5 shows efficiency gains ranging from 0.8x to >256.0x depending on policy model size. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "The compute-optimal TTS strategy depends on the choice of policy model, PRM, and problem difficulty",
       "evidence": "Figures 4-5 show different optimal methods across PRMs, Figure 7 shows dependence on policy model size, and Figures 8-9 show dependence on difficulty level. Sections 4.2-4.3.",
       "supported": "strong"
     },
     {
       "claim": "TTS is more effective than direct RL/SFT methods but less effective than distillation from strong reasoning models",
       "evidence": "Table 6 shows TTS with Qwen2.5-7B outperforms rStar-Math, Eurus-2, SimpleRL, and Satori but falls significantly behind DeepSeek-R1-Distill-Qwen-7B on AIME24 (36.7% vs 63.3%). Section 5.3.",
       "supported": "strong"
     },
     {
       "claim": "Small policy models with TTS can reduce total FLOPS by 100x-1000x compared to larger models",
       "evidence": "Table 4 shows Llama-3.2-3B total FLOPS is 1.62e23 vs Llama-3.1-405B at 3.65e25 (225x reduction), and DeepSeek-R1-Distill-7B is 7.56e23 vs DeepSeek-R1 at 5.96e25 (79x reduction). Section 5.1.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper demonstrates that compute-optimal test-time scaling (TTS) strategy depends critically on the policy model, process reward model (PRM), and problem difficulty level. Using reward-aware compute-optimal TTS, small language models can outperform much larger ones on mathematical reasoning benchmarks: a 3B model surpasses a 405B model on MATH-500 and AIME24, and a 7B model beats o1 and DeepSeek-R1. However, TTS gains diminish for larger policy models, and TTS is less effective than distillation from strong reasoning models on hard tasks. The paper also identifies systematic PRM biases including over-criticism, error neglect, error localization bias, and scoring bias related to step length.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as single point estimates without confidence intervals, error bars, or variance across runs. Given the stochastic nature of sampling-based TTS methods (temperature=0.7), the absence of uncertainty measures makes it impossible to assess whether observed differences are statistically meaningful."
     },
     {
       "flag": "Very small test set for AIME24",
       "detail": "AIME24 contains only 30 problems. Performance differences on this dataset (e.g., 30.0% vs 23.3%, which is a difference of about 2 problems) may not be statistically significant. The paper makes strong claims based on these small differences."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MATH-500 is derived from MATH (2021). All tested models were trained after 2021 and may have seen MATH problems during pre-training. The paper does not discuss this contamination risk, which could inflate absolute performance numbers and bias comparisons between models trained on different data."
     },
     {
       "flag": "Oracle difficulty labels in compute-optimal strategy",
       "detail": "The compute-optimal TTS strategy selects different methods for different difficulty levels, but difficulty is measured using Pass@1 accuracy which requires generating many samples. This oracle information would not be available in a real deployment, limiting the practical applicability of the 'compute-optimal' strategy."
     },
     {
       "flag": "Overly broad title and claims relative to scope",
       "detail": "The title 'Can 1B LLM Surpass 405B LLM?' implies general capability, but all experiments are on mathematical reasoning only (MATH-500 and AIME24). The limitations section briefly notes this but the abstract and title do not qualify the claims to math."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on compute-optimal test-time scaling that this paper directly extends and compares against."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Key baseline reasoning model; the paper compares TTS against DeepSeek-R1's distilled models and CoT performance."
     },
     {
       "title": "Scaling Test-Time Compute with Open Models",
       "authors": ["Edward Beeching", "Lewis Tunstall", "Sasha Rush"],
       "year": 2024,
       "relevance": "Introduces Diverse Verifier Tree Search (DVTS), one of the three TTS methods evaluated in this paper."
     },
     {
       "title": "Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for Problem-Solving with Language Models",
       "authors": ["Yangzhen Wu", "Zhiqing Sun", "Shanda Li", "Sean Welleck", "Yiming Yang"],
       "year": 2024,
       "arxiv_id": "2408.00724",
       "relevance": "Studies inference scaling laws for LLM problem-solving, directly related to the test-time compute scaling investigated here."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Explores scaling inference compute through repeated sampling with domain verifiers, a closely related approach to Best-of-N TTS."
     },
     {
       "title": "Let's Verify Step by Step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yuri Burda"],
       "year": 2024,
       "relevance": "Introduces process reward models and PRM800K dataset used for training several PRMs evaluated in this paper."
     },
     {
       "title": "The Lessons of Developing Process Reward Models in Mathematical Reasoning",
       "authors": ["Zhenru Zhang", "Chujie Zheng", "Yangzhen Wu"],
       "year": 2025,
       "arxiv_id": "2501.07301",
       "relevance": "Provides guidelines for PRM development and releases the Qwen2.5-Math-PRM models evaluated as the strongest open-source PRMs in this paper."
     },
     {
       "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning",
       "authors": ["Chujie Zheng", "Zhenru Zhang", "Beichen Zhang"],
       "year": 2024,
       "arxiv_id": "2412.06559",
       "relevance": "Benchmark for evaluating PRMs' process supervision abilities, used to correlate PRM quality with TTS performance in Figure 6."
     },
     {
       "title": "rStar-Math: Small LLMs Can Master Math Reasoning with Self-Evolved Deep Thinking",
       "authors": ["Xinyu Guan", "Li Lyna Zhang", "Yifei Liu"],
       "year": 2025,
       "arxiv_id": "2501.04519",
       "relevance": "Long-CoT baseline that uses MCTS for reasoning data generation, directly compared with TTS in Table 6."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Introduces majority voting for LLM reasoning, which serves as a key baseline throughout this paper's experiments."
     },
     {
       "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs",
       "authors": ["Kimi Team"],
       "year": 2025,
       "arxiv_id": "2501.12599",
       "relevance": "Recent work on scaling reinforcement learning for LLM reasoning, representing the state of the art in internal TTS approaches."
     }
   ]
 }