ai-research-survey

scan.json (25314B)

---

 {
   "paper": {
     "title": "The Art of Scaling Test-Time Compute for Large Language Models",
     "authors": ["Aradhye Agarwal", "Ayan Sengupta", "Tanmoy Chakraborty"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.02008"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The source code is available at https://github.com/Aradhye2002/art_of_tts' in a footnote on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: AIME 2024, AIME 2025-I, AIME 2025-II, and GPQA Diamond (Rein et al., 2023). All are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements.txt, Dockerfile, or detailed dependency listing is provided. The paper mentions using API-based inference via deepinfra.com but provides no environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While source code is released, the paper itself does not contain step-by-step reproduction instructions, a README description of commands to run, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows '±1 std. deviation' shaded regions for mean accuracy vs. average completion tokens. However, the main results tables (Table 4) report only point estimates without uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'beam search performance degrades', 'LFS is always suboptimal to MV') but provides no statistical significance tests. All comparisons are based on raw accuracy differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw accuracy percentages (e.g., 83.3% vs. 73.3%) without formal effect size measures like Cohen's d, odds ratios, or systematic baseline-contextualized improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The datasets are small (AIME has 30 problems per variant, GPQA Diamond has ~198 questions) but no justification or power analysis is provided for these sample sizes. With N=8 samples per configuration and 30 test questions, the statistical power for detecting meaningful differences is limited."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows ±1 standard deviation bands for accuracy vs. token length. However, variance across experimental runs (e.g., different random seeds for sampling) is not reported in the main results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares four TTS strategies (beam search, majority voting, first finish search, last finish search) and simple decoding as baselines against each other across eight models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include recently proposed methods: FFS (Agarwal et al., 2025), short-m@k (Hassid et al., 2025), beam search, and majority voting. Models used include recent ones like GPT-OSS-120B, Qwen3, and DAPO-32B."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies k and N parameters in FFS-k@N and LFS-k@N strategies (Figures 4 and 5), effectively ablating the contributions of trace selection (k) and sample count (N) to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy, total token consumption, and sequential token consumption (Section 2.4). Both compute cost and accuracy are tracked as dual metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation of automated TTS strategies on math/science reasoning tasks with ground-truth answers. Human evaluation of model outputs is not relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard held-out benchmarks (AIME 2024, AIME 2025-I, AIME 2025-II, GPQA Diamond) that are not used for any tuning or selection decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (8 models), per dataset (4 datasets), per strategy, and per problem difficulty (easy vs. hard in Table 1). Table 4 provides per-model, per-dataset breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses beam search's inverse scaling (Section 3.1), cases where FFS degrades accuracy, and identifies that DAPO does not improve length bias over GRPO (Section 3.2). These are failure mode discussions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: beam search shows inverse scaling (Section 3.1), LFS is always suboptimal to MV (Section 4), DAPO fails to mitigate length bias as claimed (Section 3.2), and FFS can substantially hurt accuracy for some models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims three findings: (1) no single TTS strategy dominates — supported by Table 4, (2) reasoning models form short-horizon and long-horizon categories — supported by Table 1, (3) optimal TTS performance scales monotonically with compute — supported by Figures 4 and 5. All are supported in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about post-training algorithms causing different reasoning horizons (e.g., 'GRPO introduces length bias', 'distinct post-training methods give rise to varying reasoning horizons') but the evidence is observational — correlating model training method with behavior without controlled experiments isolating the training algorithm as the causal factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The Art of Scaling Test-Time Compute for Large Language Models' suggests general applicability, but the study tests only 8 open-source models on 4 reasoning datasets (math and science QA). The recipe in Table 2 is presented without bounding its applicability to these specific benchmarks and models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes short-horizon vs. long-horizon behavior to post-training algorithms (GRPO vs. GSPO) but does not discuss alternative explanations such as differences in training data, model architecture, scale, or other confounds that could explain the observed behavioral differences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names (e.g., 'DeepSeek-R1', 'QwQ-32B', 'GPT-OSS-120B', 'Qwen3-32B') without specific version identifiers, snapshot dates, or API version strings. The paper mentions using deepinfra.com for inference but provides no model version or API endpoint details."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompting models to output answers within '\\boxed{}' format but does not provide the actual prompt text used for any model or dataset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 (Appendix A) provides detailed hyperparameters for each model and dataset, including top-p, temperature, max token limits, beam width, number of samples, and answer-reserve settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates standard decoding strategies (beam search, majority voting, FFS, LFS) applied to model outputs without any scaffolding infrastructure."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes the datasets, formatting (answers within '\\boxed{}'), and evaluation approach. Section 2.5 describes how problem difficulty is measured. The data pipeline from benchmark to evaluation is clear."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section 6) summarizes findings but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings/models/tasks are excluded from the conclusions. The recipe in Table 2 is presented without boundary conditions on its applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to have generated 'over thirty billion tokens' across experiments, but raw model outputs, individual trace data, or per-question results are not released. Only aggregated accuracy values are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data collection procedure: models are evaluated via API (deepinfra.com), with N=8 samples per configuration, using specified hyperparameters (Table 3), on four named datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard public benchmarks and LLM APIs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is clear: sample N traces from each model via API → extract answers from \\boxed{} → apply strategy (BS/MV/FFS/LFS) → compute accuracy. Algorithms 1 and 2 formalize the FFS and LFS strategies."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper. The first author is affiliated with Microsoft Research, which likely provides funding, but this is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Microsoft Research and Indian Institute of Technology Delhi. These are stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The first author is from Microsoft Research. Microsoft has a financial interest in LLM performance and test-time compute strategies. No discussion of potential conflict is provided. The paper does not evaluate Microsoft models directly but the funder's independence is unclear given no funding disclosure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates eight LLMs on benchmarks but does not state any model's training data cutoff date. This is important because AIME 2024 problems could be in training data of models trained after 2024."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. AIME 2024 problems have been publicly available since early 2024, and some models may have been trained on data including these problems."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPQA Diamond was published in 2023 and AIME 2024 problems have been publicly available. Many of the models tested (e.g., Qwen3, GPT-OSS-120B) were likely trained after these benchmarks were published. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption is reported in two ways: total tokens and sequential tokens (Section 2.4). Table 4 reports per-model, per-strategy token costs. The paper notes generating 'over thirty billion tokens' total."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While token counts are reported, the total computational budget in terms of API cost (dollars spent on deepinfra.com), GPU hours, or wall-clock time is not stated. The paper claims 'over thirty billion tokens' were generated but does not quantify the associated cost."
       }
     }
   },
   "claims": [
     {
       "claim": "No single TTS strategy universally dominates across all models and tasks.",
       "evidence": "Table 4 shows different strategies (BS, MV, FFS, LFS) winning on different model-dataset combinations. MV leads most often but not always. Section 3 and Appendix B provide detailed results.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning models can be categorized into short-horizon and long-horizon types based on trace-length-accuracy correlation.",
       "evidence": "Table 1 shows that short-horizon models (R1, DAPO-32B, QwQ-32B) have higher accuracy on shorter traces regardless of difficulty, while long-horizon models (GPT-OSS-120B, Qwen3-32B) prefer longer traces for hard problems.",
       "supported": "moderate"
     },
     {
       "claim": "Beam search shows inverse or no scaling for reasoning-focused datasets.",
       "evidence": "Section 3.1 and Figure 1 show that accuracy degrades or remains flat as beam width increases for short-horizon and non-reasoning models across AIME and GPQA.",
       "supported": "moderate"
     },
     {
       "claim": "LFS is always suboptimal to majority voting at the same compute cost.",
       "evidence": "Section 4 states 'the maximum performance for a given amount of total compute is always achieved when k is large (which implies k=N)' for LFS, and k=N is simply MV-N. Supported by Figure 5.",
       "supported": "moderate"
     },
     {
       "claim": "DAPO induces length bias to the same extent as GRPO.",
       "evidence": "Table 1 shows DAPO-32B has similar short-trace preference patterns to R1 (GRPO-trained). Section 3.2 notes 'any improvements in mitigating length bias over GRPO may be limited under our evaluation.'",
       "supported": "weak"
     },
     {
       "claim": "The optimal TTS strategy is independent of task difficulty.",
       "evidence": "Table 2 shows the recipe is the same regardless of 'High/Low' difficulty for each model family. However, this is derived from aggregating results over only 4 datasets with a binary easy/hard split.",
       "supported": "weak"
     },
     {
       "claim": "The choice of post-training strategy plays a key role in determining a reasoning model's effective horizon.",
       "evidence": "Section 1 correlates GRPO training with short-horizon behavior and GSPO with long-horizon behavior. However, the evidence is observational with only a few models per category and no controlled experiment.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This large-scale empirical study of test-time scaling (TTS) across eight LLMs and four reasoning benchmarks finds that no single TTS strategy universally dominates. The paper introduces a taxonomy of reasoning models into 'short-horizon' (preferring shorter traces, typically GRPO-trained) and 'long-horizon' (benefiting from longer traces on hard problems) categories. Beam search shows inverse scaling for reasoning tasks, and longest-trace filtering (LFS) is consistently suboptimal compared to majority voting. The authors provide a practical decision matrix for selecting TTS strategies based on model family and compute budget.",
   "red_flags": [
     {
       "flag": "Small benchmark sizes without power analysis",
       "detail": "AIME datasets have only 30 problems each. With N=8 samples and binary accuracy on 30 questions, differences between strategies may not be statistically meaningful. No significance tests or power analysis are provided."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (beam search degrades, LFS is suboptimal to MV, DAPO has same length bias as GRPO) are based on comparing raw accuracy percentages without any statistical testing. With small test sets, observed differences could easily be due to chance."
     },
     {
       "flag": "Causal claims from observational evidence",
       "detail": "The paper attributes short-horizon vs. long-horizon behavior to post-training algorithms (GRPO vs. GSPO) based on a small observational sample of models. These models differ in many ways beyond training algorithm (architecture, data, scale, etc.), making the causal claim unjustified."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Multiple benchmarks used (AIME 2024, GPQA Diamond) were publicly available before several tested models were trained. No analysis of potential benchmark contamination is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no dedicated limitations or threats-to-validity section, despite making broad claims about optimal TTS strategies based on a narrow set of benchmarks and models."
     },
     {
       "flag": "Broad title and claims vs. narrow evaluation",
       "detail": "The paper is titled 'The Art of Scaling Test-Time Compute for Large Language Models' and presents a general 'recipe', but evaluation is limited to math competition problems and science QA — two specific reasoning domains that may not generalize to other LLM applications."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on test-time compute scaling showing compute-optimal allocation outperforms model size scaling, directly relevant to LLM evaluation methodology."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "arxiv_id": "2203.11171",
       "relevance": "Introduces self-consistency (majority voting over reasoning paths) as a core TTS strategy for improving LLM reasoning accuracy."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational chain-of-thought prompting paper, establishing the basis for sequential test-time scaling in LLMs."
     },
     {
       "title": "s1: Simple test-time scaling",
       "authors": ["Niklas Muennighoff", "Zitong Yang", "Weijia Shi"],
       "year": 2025,
       "arxiv_id": "2501.19393",
       "relevance": "Proposes fine-tuning models for self-correction at test-time, a key sequential scaling approach evaluated in the survey scope."
     },
     {
       "title": "Don't overthink it: preferring shorter thinking chains for improved LLM reasoning",
       "authors": ["Michael Hassid", "Gabriel Synnaeve", "Yossi Adi", "Roy Schwartz"],
       "year": 2025,
       "arxiv_id": "2505.17813",
       "relevance": "Proposes short-m@k strategy showing shorter reasoning traces often outperform longer ones, directly relevant to test-time compute efficiency."
     },
     {
       "title": "Inverse scaling in test-time compute",
       "authors": ["Aryo Pradipta Gema", "Alexander Hägele"],
       "year": 2025,
       "arxiv_id": "2507.14417",
       "relevance": "Demonstrates that more test-time compute can degrade accuracy, a critical finding for LLM scaling methodology."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Edward Berman"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Introduces verbal self-reflection for sequential scaling in LLM agents, relevant to agentic AI evaluation."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Proposes structured search-based test-time scaling for LLMs, a key method in the TTS landscape relevant to agentic reasoning."
     },
     {
       "title": "GPQA: A graduate-level Google-proof Q&A benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland"],
       "year": 2023,
       "arxiv_id": "2311.12022",
       "relevance": "Key reasoning benchmark used to evaluate LLM capabilities, relevant to benchmark evaluation methodology."
     },
     {
       "title": "DAPO: An open-source LLM reinforcement learning system at scale",
       "authors": ["Qiying Yu", "Zheng Zhang", "Ruofei Zhu"],
       "year": 2025,
       "arxiv_id": "2503.14476",
       "relevance": "Open-source RL training system for LLMs, relevant to understanding how post-training affects model behavior and evaluation."
     },
     {
       "title": "A survey on test-time scaling in large language models: What, how, where, and how well?",
       "authors": ["Qiyuan Zhang", "Fuyuan Lyu", "Zexu Sun"],
       "year": 2025,
       "arxiv_id": "2503.24235",
       "relevance": "Comprehensive survey of TTS methods and their evaluation, directly relevant to systematic review of LLM scaling methodology."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yura Burda"],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Process reward model for step-level verification in reasoning, a key component of test-time scaling evaluation methodology."
     }
   ]
 }