ai-research-survey

scan.json (29733B)

---

 {
   "paper": {
     "title": "T1: Advancing Language Model Reasoning through Reinforcement Learning and Inference Scaling",
     "authors": [
       "Zhenyu Hou",
       "Xin Lv",
       "Rui Lu",
       "Jiajie Zhang",
       "Yujiang Li",
       "Zijun Yao",
       "Juanzi Li",
       "Jie Tang",
       "Yuxiao Dong"
     ],
     "year": 2025,
     "venue": "Proceedings of the 42nd International Conference on Machine Learning (ICML 2025), PMLR 267",
     "arxiv_id": "2501.11651"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'The model weights and the data for SFT and RL training are publicly available at https://github.com/THUDM/T1.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Training data sourced from publicly available datasets (MATH-train, NuminaMath) and the paper states that the SFT and RL training data are publicly available at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using SGLANG as the inference engine (with a reference) and provides learning rates and sampling parameters, but does not provide a requirements.txt, Dockerfile, or equivalent environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides training hyperparameters in the appendix (Section A.1) but does not include step-by-step reproduction instructions or README-level commands that a researcher could follow to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 are reported as single accuracy values (e.g., '92.4') with no confidence intervals, error bars, or other uncertainty quantification. AIME is evaluated 32 times and averaged, but no variance measure is reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., T1 outperforms QwQ-32B-Preview on MATH500, AIME2024, and Omni-MATH-500) without any statistical significance tests. All comparisons are based on point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage improvements with baseline context (e.g., 'over a 10% improvement on MATH-500 and over a 20% improvement on AIME over its T1-SFT version'), providing enough context to assess magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "AIME uses only 30 problems (evaluated 32 times for stability). No power analysis or justification for why these sample sizes are sufficient for the claims made is provided. Omni-MATH-500 uses 500 examples described as 'efficient yet comprehensive' without statistical justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "AIME is evaluated 32 times and averaged for stability, but no standard deviation or confidence interval is reported for these repeated evaluations. All other benchmarks appear to report single-run results only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes multiple baselines: GPT-4o, Claude-3.5-sonnet, Llama-3.3-70B-Instruct, Qwen2.5-Math-7B-Instruct, o1-preview, QwQ-32B-preview, and the base models (GLM-4-9B-chat, Qwen2.5-14B-Instruct, Qwen2.5-32B-Instruct) plus T1-SFT ablations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary systems such as QwQ-32B-Preview, o1-preview, Claude-3.5-Sonnet, and GPT-4o, all of which represent the state of the art at the time of writing in late 2024/early 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation studies on: number of sampled responses K (Table 2, Figure 3), sampling temperature (Table 2), penalty rewards (Table 3), and comparison of T1-SFT vs T1 (full RL). Individual components are isolated and their contribution measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported across four benchmarks: MATH500, AIME2024, Omni-MATH-500, and GPQA. The paper also tracks response length, KL divergence, and reward as secondary metrics during training."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper for mathematical reasoning. Human evaluation of model outputs is not applicable; correctness is determined objectively by matching against ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation is performed on standard test sets (AIME2024, MATH500 test split, Omni-MATH-500, GPQA). The training data uses MATH-train and NuminaMath; test splits are separate. Appendix A.1 states: 'We split around 12k for the SFT stage and the others for RL training.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across four different benchmark types (competition math AIME, olympiad math Omni-MATH, school math MATH500, and science GPQA), and Figure 7 shows inference scaling behavior separately for each dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a case study of a successfully solved AIME problem but does not discuss failure cases or error analysis of where T1 performs poorly. No examples of incorrect reasoning or failure modes are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that training with min-p sampling 'can make the training process more prone to collapse' (Table 2, temperature=1.2 with min-p=0.05 shows degradation from 86.4 to 78.8 on MATH500), and that low temperatures (<=1.0) often cause training collapse."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that T1 'achieves superior performance on challenging math reasoning benchmarks' and 'exhibits inference scaling behavior.' Table 1 shows T1 outperforming all baselines including QwQ-32B-Preview, and Figures 6-7 demonstrate inference scaling behavior. Claims are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (e.g., 'sampling more responses encourages exploration') and uses controlled single-variable manipulation (varying only K in Table 2 / Figure 3, varying only temperature in Table 2, varying only penalty in Table 3). This design is adequate for causal inference within the system."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on mathematical reasoning benchmarks (MATH, AIME, Omni-MATH) and GPQA science problems but the abstract and conclusion make broader claims about 'enhancing large language models' reasoning capabilities' without adequately bounding the scope. The paper acknowledges OOD generalization to GPQA but does not bound the broader generalization claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance improvements. For example, it does not consider whether the improvement could be due to training on overlapping data, or whether the reward signal design alone (vs. the exploration strategies) accounts for the gains."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies exact model versions: GLM-4-9B, Qwen2.5-14B, Qwen2.5-32B, Qwen2.5-32B-Instruct, QwQ-32B-Preview, Llama-3.3-70B-Instruct. These are specific enough to identify the models used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes in natural language how the LLM is prompted to 'examine each attempt' and 'incorporate refined attempts into a single output' but does not provide the actual prompt text used. Section 2.2.1 describes the procedure without giving the actual prompts sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports: SFT learning rate 1e-5, RL learning rate 1.5e-6, KL coefficient 2e-4, K=64 responses, temperature=1.2, top-p=0.95, max generation length 10,240 or 16,384, EMA decay, entropy bonus coefficient. This is sufficient to understand the training setup."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "T1 is a training methodology (SFT + RL), not an agentic scaffolding system. There is no external scaffolding; the model generates reasoning chains end-to-end within a single forward pass (with truncation for analysis)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes data preprocessing: answer extraction from NuminaMath using LLM, data filtering to retain only instances with pass rate in (0, 0.3), generating 16 responses per question, resulting in 30k RL training examples. The filtering criteria and resulting dataset size are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has an 'Impact Statement' section but no dedicated Limitations or Threats to Validity section. The Impact Statement is generic and does not discuss methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Impact Statement is generic ('ethical aspects and societal implications of our work align with those commonly associated with advancing the field of machine learning') and the conclusion does not discuss threats or limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it notes that T1 is 'primarily optimized for math-related tasks,' it does not explicitly bound the scope or enumerate what claims are not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Model weights and training data are released at https://github.com/THUDM/T1. The evaluation benchmarks (AIME2024, MATH, GPQA, Omni-MATH) are publicly available, enabling independent re-evaluation."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes data collection: training data from MATH-train and NuminaMath public datasets, with the answer extraction and filtering pipeline described (16 responses generated per question, retaining pass rate in (0, 0.3), yielding 30k examples)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Training and evaluation data come from publicly available mathematical datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Appendix A.1: raw data (NuminaMath + MATH-train) → answer extraction via LLM → data filtering (pass rate criterion) → 12k SFT split + 30k RL split. Transformation steps and resulting sizes are stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses funding from NSFC 62495063, Tsinghua University-Siemens Joint Research Center (JCIIOT), and the New Cornerstone Science Foundation (XPLORER PRIZE)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Tsinghua University (Hou, Lu, Zhang, Li, Yao, Li, Tang, Dong) and ZhipuAI (Lv). Affiliations are disclosed in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSFC, Tsinghua-Siemens center, New Cornerstone Science Foundation) do not have a direct financial stake in the LLM reasoning performance results. One author is from ZhipuAI (maker of GLM models), which is a minor conflict since GLM-4-9B is one of three base models used."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. One co-author (Xin Lv) is affiliated with ZhipuAI, whose GLM-4-9B model is evaluated in the paper, but this potential conflict is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoffs for the comparison models (GPT-4o, Claude-3.5-Sonnet, QwQ-32B-Preview, o1-preview). The T1 model itself trains on MATH-train and NuminaMath, but the pre-training cutoffs of the base models (Qwen2.5, GLM-4) are not specified in the paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential overlap between training data (MATH-train, NuminaMath) and evaluation benchmarks (MATH500 is a subset of MATH-test, AIME2024 is from 2024). No analysis of whether test examples could have been in training data is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH benchmark (Hendrycks et al., 2021) was published in 2021; the base models (Qwen2.5, GLM-4) could have seen it during pre-training. AIME2024 problems are newer but NuminaMath may contain similar problems. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Randomization of participants is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports token counts for inference scaling analysis (up to ~6000 thinking tokens) but does not report API costs, wall-clock time, or cost per example for the T1 system."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report GPU hours, hardware used, or total compute budget for training T1. Training details (learning rates, steps) are given but not the compute required."
       }
     }
   },
   "claims": [
     {
       "claim": "T1 (Qwen2.5-32B) achieves superior performance on MATH500 (92.4%), AIME2024 (50.6%), and Omni-MATH-500 (49.6%), outperforming QwQ-32B-Preview (90.6%, 50.0%, 46.6%) and o1-preview (85.5%, 44.6%, /) across benchmarks.",
       "evidence": "Table 1 provides accuracy numbers for all compared models across MATH500, AIME2024, Omni-MATH-500, and GPQA benchmarks. T1 (Qwen2.5-32B) achieves the highest scores on MATH500 and Omni-MATH-500.",
       "supported": "moderate"
     },
     {
       "claim": "T1 exhibits inference scaling behavior: longer generation (more thinking tokens) directly leads to better performance without requiring external verification.",
       "evidence": "Figure 6 shows accuracy increasing monotonically from 24% to 50% on AIME as average thinking tokens increase from ~500 to ~6000. Figure 7 shows similar trends across training steps and max generation lengths.",
       "supported": "strong"
     },
     {
       "claim": "Sampling more responses (higher K) during RL training improves performance: K=64 yields substantially better results than K=4, with a >6% improvement.",
       "evidence": "Figure 3 and Figure 4 show that K=64 outperforms K=16 and K=4 on MATH500 and other benchmarks. Table text states 'sampling only 4 responses shows little to no benefit (around 3%)'.",
       "supported": "strong"
     },
     {
       "claim": "High sampling temperature (1.2) during RL training is beneficial, and training with temperature ≤1.0 often collapses.",
       "evidence": "Table 2 shows performance peaks at temperature=1.2 (86.4 MATH500, 29.3 AIME, 38.6 Omni-MATH-500) compared to 0.9 (78.2, 19.1, 32.0) and 1.3 (84.6, 24.3, 36.4). Training instability at lower temperatures is reported in text.",
       "supported": "strong"
     },
     {
       "claim": "Penalty for unexpected patterns (repetition, overlong text, garbage text) prevents training collapse and is necessary for stable training.",
       "evidence": "Table 3 shows that without penalty, overlong ratio reaches 16.3% by step 120 and training diverges (accuracy drops from 79.2 to 76.4 and training stops), while penalized training maintains stability.",
       "supported": "strong"
     },
     {
       "claim": "RL training (not just SFT) is necessary to unlock inference scaling: SFT-only models show minimal gains with increased inference budgets.",
       "evidence": "Figure 7 shows that T1-SFT and 30% RL step models show flat accuracy curves as max_gen_length increases, while 100% RL trained models show steep improvement from ~1024 to ~16384 tokens.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "T1 is a reinforcement learning training pipeline for LLMs that promotes exploration through oversampling (K=64), high-temperature sampling, entropy bonuses, and on-policy KL normalization, combined with penalties for degenerate outputs. Applied to Qwen2.5-32B, T1 achieves state-of-the-art results on math reasoning benchmarks including MATH500 (92.4%) and AIME2024 (50.6%), outperforming QwQ-32B-Preview. A key finding is that T1-trained models exhibit inference scaling behavior — performance improves monotonically as more thinking tokens are generated — and that this scaling property requires sufficient RL training (SFT alone does not enable it). The paper introduces a truncation-based method to measure inference scaling without requiring external verifiers.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results in Table 1 are point estimates with no confidence intervals, error bars, or standard deviations. While AIME is evaluated 32 times for stability, the spread is not reported. Given AIME has only 30 problems, even the average of 32 runs has substantial variance that is not quantified."
     },
     {
       "flag": "No contamination analysis",
       "detail": "MATH benchmark (2021) could be in the training data of the base models (Qwen2.5, GLM-4). The paper uses MATH-train for T1 training but evaluates on MATH-test/MATH500, which is methodologically sound for T1 itself, but the comparison baselines (QwQ-32B-Preview, o1-preview) may have seen all MATH problems during pre-training. This is not discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section. The Impact Statement is purely generic ('ethical aspects and societal implications align with those commonly associated with advancing the field of machine learning') and does not discuss methodological limitations, scope boundaries, or threats to validity."
     },
     {
       "flag": "Generalization claims outrun evidence",
       "detail": "The paper claims T1 'enhances reasoning capabilities' broadly, but evaluation is limited to mathematical reasoning (MATH, AIME, Omni-MATH) and GPQA. Generalization to code generation, commonsense reasoning, or other reasoning domains is not tested."
     },
     {
       "flag": "Compute budget not reported",
       "detail": "Training 32B-parameter models with K=64 sampled responses per prompt using RL is computationally expensive, but the paper reports no GPU hours, hardware used, or total compute cost, making it difficult to assess the practical feasibility of reproducing or extending this work."
     },
     {
       "flag": "Affiliation conflict not declared",
       "detail": "Co-author Xin Lv is affiliated with ZhipuAI, whose GLM-4-9B model is one of three models used as a base in T1 experiments. While the affiliation is disclosed, no competing interests statement is made regarding this connection."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective than Scaling Model Parameters",
       "authors": [
         "Charlie Snell",
         "Jaehoon Lee",
         "Kelvin Xu",
         "Aviral Kumar"
       ],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Key reference for test-time scaling of LLMs; directly relevant to inference scaling claims made in T1."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": [
         "Bradley Brown",
         "Jordan Juravsky",
         "Ryan Ehrlich",
         "Ronald Clark",
         "Quoc V. Le",
         "Christopher Ré",
         "Azalia Mirhoseini"
       ],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Baseline approach for inference scaling via repeated sampling; contrasted with T1's single long-response approach."
     },
     {
       "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
       "authors": [
         "Zhihong Shao",
         "Peiyi Wang",
         "Qihao Zhu",
         "Runxin Xu",
         "Junxiao Song",
         "Mingchuan Zhang",
         "Y. K. Li",
         "Y. Wu",
         "Daya Guo"
       ],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Important prior work on RL for math reasoning in LLMs; baseline for group relative policy optimization (GRPO) that T1 builds upon."
     },
     {
       "title": "Does RLHF Scale? Exploring the Impacts from Data, Model, and Method",
       "authors": [
         "Zhenyu Hou",
         "P. Du",
         "Y. Niu",
         "Z. Du",
         "A. Zeng",
         "X. Liu",
         "M. Huang",
         "H. Wang",
         "J. Tang",
         "Y. Dong"
       ],
       "year": 2024,
       "arxiv_id": "2412.06000",
       "relevance": "Prior work by the same group on RLHF scaling, directly motivating T1's exploration of RL scaling for reasoning."
     },
     {
       "title": "STaR: Self-Taught Reasoner",
       "authors": [
         "Eric Zelikman",
         "Yuhuai Wu",
         "Noah D. Goodman"
       ],
       "year": 2022,
       "arxiv_id": "2203.14465",
       "relevance": "Foundational work on self-taught reasoning that T1's CoT initialization approach extends."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": [
         "Jason Wei",
         "Xuezhi Wang",
         "Dale Schuurmans",
         "Maarten Bosma",
         "Brian Ichter",
         "Fei Xia",
         "Ed H. Chi",
         "Quoc V. Le",
         "Denny Zhou"
       ],
       "year": 2022,
       "relevance": "Foundational CoT paper; central to T1's reasoning paradigm."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": [
         "Long Ouyang",
         "Jeff Wu",
         "Xu Jiang",
         "Diogo Almeida",
         "Carroll L. Wainwright",
         "Pamela Mishkin",
         "Chong Zhang",
         "Sandhini Agarwal",
         "Katarina Slama",
         "Alex Ray",
         "et al."
       ],
       "year": 2022,
       "relevance": "Original RLHF paper; T1's RL training framework is built on RLHF foundations."
     },
     {
       "title": "Qwen2.5 Technical Report",
       "authors": [
         "A. Yang",
         "B. Yang",
         "B. Zhang",
         "et al."
       ],
       "year": 2024,
       "arxiv_id": "2412.15115",
       "relevance": "Base model used in T1 experiments; Qwen2.5-14B and Qwen2.5-32B are the primary models evaluated."
     },
     {
       "title": "Omni-MATH: A Universal Olympiad Level Mathematic Benchmark for Large Language Models",
       "authors": [
         "B. Gao",
         "F. Song",
         "Z. Yang",
         "Z. Cai",
         "Y. Miao",
         "Q. Dong",
         "L. Li",
         "C. Ma",
         "L. Chen",
         "R. Xu",
         "et al."
       ],
       "year": 2024,
       "arxiv_id": "2410.07985",
       "relevance": "Primary evaluation benchmark used in T1; Olympiad-level math problems for assessing LLM reasoning."
     },
     {
       "title": "Measuring Mathematical Problem Solving With the MATH Dataset",
       "authors": [
         "Dan Hendrycks",
         "Collin Burns",
         "Saurav Kadavath",
         "Akul Arora",
         "Steven Basart",
         "Eric Tang",
         "Dawn Song",
         "Jacob Steinhardt"
       ],
       "year": 2021,
       "relevance": "Core evaluation benchmark (MATH500) and source of training data; critical for assessing benchmark contamination risks."
     },
     {
       "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
       "authors": [
         "David Rein",
         "Betty Li Hou",
         "Asa Cooper Stickland",
         "Jackson Petty",
         "Richard Yuanzhe Pang",
         "Julien Dirani",
         "Julian Michael",
         "Samuel R. Bowman"
       ],
       "year": 2023,
       "arxiv_id": "2311.12022",
       "relevance": "Out-of-domain evaluation benchmark testing generalization of T1's math-trained reasoning to graduate-level science."
     },
     {
       "title": "Math-Shepherd: Verify and Reinforce LLMs Step-by-Step without Human Annotations",
       "authors": [
         "Peiyi Wang",
         "Lei Li",
         "Zhihong Shao",
         "Runxin Xu",
         "Damai Dai",
         "Yunfei Li",
         "Daya Chen",
         "Yu Wu",
         "Zhifang Sui"
       ],
       "year": 2024,
       "relevance": "Related work on process reward models for RL training of math reasoning LLMs."
     },
     {
       "title": "VinePPO: Unlocking RL Potential for LLM Reasoning through Refined Credit Assignment",
       "authors": [
         "Amirhossein Kazemnejad",
         "Milad Aghajohari",
         "Eva Portelance",
         "Alessandro Sordoni",
         "Siva Reddy",
         "Aaron Courville",
         "Nicolas Le Roux"
       ],
       "year": 2024,
       "arxiv_id": "2410.01679",
       "relevance": "Related RL approach for LLM reasoning with credit assignment; comparison baseline context for T1."
     },
     {
       "title": "Training Language Models to Self-Correct via Reinforcement Learning",
       "authors": [
         "Aviral Kumar",
         "Vincent Zhuang",
         "Rishabh Agarwal",
         "Yi Su",
         "John D. Co-Reyes",
         "Avi Singh",
         "Kate Baumli",
         "Shariq Iqbal",
         "Colton Bishop",
         "Rebecca Roelofs",
         "et al."
       ],
       "year": 2024,
       "arxiv_id": "2409.12917",
       "relevance": "Related work on RL for self-correction in LLMs; closely related to T1's verification and trial-and-error approach."
     }
   ]
 }