ai-research-survey

scan.json (23449B)

---

 {
   "paper": {
     "title": "OpenR: An Open Source Framework for Advanced Reasoning with Large Language Models",
     "authors": [
       "Jun Wang",
       "Meng Fang",
       "Ziyu Wan",
       "Muning Wen",
       "Jiachen Zhu",
       "Anjie Liu",
       "Ziqin Gong",
       "Yan Song",
       "Lei Chen",
       "Lionel M. Ni",
       "Linyi Yang",
       "Ying Wen",
       "Weinan Zhang"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.09671"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenR is an open-source framework integrating process reward models (PRMs), reinforcement learning, and test-time guided search to enhance LLM reasoning. On the MATH500 benchmark, their PRM (Math-psa) combined with best-of-N or beam search achieves ~82% accuracy, approximately 10% relative improvement over majority voting. Online RL training with PPO and PRM feedback shows steady reward improvement on individual problems but exhibits instability on the full dataset.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The OpenR framework, including code, models, and datasets, is accessible at https://openreasoner.github.io' in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases the MATH-APS dataset and states code, models, and datasets are publicly available. They also use public datasets (PRM800K, Math-Shepherd, MATH)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper points to the website but does not include specific commands or procedures to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figure 4 show point estimates of accuracy with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims their PRM outperforms Math-Shepherd and that search methods outperform majority vote, but no significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'approximately 10%' relative improvement and shows accuracy comparisons (e.g., ~75% for majority vote vs ~82% for best methods) providing baseline context for effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "They use MATH500 (500 problems) following Lightman et al. but do not justify why 500 is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. The PPO training curves in Figure 5 show single runs with no indication of variance across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Majority voting is used as a baseline, and Math-Shepherd PRM is compared against their Math-psa PRM in Figure 4b."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Math-Shepherd (Wang et al., 2024a) and Qwen2.5-Math models (Yang et al., 2024) are contemporary baselines from 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple search methods (best-of-N vs beam search), voting strategies (majority vote, PRM-last-vote, PRM-last-max), and different PRMs, which functions as component ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy on MATH500 is reported as the evaluation metric. No secondary metrics are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for automated math benchmark evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MATH500 is a held-out subset of 500 randomly sampled problems from MATH, following Lightman et al. (2023), used specifically for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The MATH dataset has difficulty levels and problem categories, but no per-category or per-difficulty breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figures 6-12 provide case studies including incorrect answers (Cases 2, 3, 5) and show where reasoning goes wrong, with PRM score comparisons."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows PPO training on MATH500 exhibits 'more fluctuations in rewards' and the paper acknowledges 'the need for further improvements on the algorithm to enhance its adaptability for diverse problem sets.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'substantial gains' with test-time computation and RL through PRMs, which is supported by Figure 4 showing ~10% improvement and Figure 5 showing RL training progress."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (PRM improves reasoning, RL training improves accuracy) supported by controlled comparisons: same model with different search strategies (Figure 4a), same strategy with different PRMs (Figure 4b), before/after RL training (Figures 8-9)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Advanced Reasoning with Large Language Models' broadly, but experiments are limited to the MATH dataset with specific Qwen2.5 models. The abstract frames results as general LLM reasoning improvements."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed improvements. For example, the PRM advantage could stem from dataset differences rather than training methodology."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on MATH500 but frames contributions as 'enhancing reasoning capabilities' broadly. No discussion of whether MATH500 accuracy is an adequate proxy for general reasoning ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Qwen2.5-Math-7B-Instruct' for PRM training and 'Qwen2.5-1.5B-Math-Instruct' for policy learning (Sections 4.1 and 4.2)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as a primary methodology — it trains models via RL and supervised fine-tuning, and uses the models for generation and scoring."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters for RL training (learning rate, batch size, number of epochs, PPO clipping parameter, temperature for generation) are not reported. The data augmentation section mentions constants α, β, L, cpuct without specifying their values."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a framework for RL training and guided decoding, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the MATH-APS data augmentation pipeline using OmegaPRM, including tree construction, rollout selection via PUCT, and binary search for error identification. Section 4.1 notes the dataset was 'reduced to approximately 150k pairs after cleaning and preprocessing.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' explicitly discusses limited scale of experiments, model size constraints, and limited process supervision data."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific issues: 'restricted access to large-scale computing infrastructure,' evaluations on 'relatively smaller models,' and 'the scale and diversity of process supervision data remain limited.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While limitations are listed, the paper does not explicitly state what the results do NOT show or what settings/populations are excluded. No explicit boundary on what claims are not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states datasets are publicly available at the project website, and they use publicly available datasets (PRM800K, Math-Shepherd, MATH)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the MATH-APS data collection procedure in detail, including the tree construction method, PUCT selection, Monte Carlo estimation, and binary search for error identification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from standard benchmarks and automated generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the pipeline from PRM800K + Math-Shepherd + MATH-APS generation through to PRM training (Section 3.3) and policy learning (Section 3.4). The data reduction from 500k to 150k pairs is noted."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UCL, University of Liverpool, Shanghai Jiao Tong University, HKUST (Guangzhou), and Westlake University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-Math models but does not state the training data cutoff date for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "MATH is a well-known benchmark published in 2021. Qwen2.5 models trained in 2024 could have seen MATH problems. No discussion of potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH was published in 2021, well before Qwen2.5's training. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports generation budget in terms of average tokens per question (Figure 4) but does not report wall-clock time, API costs, or actual compute cost for inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Figure 5 shows wall-clock hours for PPO training (~12 hours) but no GPU type, total GPU hours, or compute budget for PRM training or data generation is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for any experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for PRM training or RL training."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares search methods and PRMs but does not explain how the best configuration was selected or whether selection was done on validation data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their Math-psa PRM against Math-Shepherd without acknowledging potential bias from evaluating their own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 plots accuracy vs generation budget (tokens per question), directly showing performance as a function of compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "MATH is used without discussing whether competition-level math problems are an adequate measure of 'advanced reasoning capabilities' as claimed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; the comparisons are between search/voting strategies applied to the same models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH (2021) predates Qwen2.5 (2024) training. No discussion of whether MATH solutions appeared in Qwen2.5's training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through the PRM or search process."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The PRM is trained on PRM800K and Math-Shepherd data derived from MATH. No discussion of whether this creates non-independence with the MATH500 test set."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are described."
       }
     }
   },
   "claims": [
     {
       "claim": "Process reward models combined with guided search improve test-time reasoning performance by approximately 10% on MATH500.",
       "evidence": "Figure 4a shows accuracy increasing from ~75% (majority vote) to ~82% (best PRM-guided methods) on MATH500.",
       "supported": "moderate"
     },
     {
       "claim": "Their Math-psa PRM achieves higher test accuracy than Math-Shepherd PRM across all computation budgets.",
       "evidence": "Figure 4b shows Math-psa consistently outperforming Math-Shepherd in best-of-N comparisons.",
       "supported": "weak"
     },
     {
       "claim": "PPO with PRM feedback steadily improves reasoning accuracy on individual math problems.",
       "evidence": "Figure 5 (left) shows reward increasing and stabilizing after ~6 hours of training on a single problem.",
       "supported": "weak"
     },
     {
       "claim": "OpenR is the first open-source framework to explore core methods of OpenAI's o1 model with reinforcement learning.",
       "evidence": "Stated in abstract and Section 1. No systematic comparison with other open-source RL-for-reasoning frameworks.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run results without variance",
       "detail": "All experiments appear to be single runs with no error bars, standard deviations, or confidence intervals. Results could vary substantially across seeds."
     },
     {
       "flag": "No contamination analysis",
       "detail": "MATH (2021) is well within Qwen2.5's training window. The base model may already know many MATH solutions, inflating reported accuracy for all methods equally and potentially biasing PRM comparisons."
     },
     {
       "flag": "PRM trained on MATH-derived data, tested on MATH",
       "detail": "The PRM is trained on PRM800K, Math-Shepherd, and MATH-APS — all derived from the MATH dataset. Testing on MATH500 (a subset of MATH) creates potential data leakage. No analysis of overlap between PRM training data and MATH500 test problems."
     },
     {
       "flag": "Overclaiming scope",
       "detail": "The paper claims to provide a framework for 'Advanced Reasoning with Large Language Models' but only evaluates on a single math benchmark (MATH500) with specific Qwen2.5 models."
     },
     {
       "flag": "Labeled 'Technical Report: Work in progress'",
       "detail": "The paper self-describes as a work-in-progress technical report, yet makes broad claims about being 'the first open-source framework' for o1-style reasoning."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Core reference for test-time compute scaling, directly replicated in OpenR's experiments."
     },
     {
       "title": "Let's Verify Step by Step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yura Burda"],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Foundational work on process reward models and PRM800K dataset used in this framework."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational chain-of-thought prompting work that motivated o1-style reasoning approaches."
     },
     {
       "title": "Improve Mathematical Reasoning in Language Models by Automated Process Supervision",
       "authors": ["Liangchen Luo", "Yinxiao Liu", "Rosanne Liu"],
       "year": 2024,
       "arxiv_id": "2406.06592",
       "relevance": "OmegaPRM method used for automated process supervision data collection in MATH-APS."
     },
     {
       "title": "Math-Shepherd: Verify and Reinforce LLMs Step-by-Step without Human Annotations",
       "authors": ["Peiyi Wang", "Lei Li", "Zhihong Shao"],
       "year": 2024,
       "relevance": "Baseline PRM compared against in experiments; its training data is also used for Math-psa."
     },
     {
       "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "GRPO algorithm used in OpenR's RL training pipeline."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF work that motivates RL-based training of reasoning models."
     },
     {
       "title": "Generative Verifiers: Reward Modeling as Next-Token Prediction",
       "authors": ["Lunjun Zhang", "Arian Hosseini", "Hritik Bansal"],
       "year": 2024,
       "arxiv_id": "2408.15240",
       "relevance": "Generative reward model approach discussed as alternative to scalar PRMs."
     },
     {
       "title": "AlphaZero-like Tree-Search Can Guide Large Language Model Decoding and Training",
       "authors": ["Xidong Feng", "Ziyu Wan", "Muning Wen"],
       "year": 2024,
       "relevance": "MCTS-based decoding for LLMs, directly related to OpenR's search methods."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work that OpenR extends from training-time to inference-time compute."
     }
   ]
 }