ai-research-survey

scan.json (25459B)

---

 {
   "paper": {
     "title": "A Comparative Study on Reasoning Patterns of OpenAI's o1 Model",
     "authors": [
       "Siwei Wu",
       "Zhongyuan Peng",
       "Xinrun Du",
       "Tuney Zheng",
       "Minghao Liu",
       "Jialong Wu",
       "Jiachen Ma",
       "Yizhi Li",
       "Jian Yang",
       "Wangchunshu Zhou",
       "Qunshu Lin",
       "Junbo Zhao",
       "Zhaoxiang Zhang",
       "Wenhao Huang",
       "Ge Zhang",
       "Chenghua Lin",
       "J.H. Liu"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.13639"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenAI's o1 model outperforms GPT-4o with various test-time compute methods (BoN, Step-wise BoN, Self-Refine, Agent Workflow) on math, coding, and commonsense reasoning benchmarks. The study identifies six reasoning patterns in o1 (Systematic Analysis, Method Reuse, Divide and Conquer, Self-Refinement, Context Identification, Emphasizing Constraints), with Divide and Conquer and Self-Refinement being most frequent. Reward model quality and search space both limit the upper bound of search-based methods like BoN. Agent Workflow with domain-specific prompts comes closest to o1 performance among the baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in abstract: https://github.com/Open-Source-O1/o1_Reasoning_Patterns_Study"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'code and dataset are released' at the GitHub link. The benchmarks used (HotpotQA, Collie, USACO, AIME) are publicly available, and the filtered versions are indicated as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no instructions for running experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and figures are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims o1 'achieves the best results' and various methods 'outperform' others based solely on comparing raw numbers with no statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw percentage scores are reported but no effect sizes (Cohen's d, etc.) are provided. Some percentage differences are mentioned (e.g., '2.55%') but without baseline context framing as effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The filtered benchmark sizes (274, 226, 139, 90 samples in Table 3) are stated but no justification for why these sizes are adequate is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four test-time compute baselines (BoN, Step-wise BoN, Self-Refine, Agent Workflow) and direct GPT-4o are compared against o1 in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o and state-of-the-art agent frameworks (Zhou et al., 2024), which are contemporary to the study period."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper varies N for BoN (N=1,4,8,16 in Fig. 5), tests different reward models (Section 4.5, Fig. 4), and tests different backbone models (GPT-4o, Qwen2.5, Llama3), which serve as ablation-like analyses."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is used as the evaluation metric across all benchmarks. No secondary metrics are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is used as a reward model baseline in Section 4.5 (Fig. 4), where humans judge the most suitable BoN response. Additionally, USACO code is manually run on test cases (Section 3.3)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks (HotpotQA, Collie, USACO, AIME) are used as test sets. The data filtering uses separate models (Llama3, Qwen, Yi, Claude) to select hard samples, and evaluation is done on these filtered sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (HotpotQA, Collie, USACO, AIME) across three domains in Table 1, and reasoning pattern analysis is broken down per benchmark in Fig. 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failures of Self-Refine (performance decline on Collie, Section 4.1), Step-wise BoN failures on complex tasks (Section 4.3), and BoN performance decline with increasing N."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Self-Refine shows performance decline on Collie compared to GPT-4o. Step-wise BoN drops significantly on complex benchmarks. BoN performance stabilizes or declines when N>8 (Section 4.6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about o1 achieving best performance, reward model limitations, Agent Workflow superiority over Step-wise BoN, and six reasoning patterns are all supported by results in Table 1, Figs. 1-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'the domain-specific system prompt is crucial' and reward models 'limit the upper boundary' without controlled experiments isolating these factors. The Agent Workflow comparison conflates multiple differences (domain-specific prompts, tool use, workflow structure)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to study 'Reasoning Patterns of OpenAI's o1 Model' generally, but the reasoning pattern analysis is based on manual inspection of only 20-30 samples per benchmark. The paper does not bound its claims to these specific benchmarks and sample sizes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For example, the paper does not consider whether o1's advantage comes from larger training compute, different training data, or RLHF rather than reasoning patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates benchmark accuracy with 'reasoning capability' without discussing whether these benchmarks actually capture reasoning ability vs. memorization, pattern matching, or other factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'o1-preview', 'o1-mini', and 'GPT-4o' without specifying exact API versions or snapshot dates. Open-source models mention sizes (e.g., Llama3-72B, Qwen-72B) but no exact version strings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'domain-specific system prompts' for Agent Workflow and refers to external GPTs, but no actual prompt text is provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the LLM calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The Agent Workflow uses 'state-of-the-art agent framework (Zhou et al., 2024)' and GPTs but the actual scaffolding details (tool descriptions, workflow, retry logic) are not described. The paper defers to external references."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data filtering process: using four LLMs to answer each sample, then filtering out samples that more than two models answer correctly (LIME method). Table 3 shows filtered benchmark sizes."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No scope boundaries are defined beyond the implicit choice of benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and dataset are released at the GitHub repository, which would include the filtered benchmark data and model outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes benchmark selection and the LIME-based filtering procedure. Section 3.3 describes evaluation metrics and how correctness is determined."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited. The study uses standard benchmarks and LLM APIs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw benchmarks through LIME filtering to final evaluation is documented in Sections 3.1 and 3.3, with filtered sample counts in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: M-A-P, University of Manchester, OpenO1 Team, 2077AI, Abaka AI, Zhejiang University, University of Chinese Academy of Sciences."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The OpenO1 Team affiliation suggests potential interest in o1-related research outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (o1, GPT-4o, Llama3, Qwen)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems appeared in model training data. HotpotQA (2018), AIME problems, and USACO are all publicly available and could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks (HotpotQA 2018, AIME 22-24) were available online before model training. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time reported despite using paid APIs (GPT-4o, o1) with multiple calls per sample (BoN with N up to 16)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget or API spend is stated. The reasoning token counts for o1 are reported (Fig. 3) but not the total cost."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The choice of N values (1,4,8,16) for BoN is not justified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why specific N values, reward models, or agent configurations were selected as the reported results."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparing many methods across multiple benchmarks."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge any bias from evaluating their own framework choices. They use their own implementations of baselines without discussing this limitation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "o1 likely uses far more compute than GPT-4o direct inference, and BoN with N=16 uses 16x the compute, but compute-normalized comparisons are not provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure 'reasoning capability' as claimed. The paper assumes benchmark accuracy equals reasoning ability without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Agent Workflow uses different scaffolding (domain-specific prompts, GPTs, agent framework) compared to other methods, but the paper attributes performance differences to the method rather than controlling for scaffold differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (2018), AIME problems (2022-2024), and USACO problems predate model training. No discussion of whether models saw these problems during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The LIME-based filtering uses model responses which could introduce selection bias favoring certain model behaviors."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether filtered benchmark samples are independent or share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used despite using well-known public benchmarks with closed-source models."
       }
     }
   },
   "claims": [
     {
       "claim": "o1 achieves the best performance across almost all benchmarks compared to previous test-time compute methods and GPT-4o",
       "evidence": "Table 1 shows o1-preview or o1-mini outperform all baselines on most benchmarks. o1-mini achieves 62% on AIME vs 12.22% for GPT-4o; o1-preview achieves 44.6% on USACO vs 5.04% for GPT-4o.",
       "supported": "moderate"
     },
     {
       "claim": "Agent Workflow achieves significant improvements across all benchmarks and is closest to o1's performance",
       "evidence": "Table 1 shows Agent Workflow at 24.70% overall vs o1-preview's 34.32%. Agent Workflow reaches 22.22% on USACO (vs 5.04% GPT-4o) and 46.07% on Collie.",
       "supported": "moderate"
     },
     {
       "claim": "Reward model quality limits the upper boundary of search-based methods (BoN)",
       "evidence": "Fig. 4 shows human reward model achieving ~33% on HotpotQA vs ~13-15% for automated reward models. Gap between human and automated reward models demonstrates the limitation.",
       "supported": "moderate"
     },
     {
       "claim": "Six reasoning patterns are identified in o1, with Divide and Conquer and Self-Refinement being most frequent",
       "evidence": "Fig. 2 shows frequency distribution of six patterns based on manual analysis of 20-30 samples per benchmark. DC and SR appear most frequently.",
       "supported": "weak"
     },
     {
       "claim": "Step-wise BoN is limited by long-context inference problems on complex tasks",
       "evidence": "Table 2 shows average reasoning tokens >200 for all tasks. Step-wise BoN achieves <12% on Collie and only half of other methods' AIME performance (Section 4.3).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All comparisons are based on single point estimates with no error bars, confidence intervals, significance tests, or multiple runs. Claims of superiority are made by comparing raw numbers."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Closed-source models (o1, GPT-4o) are tested on public benchmarks (HotpotQA from 2018, AIME problems) without any discussion of whether these appeared in training data. o1 may have been specifically trained on math competition problems."
     },
     {
       "flag": "Tiny sample for reasoning pattern analysis",
       "detail": "The six reasoning patterns are identified from manual inspection of only 20-30 samples per benchmark. This is too small for reliable pattern categorization and the categorization methodology is not described."
     },
     {
       "flag": "Uncontrolled comparisons",
       "detail": "Agent Workflow uses fundamentally different scaffolding (domain-specific prompts, GPTs, specialized agent frameworks) than other baselines, making it impossible to attribute performance differences to any single factor."
     },
     {
       "flag": "No compute-normalized comparison",
       "detail": "o1 uses significantly more inference compute than GPT-4o, and BoN with N=16 uses 16x the compute of direct inference. No cost or compute normalization is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries discussion despite significant methodological weaknesses."
     },
     {
       "flag": "Potential selection bias in data filtering",
       "detail": "The LIME-based filtering selects problems that multiple models get wrong, which may bias the benchmark toward specific types of difficult problems rather than being representative of general difficulty."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Core reference on test-time compute scaling, directly relevant to understanding inference-time reasoning strategies."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2024,
       "relevance": "Key baseline method for LLM self-improvement through iterative feedback, relevant to agentic coding workflows."
     },
     {
       "title": "Can Language Models Solve Olympiad Programming?",
       "authors": ["Quan Shi", "Michael Tang", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "arxiv_id": "2404.10952",
       "relevance": "USACO benchmark used in the study; evaluates LLM coding capabilities on competitive programming."
     },
     {
       "title": "Agents: An Open-Source Framework for Autonomous Language Agents",
       "authors": ["Wangchunshu Zhou"],
       "year": 2023,
       "arxiv_id": "2309.07870",
       "relevance": "Agent framework used as baseline; relevant to agentic AI architecture design."
     },
     {
       "title": "Symbolic Learning Enables Self-Evolving Agents",
       "authors": ["Wangchunshu Zhou"],
       "year": 2024,
       "arxiv_id": "2406.18532",
       "relevance": "State-of-the-art agent workflow framework used in evaluations; relevant to agentic AI capabilities."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code evaluation benchmark, relevant to LLM code generation methodology."
     },
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "authors": ["Nathan Lambert"],
       "year": 2024,
       "arxiv_id": "2403.13787",
       "relevance": "Benchmark for evaluating reward models, relevant to understanding limitations of search-based inference methods."
     },
     {
       "title": "Think Before You Speak: Training Language Models with Pause Tokens",
       "authors": ["Sachin Goyal"],
       "year": 2023,
       "arxiv_id": "2310.02226",
       "relevance": "Early work on enabling models to use additional compute before responding, precursor to test-time compute scaling."
     },
     {
       "title": "Quiet-STaR: Language Models Can Teach Themselves to Think Before Speaking",
       "authors": ["Eric Zelikman"],
       "year": 2024,
       "arxiv_id": "2403.09629",
       "relevance": "Unsupervised thought token learning for test-time compute, relevant to understanding reasoning capabilities."
     },
     {
       "title": "PAL: Program-Aided Language Models",
       "authors": ["Luyu Gao"],
       "year": 2023,
       "relevance": "Tool-augmented LLM reasoning approach, relevant to test-time compute and agentic methods."
     }
   ]
 }