ai-research-survey

scan.json (27676B)

---

 {
   "paper": {
     "title": "Atom of Thoughts for Markov LLM Test-Time Scaling",
     "authors": ["Fengwei Teng", "Quan Shi", "Zhaoyang Yu", "Jiayi Zhang", "Yuyu Luo", "Chenglin Wu", "Zhijiang Guo"],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2502.12018"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'We submit our code alongside this paper and will make it publicly available to facilitate reproducibility and future research.' This is a promise of future release, not an actual working URL or archive provided in the paper. No repository link is given."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: MATH, GSM8K, MBPP, LiveCodeBench, AIME (with HuggingFace link provided in footnote 1: https://huggingface.co/datasets/Maxwell-Jia/AIME_2024), and LongBench (HotpotQA, MuSiQue, 2WikiMultiHopQA). All datasets are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section listing library versions is provided in the paper or appendices."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While prompt templates and hyperparameters are described in the appendix, there are no step-by-step reproduction instructions, README with commands, or scripts to replicate experiments. The code is promised but not yet released."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., 83.6 on MATH, 95.0 on GSM8K) with no confidence intervals, error bars, or ± notation. Figure 3 shows cost-performance curves with no uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AOT outperforms baselines across multiple benchmarks but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance numbers with baseline context, allowing readers to compute effect sizes. For example, Table 1 shows AOT at 83.6 vs CoT at 78.3 on MATH (GPT-4o-mini), and Section 4.4 reports 'on MATH, this full integration achieves 84.9% accuracy compared to ToT's 82.0%.' Baseline and AOT numbers are consistently provided together."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates on 'the first 1,000 cases from MATH for efficiency' (Appendix A.3.3) but does not justify why 1,000 is sufficient or discuss statistical power. Other benchmarks are evaluated in their entirety but without justification for adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across experimental runs. All results in Table 1 appear to be single-run numbers. There is no mention of averaging over multiple runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes extensive baselines: CoT, CoT-SC, Self-Refine (SR), Analogical Reasoning (AR), AFlow, ToT, GoT, and FoT. Section 4.1 describes the baseline selection rationale."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods: AFlow (2024), FoT (2024), and the paper tests on recent models including O3-mini (2025) and DeepSeek-R1 (2025). CoT and ToT are foundational methods that remain standard baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents ablation studies examining two variants: (1) without decomposition (directly contracts without DAG), and (2) without DAG-guided contraction (decomposition without structural guidance). Figure 3 shows both ablations degrade performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass rates for mathematical and coding benchmarks, and F1 scores for multi-hop QA tasks (Section 4.1). Additionally, Table 2 in Appendix B.1 reports Answer Equivalence Maintenance, Test-time Complexity Reduction, and LLM-as-a-Judge Selection Rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of AOT's outputs is included. All evaluation is automated through benchmark pass rates, F1 scores, and LLM-as-a-judge assessments. Given claims about interpretability and reasoning quality, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets (MATH, GSM8K, MBPP, LiveCodeBench, AIME, LongBench). These are standard held-out test sets from their respective benchmark suites, not used for tuning the method."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (MATH, GSM8K, MBPP, LongBench, AIME, LiveCodeBench) and by model category (non-reasoning LLMs vs reasoning LLMs). Table 2 in Appendix B.1 provides per-dataset quality metrics. Figures 5-7 analyze structural properties per problem."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses failure modes including error propagation from memoryless transitions and semantic divergence. The Limitations section (Appendix C) discusses where the approach may struggle: fixed transition counts, computational overhead, and dependency on LLM capability for DAG generation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Section 4.3) reports that removing DAG-guided contraction causes 'a more severe drop' than removing decomposition entirely, showing that partial structural cues 'can be more harmful than providing none at all.' AR baseline performs notably poorly and is excluded from some figures. Appendix B.3 reports accuracy decreases with increased structural complexity."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AOT 'consistently outperforms existing baselines as computational budgets increase' — supported by Table 1 and Figure 3. It claims 'seamless integration with existing reasoning frameworks and different LLMs' — supported by Section 4.4 integration experiments with ToT, FoT, and both reasoning and non-reasoning LLMs."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about component contributions (e.g., 'both ablations significantly degrade performance'). These are supported by controlled ablation experiments (Section 4.3) that systematically remove individual components while keeping other variables fixed, which is adequate for such claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract claim a 'general-purpose reasoning framework' but tests only on math, code generation, and multi-hop QA tasks. Section 5 claims AOT enables 'scalable, high-performance inference' without bounding to the tested domains. The Limitations section (Appendix C) does not explicitly state what domains or task types were NOT tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, performance improvements could be partly due to ensembling effects (the judge selects from three candidate solutions), additional LLM calls, or prompt engineering rather than the Markovian structure specifically. No threats-to-validity section or alternative explanations section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o-mini', 'DeepSeek-V3', 'O3-mini', and 'DeepSeek-R1' without specifying exact API versions or snapshot dates. For example, no version like 'gpt-4o-mini-2024-07-18' is provided. Marketing names alone do not constitute specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 provides complete prompt templates as Python code for all four prompt types (direct, decompose, contract, judge) across all three task domains (math, code, multi-hop QA). The templates include placeholders with clear variable names that receive the actual question/context values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 reports key hyperparameters: temperature T=1.0, maximum Markov chain length of 3, ToT branches of 3, FoT trees of {2, 4, 8}. Section 4.1 also references these settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed descriptions of the agentic scaffolding: the two-phase transition mechanism (decomposition and contraction), the DAG construction process, the LLM-as-a-judge termination strategy, and integration with tree search and reflective refinement (Sections 3.1-3.2). Figure 2 provides an overview diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3.3 documents dataset-specific processing: 'For the MATH dataset, we filter out questions with non-integer or non-decimal answers to ensure consistent evaluation. We evaluate the first 1,000 cases from MATH for efficiency, while assessing the remaining benchmarks in their entirety.' LongBench preprocessing is referenced via [2]."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides a dedicated Limitations section discussing three specific limitations: fixed maximum transition count, computational overhead of decomposition, and dependency on LLM capability for DAG generation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Appendix C) identifies specific threats: 'the current implementation relies on a fixed maximum transition count (set to 3), which may not be optimal for all problem types,' 'the decomposition process adds computational overhead compared to direct inference,' and 'weaker models may struggle to generate valid dependency graphs, potentially degrading performance.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. The Limitations section discusses weaknesses of the approach but does not bound the scope of claims (e.g., it does not state which task types, languages, or domains are excluded from its claims). The title and abstract present it as a general-purpose framework without explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual problem-level results, model outputs, DAG structures generated) is made available. Only aggregated results in tables and figures are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly describes data sources: standard benchmarks (MATH, GSM8K, MBPP, LiveCodeBench, AIME from HuggingFace, LongBench) with citations and links. Appendix A.3.3 describes dataset-specific filtering and selection."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks, making recruitment methods not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: benchmarks are selected (Section 4.1), MATH is filtered to integer/decimal answers and sampled to 1,000 cases (Appendix A.3.3), and evaluation metrics are specified (pass rates for math/code, F1 for QA). The reasoning pipeline (decomposition, contraction, judging) is described with examples in Appendix B.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper. Authors are affiliated with HKUST(GZ), DeepWisdom, and Renmin University of China, but no funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: HKUST(GZ), DeepWisdom, and Renmin University of China. These are provided on the first page of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Authors from DeepWisdom (a company) may have commercial interest in the results, but this is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. DeepWisdom is a commercial entity, and potential financial interests related to the work are not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4o-mini, DeepSeek-V3, O3-mini, and DeepSeek-R1 on benchmarks including MATH and GSM8K. No training data cutoff dates are stated for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MATH (2021), GSM8K (2021), and MBPP (2021) are all older benchmarks that could plausibly be in the training data of 2024-2025 models. This is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH, GSM8K, and MBPP were all published before 2022 and are widely used; any model trained after 2022 may have seen them. The paper uses LiveCodeBench, which is designed for contamination-free evaluation, but does not discuss contamination risks for the other benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of a reasoning framework."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 plots performance vs cost in USD (log2 scale) for all methods. Figure 4 shows cost in absolute terms for integration experiments. The paper explicitly analyzes cost-performance tradeoffs as a core contribution."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While relative costs are shown in figures, the total computational budget (total API spend, total number of API calls, total tokens consumed) is not stated. The paper shows cost curves but does not quantify the total resources spent on all experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "AOT consistently outperforms existing baselines across both reasoning and non-reasoning LLMs on multiple benchmarks.",
       "evidence": "Table 1 shows AOT achieves highest scores across MATH (83.6 with GPT-4o-mini, 96.5 with DeepSeek-V3), GSM8K (95.0/98.2), MBPP (75.2/79.6), LongBench (68.5/71.0), AIME (83.0/81.7), and LiveCodeBench (32.2/30.9) compared to CoT, CoT-SC, SR, AR, AFlow, ToT, GoT, and FoT baselines (Section 4.2).",
       "supported": "moderate",
       "reason": "Results consistently favor AOT across all benchmarks and models, but no statistical tests are provided, no variance/error bars are reported, and results appear to be from single runs. The consistency across models and benchmarks provides moderate confidence."
     },
     {
       "claim": "Both ablation variants (without decomposition and without DAG-guided contraction) significantly degrade performance, with partial structural cues being more harmful than none.",
       "evidence": "Figure 3 shows both ablation curves fall below AOT across MATH, GSM8K, LongBench, AIME, and LiveCodeBench. Section 4.3 states the variant without DAG-guided contraction causes 'a more severe drop' (Section 4.3).",
       "supported": "moderate",
       "reason": "The ablation is well-designed as controlled single-variable manipulation, but without statistical tests or variance reporting, the significance of the degradation cannot be assessed precisely."
     },
     {
       "claim": "The DAG generation process maintains answer equivalence at >99% across all datasets.",
       "evidence": "Table 2 (Appendix B.1) reports Answer Equivalence Maintenance rates of 99.2% (MATH), 99.5% (GSM8K), 99.7% (MBPP), and 99.3% (LongBench).",
       "supported": "moderate",
       "reason": "The metric is assessed by LLM evaluation rather than ground-truth verification, so the 99%+ figure depends on the LLM evaluator's accuracy. The methodology for assessment is described but relies on another model's judgment."
     },
     {
       "claim": "Deeper Markov chains exhibit an emergent atomic reasoning structure where reasoning tokens converge toward minimal forms.",
       "evidence": "Figure 4 shows Token Ratio approaching 1.0 as cost increases, indicating convergence toward atomic structure. Section 4.4 describes this convergence pattern (Section 3.2 and 4.4).",
       "supported": "moderate",
       "reason": "The convergence trend is visible in Figure 4 but is demonstrated only on GPT-4o-mini. The 'emergence' claim is primarily observational and the atomic structure is defined by the authors rather than independently validated."
     },
     {
       "claim": "Integrating AOT with tree search and reflective refinement achieves compounding performance gains (e.g., 84.9% on MATH vs ToT's 82.0%, 81.2% on AIME vs ToT's 78.0%).",
       "evidence": "Section 4.4 reports specific numbers for the full integration (ToT + Markov chain + Reflective Refinement) achieving 84.9% on MATH and 81.2% on AIME.",
       "supported": "moderate",
       "reason": "Specific numbers are provided with clear baseline comparisons, but again no statistical tests or variance reporting. The improvements are consistent but of moderate magnitude."
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "AOT (Atom of Thoughts) proposes a Markovian reasoning framework that decomposes complex problems through iterative DAG-based decomposition and contraction, reducing historical dependencies in LLM reasoning. The method achieves consistent improvements over baselines (CoT, ToT, GoT, FoT, AFlow) across math, code, and multi-hop QA benchmarks with both reasoning (O3-mini, DeepSeek-R1) and non-reasoning (GPT-4o-mini, DeepSeek-V3) LLMs. Ablation studies confirm that both the decomposition and DAG-guided contraction components are necessary. The framework demonstrates an emergent 'atomic reasoning' property where deeper Markov chains converge toward irreducible problem representations.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results in Table 1 appear to be single-run point estimates with no error bars, confidence intervals, or standard deviations. For stochastic methods using temperature 1.0, this is a significant omission — results could vary meaningfully across runs."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims AOT 'outperforms' baselines across all benchmarks based solely on comparing point estimates. With no statistical tests, it is impossible to determine whether the differences are statistically significant or within normal variation."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MATH (2021), GSM8K (2021), and MBPP (2021) are older benchmarks that 2024-2025 models may have seen during training. The paper uses LiveCodeBench (designed for contamination-free evaluation) for code but does not discuss contamination for math benchmarks. Since all methods use the same models, this may not affect relative comparisons, but absolute performance levels could be inflated."
     },
     {
       "flag": "Overclaimed generality",
       "detail": "The paper claims AOT is a 'general-purpose reasoning framework' but tests only on math, code generation, and multi-hop QA. Many other reasoning domains (logical reasoning, commonsense reasoning, planning, scientific reasoning) are not tested."
     },
     {
       "flag": "LLM-as-evaluator circularity",
       "detail": "The answer equivalence and complexity reduction metrics in Table 2 (Appendix B.1) are assessed by LLM evaluation rather than ground-truth verification. Using an LLM to evaluate the quality of an LLM reasoning framework introduces potential circularity."
     }
   ],
   "cited_papers": [
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Tom Griffiths", "Yuan Cao", "Karthik Narasimhan"],
       "year": 2023,
       "relevance": "Foundational test-time reasoning framework that AOT extends and compares against as a baseline."
     },
     {
       "title": "Graph of thoughts: Solving elaborate problems with large language models",
       "authors": ["Maciej Besta", "Nils Blach", "Ales Kubicek"],
       "year": 2024,
       "relevance": "Graph-based reasoning framework that AOT compares against and claims to improve upon in efficiency."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique that AOT builds upon and benchmarks against."
     },
     {
       "title": "Forest-of-thought: Scaling test-time compute for enhancing LLM reasoning",
       "authors": ["Zhenni Bi", "Kai Han", "Chuanjian Liu", "Yehui Tang", "Yunhe Wang"],
       "year": 2024,
       "arxiv_id": "2412.09078",
       "relevance": "Recent test-time scaling framework used as a key baseline and integration partner for AOT."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": ["Jiayi Zhang", "Jinyu Xiang", "Zhaoyang Yu", "Fengwei Teng"],
       "year": 2024,
       "arxiv_id": "2410.10762",
       "relevance": "Agentic workflow framework used as a baseline; shares authors with AOT paper."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Key paper on test-time compute scaling that motivates AOT's approach to efficient inference-time reasoning."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Self-consistency sampling method used as a baseline in AOT's evaluation."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Self-refinement framework used as a baseline and integrated into AOT's reflective refinement component."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "relevance": "Contamination-free code benchmark used for evaluating AOT on code generation tasks."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "State-of-the-art reasoning LLM used as a backbone model in AOT's evaluation experiments."
     },
     {
       "title": "Are more LLM calls all you need? Towards scaling laws of compound inference systems",
       "authors": ["Lingjiao Chen", "Jared Quincy Davis", "Boris Hanin"],
       "year": 2024,
       "arxiv_id": "2403.02419",
       "relevance": "Studies scaling laws for compound LLM inference systems, directly relevant to AOT's test-time scaling claims."
     },
     {
       "title": "Language agent tree search unifies reasoning, acting, and planning in language models",
       "authors": ["Andy Zhou", "Kai Yan", "Michal Shlapentokh-Rothman"],
       "year": 2024,
       "relevance": "Tree search approach for LLM agents that AOT's modular integration extends."
     }
   ]
 }