ai-research-survey

scan.json (23899B)

---

 {
   "paper": {
     "title": "Thinking Isn't an Illusion: Overcoming the Limitations of Reasoning Models via Tool Augmentations",
     "authors": ["Zhao Song", "Song Yue", "Jiahao Zhang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.17699",
     "doi": "10.48550/arXiv.2507.17699"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Tool-augmented LRMs consistently outperform their non-reasoning LLM counterparts on Apple's thinking-illusion benchmark, challenging the narrative that reasoning in LRMs is illusory. Program-of-Thought (PoT) with external Python interpreters yields the largest gains, enabling perfect accuracy on Hanoi Tower and ~80% on River Crossing/Blocks World for DeepSeek-R1. However, Checker Jumping remains unsolved for N≥3 across all models and tools, and weaker base models (Qwen 3) benefit less from tool augmentation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in abstract: https://github.com/magiclinux/thinking_is_not_an_illusion"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses Apple's publicly available thinking-illusion benchmark puzzles. The paper reuses their problem descriptions (Section 3.1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only 'Python version 3.11.13' is mentioned for the PoT interpreter (Section 4.1). No requirements.txt, dependency list, or environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as success counts out of 5 trials (e.g., '4/5') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LRMs outperform LLMs but provides no statistical significance tests. Comparisons are made by eyeballing success counts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Results are raw success counts (e.g., 4/5 vs 0/5) without any formal effect size measure."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment is repeated 5 times with no justification for why 5 runs is sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as k/5 success counts. No standard deviation, variance, or spread measure is provided across the 5 runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Direct prompting (no tool use) serves as the baseline, and LLM counterparts (DeepSeek-V3 vs R1, Qwen 3 vs Qwen 3 Thinking) are compared (Tables 2-3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "DeepSeek-V3/R1 and Qwen 3 are 2024-2025 models. Apple's thinking-illusion benchmark is from 2025. All are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three tool-use frameworks (PoT, Think-and-Execute, Scratchpad) are compared against direct prompting, effectively ablating the tool component. Hyperparameter studies in Section 4.3 examine scratchpad chain length and token consumption."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (success/trial out of 5) is reported. No other metrics such as token efficiency, solution optimality, or partial credit."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The puzzles have automatically verifiable solutions (Section 3.1). Human evaluation is not relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This uses fixed puzzle instances with controlled complexity parameter N, not a train/test split paradigm."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by all 4 puzzle types (Hanoi, Checker, River, Block) and 6 complexity levels (N=3 to N=13) in Tables 2-3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Observation 2 (Section 4.2): 'Checker Jumping remains unsolved for N≥3 across all models and tool-use methods.' Observation 3 discusses that weaker models benefit less from tools."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Checker Jumping fails across all conditions. Think-and-Execute provides minimal gains. Qwen 3 shows limited benefit from tools. These are explicitly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that 'with proper tool use, LRMs consistently outperform their non-reasoning counterparts across all levels of task complexity' are supported by Tables 2-3, though 'consistently' is generous given Checker Jumping failures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('tool use can unlock the reasoning potential of LRMs') and the study design — controlled comparisons of LRM vs LLM with and without tools on fixed puzzles — provides adequate evidence for these component-level claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim to show that 'thinking isn't an illusion' and that tool augmentation overcomes 'limitations of reasoning models' broadly, but results are limited to 4 specific puzzle types from one benchmark with 2 model families. The paper does not bound these generalizations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, PoT success on Hanoi could simply mean the models know the recursive algorithm (a memorization argument), not that they 'reason' better. The output length limitation hypothesis is discussed but no alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures success on 4 algorithmic puzzles and frames this as evidence about 'reasoning capabilities' broadly. No discussion of whether puzzle-solving is a valid proxy for general reasoning ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'DeepSeek-V3', 'DeepSeek-R1', 'Qwen 3', 'Qwen 3 Thinking' (Table 1) without API snapshot dates or version identifiers. 'deepseek-reasoner' model name is mentioned but no version date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states they use 'the same prompts as in [SMA+25]' and adapt templates from [CKK+24] but does not include the actual prompt text. Scratchpad description D and in-context examples Em are described conceptually but not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Scratchpad T=5 and m=3 in-context examples are reported. Python 3.11.13 is stated. But no temperature, top-p, max_tokens, or other API parameters are reported for model calls. Only a timeout=1200 recommendation is mentioned."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The scratchpad framework is described in detail (Section 3.3) with key components, step-wise prompting, early stopping mechanism, and final answer concatenation. PoT and Think-and-Execute are also described (Section 3.2, Figure 2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The puzzle generation follows Apple's benchmark with complexity parameter N. Section 3.1 describes the four puzzle types and how they map to the original benchmark's problem descriptions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion mentions 'future directions' but does not discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. The paper does not state what the results do NOT show or what settings were not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate success counts (k/5) are reported. Individual run outputs, model responses, and generated code are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the experimental setup: models used, API interaction method, prompt structure, and repetition count."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is from standardized benchmark puzzles."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from model output to success/failure determination is not documented. How outputs are parsed and verified against puzzle solutions is not described beyond mentioning 'simple verifiers'."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UC Berkeley and Northeastern University. One author has no institutional affiliation listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models may have seen these puzzle types or solutions during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Apple's thinking-illusion benchmark and classic puzzles like Tower of Hanoi are widely known. The paper does not discuss whether models have been trained on these puzzle solutions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Token consumption is analyzed for Qwen 3 Thinking (Figures 5-8) but no actual monetary costs or wall-clock times are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, API costs, or total token expenditure is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 5 runs but no seed sensitivity analysis or variance across seeds is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'All experiments in this paper are repeated five times, and we report the average results across runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Settings like T=5 and m=3 appear chosen without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of T=5 scratchpad steps and m=3 in-context examples is not justified. No sensitivity analysis for these choices beyond the scratchpad chain length study."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across the many model×tool×puzzle×N conditions."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement the tool-augmentation frameworks themselves and compare against Apple's results without acknowledging potential implementation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption analysis (Figures 5-8) examines the relationship between tool use and token cost, showing that tool use does not necessarily increase token consumption."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 4 algorithmic puzzles actually measure 'reasoning capability' as claimed. The paper accepts Apple's benchmark at face value without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "When comparing LRMs vs LLMs with tools, different models may interact with tools differently. The paper does not address whether the observed LRM advantage comes from better reasoning or better tool use. The scaffold is identical, but the confound between reasoning ability and code generation ability is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Tower of Hanoi is a classic algorithm taught in CS courses and widely present in training data. The paper does not discuss whether models have memorized solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the tool-use prompts or in-context examples leak solution strategies."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The puzzle instances at different N values share the same structure. No discussion of whether performance on N=3 informs N=5, etc."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used."
       }
     }
   },
   "claims": [
     {
       "claim": "With proper tool use, LRMs consistently outperform their non-reasoning counterparts across all levels of task complexity.",
       "evidence": "Tables 2-3 show DeepSeek-R1 outperforming DeepSeek-V3 with PoT on River Crossing (4/5 vs 0/5 across all N) and Blocks World (5/5 vs 1/5). However, on Hanoi with PoT both achieve 5/5, and on Checker both achieve 0/5.",
       "supported": "moderate"
     },
     {
       "claim": "PoT enables major improvements for LRMs on previously unsolvable problems like River Crossing and Blocks World.",
       "evidence": "Table 3: DeepSeek-R1 with PoT achieves 4/5 on River Crossing across all N values (vs 0-1/5 with direct prompting) and 5/5 on Blocks World across all N values (vs 0-5/5).",
       "supported": "strong"
     },
     {
       "claim": "Some hard problems remain unsolved even with tool use (Checker Jumping for N≥3).",
       "evidence": "Table 2: All models score 0/5 on Checker Jumping for N≥3 across all tool-use conditions.",
       "supported": "strong"
     },
     {
       "claim": "Tool use does not necessarily increase token consumption for LRMs.",
       "evidence": "Figures 5-8 show token consumption for Qwen 3 Thinking, where Scratchpad and Think-and-Execute sometimes use fewer tokens than direct prompting on Checker Jumping and Blocks World.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small sample size",
       "detail": "Only 5 repetitions per condition with no statistical tests. Many comparisons are based on differences like 1/5 vs 0/5, which are not statistically distinguishable."
     },
     {
       "flag": "Overclaiming from limited evidence",
       "detail": "The title claims 'Thinking Isn't an Illusion' based on 4 algorithmic puzzles with 2 model families. On Checker Jumping (25% of tasks), no model benefits from tools. On Hanoi with PoT, LRMs and LLMs perform identically (all 5/5). The LRM advantage is mainly visible on 2 of 4 tasks with 1 of 3 tools."
     },
     {
       "flag": "Memorization confound unaddressed",
       "detail": "Tower of Hanoi is a canonical CS exercise. When PoT achieves 5/5 for all models, this likely reflects memorized algorithms rather than reasoning. The paper does not distinguish code generation from reasoning ability."
     },
     {
       "flag": "No statistical testing",
       "detail": "Claims of 'consistently outperform' are made without any statistical tests. With n=5 binary trials, many observed differences are within chance variation."
     },
     {
       "flag": "Missing limitations section",
       "detail": "No limitations, threats to validity, or scope boundaries are discussed despite substantial overclaiming."
     }
   ],
   "cited_papers": [
     {
       "title": "The illusion of thinking: Understanding the strengths and limitations of reasoning models via the lens of problem complexity",
       "authors": ["Parshin Shojaee", "Iman Mirzadeh", "Keivan Alizadeh", "Maxwell Horton", "Samy Bengio", "Mehrdad Farajtabar"],
       "year": 2025,
       "arxiv_id": "2506.06941",
       "relevance": "The primary benchmark and finding this paper responds to — claims LRMs do not outperform LLMs under controlled complexity."
     },
     {
       "title": "GSM-symbolic: Understanding the limitations of mathematical reasoning in large language models",
       "authors": ["Seyed Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi"],
       "year": 2025,
       "relevance": "Key prior work questioning LRM reasoning, finding models rely on pattern matching rather than generalizable reasoning."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "DeepSeek-R1 is one of the primary LRMs evaluated in this study."
     },
     {
       "title": "Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks",
       "authors": ["Wenhu Chen", "Xueguang Ma", "Xinyi Wang", "William W. Cohen"],
       "year": 2023,
       "relevance": "PoT is a core tool-augmentation method evaluated in this paper."
     },
     {
       "title": "Language models as compilers: Simulating pseudocode execution improves algorithmic reasoning in language models",
       "authors": ["Hyungjoo Chae", "Yeonghyeon Kim"],
       "year": 2024,
       "relevance": "Think-and-Execute framework evaluated as one of the tool-use methods."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on CoT reasoning that LRMs build upon."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling parameters for reasoning",
       "authors": ["Charlie Victor Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2025,
       "relevance": "Directly relevant to test-time compute scaling which is the mechanism behind LRMs."
     },
     {
       "title": "Stop overthinking: A survey on efficient reasoning for large language models",
       "authors": ["Yang Sui", "Yu-Neng Chuang", "Guanchu Wang"],
       "year": 2025,
       "arxiv_id": "2503.16419",
       "relevance": "Survey on LRM efficiency and overthinking, directly relevant to claims about reasoning model limitations."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu"],
       "year": 2023,
       "relevance": "Foundational work on LLM tool use, relevant to the tool augmentation framework."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang", "Anfeng Li", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for Qwen 3 models evaluated in this study."
     }
   ]
 }