ai-research-survey

scan.json (22388B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Improving LLM Reasoning with Multi-Agent Tree-of-Thought Validator Agent",
     "authors": ["Fatemeh Haji", "Mazal Bethany", "Maryam Tabar", "Cho-Yu Jason Chiang", "Anthony Rios", "Peyman Najafirad"],
     "year": 2024,
     "venue": "1st Workshop on Safe and Trustworthy Agents @NeurIPS 2024",
     "arxiv_id": "2409.11527",
     "doi": "10.48550/arXiv.2409.11527"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub URL provided in abstract: https://github.com/SecureAIAutonomyLab/MA-ToT"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "GSM8K is a publicly available benchmark dataset. The paper uses a random subset of 500 samples from it."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (4x NVIDIA DGX A100 80GB) but no requirements.txt, Dockerfile, or library version details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper itself contains no README-level reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., 84.2%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'outperforms' based on comparing raw accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point improvements with baseline context, e.g., '8.8 percentage points over ToT for GPT-3.5-turbo (from 75.4% to 84.2%)'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 500 samples from GSM8K with no justification for why 500 was chosen or whether this is sufficient for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: Standard IO, Chain of Thought (CoT), and Tree of Thoughts (ToT)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CoT (2022) and ToT (2023) are recent and standard baselines for LLM reasoning evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The system has multiple components (ToT Reasoners, Thought Validator, consensus voting, iterative refinement) but none are individually ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is reported as an evaluation metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The paper makes claims about 'trustworthiness' of reasoning but only uses automated accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "They use a random subset of 500 samples from the GSM8K test set. No tuning on this set is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only overall accuracy numbers per model are reported. No breakdown by problem difficulty, type, or category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses failure modes: fixed tree depth causing unnecessary complexity for easy problems and insufficient depth for hard ones. The appendix shows detailed examples of incorrect reasoning."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that ToT and their method show diminishing returns for models that already perform well on standard IO (e.g., GPT-4o-mini and Llama-3.1-70B show smaller improvements)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'outperforming the standard ToT strategy by an average 5.6% across four LLMs' which is supported by Table 1 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the Thought Validator 'enhances' and 'improves' reasoning but does not ablate the Validator component separately from the multi-agent setup to establish causality."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title says 'Improving LLM Reasoning' broadly but evaluation is only on GSM8K (arithmetic reasoning). The paper acknowledges this in limitations but the title and framing overclaim scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. The improvement could be due to simple ensemble effects (more compute/calls) rather than the specific Validator mechanism, but this is not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the method improves 'trustworthiness' of reasoning but only measures accuracy on GSM8K. No discussion of the gap between accuracy and trustworthiness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions specified: GPT-3.5-turbo-0125, GPT-4o-mini-2024-07-18, Llama-3.1-8B, Llama-3.1-70B."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all components (IO, CoT, ToT, and Verifier) is provided in the Appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=1, top_p=1 for IO/CoT/ToT; temperature=0.5, top_p=0.4 for Validator. Tree depth=2, width=5."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail in Section 3: parallel Reasoner agents with ToT, state evaluation, Thought Validator, consensus voting, and iterative refinement. Figure 1 provides a workflow diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'a random subset of 500 samples from the GSM8K dataset as the test set' without describing how the random selection was done or providing a seed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations and Conclusion' and contains substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: fixed tree depth causing problems for both easy and hard tasks, high computational cost (token usage quantified), and single-benchmark evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'while our evaluation on GSM8K demonstrates the effectiveness of our approach for arithmetic reasoning, testing on additional reasoning-intensive benchmarks would help establish the method's generalizability.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (model responses, reasoning trees, per-example results) are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "GSM8K is a well-known benchmark and the paper describes using a random 500-sample subset from it."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard benchmark (GSM8K)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of data pipeline: how 500 samples were selected, how model outputs were collected and scored, or how consensus was tracked."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. One author is from Peraton Labs (industry) but no funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Texas at San Antonio and Peraton Labs."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding not disclosed, so independence cannot be assessed. One author is affiliated with Peraton Labs, a defense contractor, with no disclosure of potential interest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the four models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K was published in 2021. All models used may have been trained on it. No discussion of potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K is a well-known benchmark likely in training data of GPT-3.5-turbo and GPT-4o-mini. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token usage per question reported: CoT 256 tokens vs ToT 4000 tokens for GPT-3.5-turbo, 341 vs 10,600 for GPT-4o-mini. API calls per problem (~20 per Reasoner) also noted."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified (4x NVIDIA DGX A100 80GB GPUs) and total time (18 hours for all experiments in parallel)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The Validator uses different temperature/top_p (0.5/0.4) than other components (1/1), but no search budget or justification for these specific values beyond a brief rationale."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Tree depth=2 and width=5 are stated as following Yao et al., but no exploration of alternatives or validation set selection is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own version of ToT and compare against it without acknowledging potential bias from implementing baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The method uses ~20 API calls per Reasoner agent times multiple agents, dramatically more compute than baselines, but performance is not shown as a function of compute. The token cost is mentioned in limitations but not normalized against performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K is used as a proxy for 'reasoning' capability without discussing whether arithmetic word problems measure the kind of reasoning the paper claims to improve."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The proposed method adds multiple layers of scaffolding (parallel agents, validator, voting, iterative refinement) compared to baselines. Improvements could be due to the scaffolding rather than the specific ToT+Validator design, but this confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K was published in 2021. All models (GPT-3.5-turbo, GPT-4o-mini, Llama-3.1) were trained after 2021 and may have seen GSM8K solutions. Not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the 500 sampled test examples or potential similarity to training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Multi-agent ToT with Thought Validator outperforms standard ToT by an average 5.6% across four LLMs on GSM8K",
       "evidence": "Table 1 shows MA-ToT with Validator scores: 84.2, 92.2, 89.0, 94.8 vs ToT: 75.4, 91.6, 80.2, 92.8 across four models.",
       "supported": "moderate"
     },
     {
       "claim": "The improvement is 8.8 percentage points for GPT-3.5-turbo (from 75.4% to 84.2%)",
       "evidence": "Table 1 shows ToT at 75.4% and MA-ToT+Validator at 84.2% for GPT-3.5-turbo.",
       "supported": "weak"
     },
     {
       "claim": "ToT benefits are more pronounced for weaker models and diminish for stronger ones",
       "evidence": "Table 1: GPT-3.5-turbo gains 8.8pp, Llama-3.1-8B gains 8.8pp, but GPT-4o-mini gains only 0.6pp and Llama-3.1-70B gains 2.0pp.",
       "supported": "moderate"
     },
     {
       "claim": "The Thought Validator prevents incorrect reasoning from leading to errors in the final answer",
       "evidence": "Appendix examples show the Validator rejecting incorrect reasoning from R1 and R2 while validating correct R3 reasoning.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes combining Tree-of-Thought reasoning with a Thought Validator agent in a multi-agent framework. On a 500-sample subset of GSM8K, the approach improves accuracy over standard ToT by 0.6-8.8 percentage points across four LLMs, with larger gains for weaker models. The method is computationally expensive, requiring ~20 API calls per Reasoner agent, with token usage increasing from 256 (CoT) to 4000+ (ToT) per question.",
   "red_flags": [
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results are single point estimates on a 500-sample subset with no variance reporting, making it impossible to assess whether differences are statistically meaningful."
     },
     {
       "flag": "Compute-unfair comparison",
       "detail": "MA-ToT+Validator uses orders of magnitude more compute (multiple agents × ~20 API calls each + validation + potential iterative rounds) than baselines (single call for IO/CoT). The improvement may simply be due to more compute, not the specific method."
     },
     {
       "flag": "Single benchmark evaluation",
       "detail": "Claims about 'improving LLM reasoning' are based solely on GSM8K arithmetic reasoning. No evaluation on other reasoning types."
     },
     {
       "flag": "No ablation study",
       "detail": "Cannot determine whether gains come from multi-agent voting, the ToT component, the Validator, or the iterative refinement. Missing: ToT-only multi-agent without Validator, Validator without ToT, etc."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "GSM8K (2021) is likely in the training data of all four models tested. High baseline scores (91-93% for stronger models) may reflect memorization rather than reasoning."
     }
   ],
   "cited_papers": [
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen"],
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K benchmark used for evaluation; relevant to LLM reasoning evaluation methodology."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B. Tenenbaum", "Igor Mordatch"],
       "arxiv_id": "2305.14325",
       "relevance": "Core multi-agent debate approach for improving LLM reasoning, directly relevant to agentic AI methodology."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Foundational ToT method that this paper extends; key technique for structured LLM reasoning."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "arxiv_id": "2201.11903",
       "relevance": "Foundational CoT prompting technique for LLM reasoning, used as baseline."
     },
     {
       "title": "Large language model based multi-agents: A survey of progress and challenges",
       "authors": ["Taicheng Guo", "Xiuying Chen", "Yaqi Wang"],
       "year": 2024,
       "relevance": "Survey of multi-agent LLM systems, directly relevant to agentic AI survey scope."
     },
     {
       "title": "Counterfactual debating with preset stances for hallucination elimination of LLMs",
       "authors": ["Yi Fang", "Moxin Li", "Wenjie Wang"],
       "arxiv_id": "2406.11514",
       "relevance": "Multi-agent debate framework for mitigating LLM hallucinations, relevant to AI safety and agentic systems."
     },
     {
       "title": "HaluEval: A large-scale hallucination evaluation benchmark for large language models",
       "authors": ["Junyi Li", "Xiaoxue Cheng", "Wayne Xin Zhao"],
       "arxiv_id": "2305.11747",
       "relevance": "Hallucination evaluation benchmark for LLMs, relevant to AI evaluation methodology."
     },
     {
       "title": "Towards CausalGPT: A multi-agent approach for faithful knowledge reasoning via promoting causal consistency in LLMs",
       "authors": ["Ziyi Tang", "Ruilin Wang", "Weixing Chen"],
       "arxiv_id": "2308.11914",
       "relevance": "Multi-agent approach for reasoning verification in LLMs."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Describes Llama 3.1 models used in experiments; relevant to LLM capability evaluation."
     }
   ]
 }