ai-research-survey

scan.json (21645B)

---

 {
   "paper": {
     "title": "CoT-RAG: Integrating Chain of Thought and Retrieval-Augmented Generation to Enhance Reasoning in Large Language Models",
     "authors": ["Feiyang Li", "Peng Fang", "Zhan Shi", "Arijit Khan", "Fang Wang", "Weihao Wang", "Xin Zhang", "Yongjian Cui"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.13534"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/hustlfy123/CoT-RAG. The abstract and Appendix C state 'we have released the codebase, datasets, and manually designed decision trees.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses nine public benchmark datasets (AQuA, GSM8K, MultiArith, SingleEq, HotpotQA, CSQA, SIQA, Last Letter, Coin Flip) and four vertical domain datasets. The GitHub repo claims to include datasets and decision trees."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found in the paper. Only mentions using LLMs via API with temperature 0 and max tokens 1000."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. The GitHub link is provided but the paper itself does not contain a reproducing results section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (accuracy percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoT-RAG outperforms baselines across all datasets but provides no statistical significance tests. Comparisons are made solely by comparing accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'accuracy improvements from 4.0% to 44.3%' and per-baseline breakdowns like 'compared to Manual-CoT... the average accuracy across datasets increases by 4.0%-15.2%'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes or number of examples used. The paper mentions adapting datasets using an LLM (Appendix H) but does not justify the sample sizes chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures reported. The paper states temperature is set to 0 for deterministic outputs, but does not discuss variance across any dimension."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines included: 13 methods for general domains (Zero-shot, Zero-shot-CoT, Manual-CoT, Auto-CoT, Complex-CoT, PS, QDMRPS, KD-CoT, IRCoT, KG-CoT, Iter-CoT, ZEUS, Pattern-CoT) and 8 graph-form RAG methods for vertical domains."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods such as ZEUS (Kumar et al., 2025), Pattern-CoT (Zhang et al., 2025), ToG-2 (Ma et al., 2025), and PoG (Tan et al., 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 presents ablation study removing four components: node decomposition, RAG, PsePrompting, and expert inspection, tested on four datasets (Figure 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is used as the evaluation metric across all experiments. No other metrics (F1, precision, recall, etc.) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. All evaluation is via automated accuracy on benchmark datasets."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard public benchmark test sets (AQuA, GSM8K, etc.) for evaluation. These are established benchmarks with defined test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset across three reasoning categories (arithmetic, commonsense, symbolic) in Table 1, and per vertical domain dataset in Table 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis or error examples are discussed. The paper only shows where accuracy is lower but does not examine specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study shows accuracy drops when components are removed. The paper also notes CoT-RAG has slightly higher runtime than simpler baselines (Manual-CoT, Zero-shot) and that CoT-RAG (Zero-expert) performs 7.8% worse than the expert version."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'accuracy gains ranging from 4.0% to 44.3% over state-of-the-art methods' which is supported by Table 1 results. Claims about domain-specific datasets are supported by Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies (Appendix C.1) where individual components are removed while others are held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to 'Enhance Reasoning in Large Language Models' broadly, but results are primarily on proprietary API-based LLMs (ERNIE, GPT-4o mini, GPT-4o, GLM-4-flash). The limitations section acknowledges this only partially, noting smaller LLMs are excluded. The paper does not bound claims to the specific models and datasets tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the observed improvements. The paper does not consider confounds such as whether the expert-provided decision trees simply encode the answer structure, or whether improvements come from additional context length rather than the framework design."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'GPT-4o mini', 'GPT-4o', 'ERNIE-Speed-128K', 'GLM-4-flash'. No API snapshot dates or version identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides pseudo-program prompt templates with actual fill values in the appendices (Tables 11-13 in Appendix E, and detailed prompts in Appendix F). The full prompt structures including sub-questions, sub-cases, and sub-descriptions are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'we invoke LLMs that are not fine-tuned via API and set the temperature to 0 to ensure deterministic outputs' and 'we set the max tokens to 1000'. Number of demonstration examples is also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage framework (KG-driven CoT Generation, Learnable Knowledge Case-aware RAG, Pseudo-Program Prompting Execution) is described in detail in Section 3, with algorithms in Appendix A."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'following GraphRAG and Graph-CoT, we employ an LLM to adapt the datasets to satisfy our testing needs' but details are deferred to Appendix H. The adaptation process using an LLM to transform datasets raises questions about what transformations were applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' discusses two key limitations: reliance on proprietary LLMs with advanced capabilities and the influence of expert knowledge on decision tree construction."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: 'publicly available LLMs, especially those with a smaller scale (e.g., 7B or 13B parameters), fall short on these requirements' and 'the construction of decision trees is influenced by the expert's domain-specific knowledge and background.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Limitations mention constraints but do not clearly bound what generalizations readers should not make from the results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets and claims to release code, datasets, and decision trees at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the datasets used, their sources, and categories. Standard public benchmarks are referenced with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a benchmark evaluation study using standard datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an LLM to adapt datasets (Section 4.1: 'we employ an LLM to adapt the datasets to satisfy our testing needs') but does not clearly document the full pipeline from original datasets to final evaluation data, including what transformations were applied and how many examples were affected."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Acknowledgements' lists multiple funding sources: National Key Research and Development Program of China, NSFC grants, China Postdoctoral Science Foundation, and Novo Nordisk Foundation Grant."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed, including Huawei Technologies Co., Ltd for three authors (Weihao Wang, Xin Zhang, Yongjian Cui). The paper does not evaluate Huawei products specifically."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government research foundations (NSFC, China Postdoctoral Science Foundation) and Novo Nordisk Foundation, none of which have a direct financial stake in the outcomes of this reasoning framework evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper. Three authors are affiliated with Huawei, but no conflicts of interest declaration is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, GPT-4o mini, ERNIE, and GLM models on public benchmarks but does not state training data cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether test examples (from benchmarks like GSM8K, AQuA, etc.) could have appeared in the training data of the LLMs used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like GSM8K (2021), AQuA (2017), and MultiArith (2015) were published well before the training cutoffs of GPT-4o and other models used. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.4 reports average runtime per question and token consumption across methods and datasets (Tables 8 and 9), including comparisons showing 29.2% runtime reduction and 33.4% token decrease vs GraphRAG."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, total API spend, or total GPU hours stated. Only per-question averages are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CoT-RAG achieves accuracy improvements ranging from 4.0% to 44.3% over state-of-the-art CoT methods on nine public datasets.",
       "evidence": "Table 1 shows results on ERNIE-Speed-128K across nine datasets. CoT-RAG achieves 89.1% average accuracy vs next best ZEUS at 77.7% (Section 5).",
       "supported": "strong"
     },
     {
       "claim": "CoT-RAG increases accuracy by 8.9% to 80.6% on four vertical domain datasets compared to graph-form LLM-based RAG methods.",
       "evidence": "Table 2 shows CoT-RAG at 95.2% average accuracy vs best baseline (CoT-RAG IndexIVFPQ at 87.7% and RoG at 83.7%) on GPT-4o mini.",
       "supported": "strong"
     },
     {
       "claim": "Node decomposition is the most critical component of CoT-RAG.",
       "evidence": "Ablation study (Appendix C.1, Figure 4) shows removing node decomposition causes the largest accuracy drop across all four tested datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Expert involvement is indispensable for vertical domain applications, with CoT-RAG (Zero-expert) achieving 7.8% lower accuracy.",
       "evidence": "Table 2 shows CoT-RAG (Zero-expert) at 87.4% average vs CoT-RAG at 95.2%.",
       "supported": "moderate"
     },
     {
       "claim": "CoT-RAG achieves competitive runtime with 29.2% reduction in average runtime and 33.4% decrease in average token consumption compared to GraphRAG.",
       "evidence": "Appendix C.4, Table 9 reports runtime and token comparisons.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoT-RAG integrates knowledge graph-driven CoT generation, case-aware RAG, and pseudo-program prompting to improve LLM reasoning. On nine public benchmarks across arithmetic, commonsense, and symbolic reasoning, it achieves 89.1% average accuracy (ERNIE-Speed-128K), outperforming 13 baselines by 4.0-44.3%. On four vertical domain datasets (law, finance, logic), it achieves 95.2% average accuracy with GPT-4o mini. Ablation shows node decomposition is the most impactful component, and expert-crafted decision trees provide a 7.8% advantage over LLM-generated ones.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons are based on raw accuracy numbers with no significance tests, confidence intervals, or variance measures. With temperature set to 0, determinism is claimed, but this does not address sampling variability from dataset composition."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only accuracy is used across all experiments. No secondary metrics (F1, precision, recall, calibration) are reported, making it impossible to understand performance characteristics beyond aggregate correctness."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "Several benchmarks (AQuA 2017, MultiArith 2015, GSM8K 2021) predate the training data of GPT-4o and other models. No contamination analysis is provided, yet the framework's improvements are measured against these potentially contaminated benchmarks."
     },
     {
       "flag": "Dataset adaptation via LLM inadequately documented",
       "detail": "The paper states datasets were adapted using an LLM to 'satisfy testing needs' but the transformation process is not transparently documented in the main text. This could introduce systematic bias favoring the proposed method."
     },
     {
       "flag": "Suspiciously large improvements on some benchmarks",
       "detail": "CoT-RAG achieves 98.4% on HotpotQA and 98.7% on SIQA with ERNIE-Speed-128K, while baselines top out around 87-90%. These near-perfect scores on challenging benchmarks warrant scrutiny, especially given the LLM-adapted datasets."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Foundational CoT prompting paper that this work builds upon and compares against."
     },
     {
       "title": "From Local to Global: A Graph RAG Approach to Query-Focused Summarization",
       "authors": ["Darren Edge"],
       "year": 2024,
       "arxiv_id": "2404.16130",
       "relevance": "GraphRAG baseline combining RAG with knowledge graphs for question answering."
     },
     {
       "title": "Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks",
       "authors": ["Wenhu Chen"],
       "year": 2023,
       "relevance": "Key baseline using code prompts for reasoning, motivating CoT-RAG's pseudo-program approach."
     },
     {
       "title": "Navigate through Enigmatic Labyrinth: A Survey of Chain of Thought Reasoning",
       "authors": ["Zheng Chu"],
       "year": 2024,
       "relevance": "Comprehensive survey of CoT reasoning advances and limitations in LLMs."
     },
     {
       "title": "Reasoning on Graphs: Faithful and Interpretable Large Language Model Reasoning",
       "authors": ["Linhao Luo"],
       "year": 2024,
       "relevance": "RoG baseline combining KG-grounded reasoning paths with LLMs."
     },
     {
       "title": "Think-on-Graph 2.0: Deep and Faithful Large Language Model Reasoning with Knowledge-Guided Retrieval Augmented Generation",
       "authors": ["Shengjie Ma"],
       "year": 2025,
       "relevance": "ToG-2 baseline using KGs to connect documents for enhanced LLM reasoning."
     },
     {
       "title": "Large Language Models are Zero-Shot Reasoners",
       "authors": ["Takeshi Kojima"],
       "year": 2022,
       "relevance": "Zero-shot-CoT baseline demonstrating LLM reasoning with minimal prompting."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieval-augmented generation paradigm."
     },
     {
       "title": "Graph Chain-of-Thought: Augmenting Large Language Models by Reasoning on Graphs",
       "authors": ["Bowen Jin"],
       "year": 2024,
       "relevance": "Graph-CoT baseline for iterative graph-based LLM reasoning."
     },
     {
       "title": "Enhancing Zero-Shot Chain of Thought Prompting via Uncertainty-Guided Strategy Selection",
       "authors": ["Shanu Kumar"],
       "year": 2025,
       "relevance": "ZEUS baseline improving CoT with uncertainty-guided demonstration selection."
     }
   ]
 }