ai-research-survey

scan.json (27928B)

---

 {
   "paper": {
     "title": "Chain-of-Thought Prompting Obscures Hallucination Cues in Large Language Models: An Empirical Evaluation",
     "authors": [
       "Jiahao Cheng",
       "Tiancheng Su",
       "Jia Yuan",
       "Guoxiu He",
       "Jiawei Liu",
       "Xinqi Tao",
       "Jingwen Xie",
       "Huaxia Li"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.17088"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is publicly available at: https://github.com/ECNU-Text-Computing/cot-hallu-detect' providing a direct GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets (CommonsenseQA, ARC-Challenge, MMLU, TruthfulQA, TriviaQA, PopQA, HaluEval, CNN/Daily Mail) and specifies exactly which splits are used. All datasets are publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM for inference (Appendix F) and LLaMA-Factory for fine-tuning, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the code repository is linked, the paper itself does not contain step-by-step reproduction instructions. The implementation details in Appendix F describe settings but not a reproducible workflow a researcher could follow without examining the repository."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1-11 report only point estimates (single AUROC values, accuracy percentages). No confidence intervals, error bars, or ± notation are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Kolmogorov-Smirnov (K-S) tests to evaluate whether score distributions differ significantly between conditions (Section 5.2, Figure 4, with p < 0.01 stated). The pilot experiment also uses K-S tests (Figure 2, p < 0.01)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw differences in AUROC and accuracy scores but does not report standardized effect sizes (e.g., Cohen's d, odds ratios). While percentage differences can be computed from the tables, the paper does not explicitly contextualize the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states dataset sizes (e.g., 1,221 samples for CommonsenseQA, 817 for TruthfulQA) but does not justify why these sizes are sufficient or provide any power analysis. The total of 768 experimental configurations is stated but not justified as adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance measures, or results from multiple runs are reported. All tables present single-run point estimates. The paper does not mention running experiments multiple times with different seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares CoT prompting conditions against a 'base' (no CoT) baseline scenario across all experiments (Tables 1-11). Each detection method's performance without CoT serves as the baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The detection methods evaluated include recent work: LLM-Check (NeurIPS 2024), INSIDE (ICLR 2024), In-Context Sharpness (ICML 2024), Verbalized Certainty (ACL 2024), and MIND (ACL 2024 Findings). These are contemporary baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies three CoT prompting methods (CoT, LtM, MRPP) across multiple dimensions, effectively ablating the type of reasoning strategy. The comparison between different detection method categories (consistency-based, internal-state-based, self-evaluation-based) also serves as an ablation of detection approach."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: AUROC for classification performance, Spearman's rank correlation for monotonic relationships, Expected Calibration Error (ECE) for confidence alignment, plus accuracy and entropy in the pilot experiments (Section 4.1, Appendix E)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'We validate our automated metrics by having two annotators manually annotate 150 samples, achieving a Cohen's Kappa of 0.863.' This validates the automated evaluation pipeline with human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard test/validation splits: ARC-Challenge test split, MMLU test split, CommonsenseQA validation split (since test labels unavailable), TruthfulQA validation subset, TriviaQA validation subset, PopQA test subset (Section 3.1, 4.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (HaluEval, PopQA, TriviaQA, CNN/Daily Mail, TruthfulQA), per LLM (Llama-8B, Mistral-7B, Llama-70B, DeepSeek), per detection method (8 methods), and per annotation protocol (Textual vs. Semantic) across Tables 3, 7-11."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides a qualitative failure example: the TriviaQA question about Walter Matthau's first movie where LtM prompting causes Verbalized Certainty to increase from 0.2 to 1.0 despite an incorrect answer. The paper also discusses where consistency-based methods are more robust."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding IS a negative result: CoT prompting degrades hallucination detection. The paper also reports that not all CoT variants consistently improve performance (Section 5.1), and that consistency-based methods show greater robustness (a nuanced finding contradicting a simple 'everything degrades' narrative)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that CoT prompting 'obscures critical signals used for detection, impairing the effectiveness of various detection methods' — supported by Table 3 showing 465/768 configurations with AUROC degradation. The claim about consistency-based methods being more robust is supported by Tables 3-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about CoT prompting causing changes in detection performance. The experimental design is adequate: it uses controlled comparisons where the only variable changed is the prompting method (base vs. CoT/LtM/MRPP), with all other factors held constant. The K-S tests confirm distributional shifts in internal states."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims CoT prompting 'Obscures Hallucination Cues in Large Language Models' broadly, but the study only uses 4 open-source LLMs (all ≤70B parameters) and cannot test closed-source models. The Limitations section acknowledges this ('unable to conduct experiments on larger-scale LLMs' and 'prevented us from evaluating... closed-source LLMs') but the title and abstract do not bound the claims to open-source models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several alternative explanations: that the effect might be due to evaluation metric limitations (Section 5.1, validated with human annotation), that different LLMs respond differently due to 'differences in response generation tendencies' (Section 5.3), and that the effect varies by detection paradigm type. The Limitations section discusses metric choice."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names with versions are provided: Llama-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Llama-3.1-70B-Instruct, DeepSeek-R1-Distill-Llama-8B, with HuggingFace URLs in Appendix F.2. These are specific enough to reproduce."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides the actual prompt templates used: CoT ('Think about it step by step.'), LtM ('What subproblems must be solved before answering the inquiry?'), MRPP ('You need to perform multi-step reasoning, with each step carrying out as many basic operations as possible.'). The system prompt is stated: 'You are a helpful assistant.' (Appendix F.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F reports: temperature=0.5, max_tokens=512 for reasoning generation; INSIDE sampling n=15, temperature=0.5, top_p=0.99, top_k=5; SelfCheckGPT n=20, temperature=1; Sharpness alpha=1.0; specific layer numbers for internal-state methods; LoRA fine-tuning: lr=5e-05, batch_size=32, epochs=5, lora_rank=8, lora_alpha=16."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiments involve direct prompting of LLMs without any agent framework, tool use, or multi-step orchestration."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1 and 4.2 describe which splits and subsets are used for each dataset (e.g., CommonsenseQA validation split with 1,221 samples, TriviaQA rc.wikipedia.nocontext validation, PopQA test in closed-book setting). Appendix F describes the normalization for token probabilities (Eq. 8) and ECE normalization (3%-97% range scaling)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the Discussion and Conclusion that discusses computational constraints, inability to test closed-source LLMs, and evaluation metric limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: 'Due to computational resource constraints, we are unable to conduct experiments on larger-scale LLMs, which limits the generalizability of our conclusions.' It also addresses the specific limitation of Exact Match and ROUGE-L metrics for assessing response correctness in QA tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'Addressing the impact of CoT prompting on hallucination detection is beyond the scope of this study and is left for future work.' It also notes the limitation to open-source LLMs and the specific detection methods that do not rely on external knowledge bases (Section 2)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the code repository is linked, the paper does not explicitly state that raw experimental outputs (model predictions, hallucination scores, per-sample results) are available for independent verification. Only aggregated metrics are presented in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: all datasets are standard public benchmarks with specific splits identified (Section 3.1, 4.2). The pipeline for generating responses, computing hallucination scores, and evaluating correctness is documented in Section 4.1 and Appendix F."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The two annotators for validation (150 samples) are mentioned but this is a minor validation component using standard benchmark data, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 illustrates the full experimental pipeline: LLM response generation → correctness measurement (textual/semantic) → hallucination detection scoring → evaluation across three dimensions. Section 4.1 and Appendix E describe each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section lists funding sources: National Natural Science Foundation of China (72204087), Shanghai Planning Office of Philosophy and Social Science Youth Project (2022ETQ001), Chenguang Program, Shanghai Pujiang Program (23PJC030), Young Elite Scientists Sponsorship Program by CAST, and Fundamental Research Funds for Central Universities."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: East China Normal University, Wuhan University, and Xiaohongshu Inc. Three authors are from Xiaohongshu (a commercial company), which is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are Chinese government research foundations and academic programs (NSFC, Shanghai municipal programs, CAST). None have a financial stake in whether CoT prompting helps or hinders hallucination detection."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Three authors are affiliated with Xiaohongshu Inc. (a commercial tech company), but no declaration of competing interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the four LLMs evaluated (Llama-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Llama-3.1-70B-Instruct, DeepSeek-R1-Distill-Llama-8B), despite evaluating them on benchmarks that may be in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The benchmarks used (CommonSenseQA from 2019, ARC from 2018, MMLU from 2021, TruthfulQA from 2022, TriviaQA from 2017, HaluEval from 2023) are all publicly available and were likely in the training data of models released in 2024-2025."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of benchmark contamination risk. Most benchmarks (TriviaQA 2017, ARC 2018, CommonsenseQA 2019, MMLU 2021) are old enough to almost certainly be in the training data of models from 2024. This is a notable omission given that contamination could affect the base accuracy numbers."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is not a human subjects study. The two annotators who validated 150 samples is a minor annotation task, not a human study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper includes an Ethics Statement but it confirms 'this research does not involve the collection or use of personal data.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, or computational time per experiment are reported despite running 768 experimental configurations across multiple LLMs including a 70B parameter model. Consistency-based methods requiring 15-20 samples per input are noted as having 'high computational overhead' but no quantification is given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not mention GPU hours, total time, or hardware used for the experiments despite running 768 configurations across 4 LLMs (including a 70B model) with multiple detection methods requiring multiple forward passes."
       }
     }
   },
   "claims": [
     {
       "claim": "CoT prompting improves LLM accuracy but simultaneously degrades hallucination detection performance across multiple detection methods.",
       "evidence": "Table 1 shows accuracy improvements with CoT (e.g., Llama accuracy increases from 90.5% to 97.67% on ARC-Challenge). Table 3 shows 465 out of 768 experimental configurations exhibit AUROC degradation for hallucination detection under CoT prompting (Section 5.3).",
       "supported": "strong"
     },
     {
       "claim": "CoT prompting significantly alters the probability distribution of final answer tokens, making hallucinated answers appear more confident.",
       "evidence": "Table 1 shows entropy consistently decreases after CoT (e.g., Llama entropy drops from 23.96 to 6.63 on ARC-Challenge). Figure 2 shows K-S tests confirming significant distributional shifts (p < 0.01). Section 3.2 discusses how CoT increases confidence even for incorrect predictions.",
       "supported": "strong"
     },
     {
       "claim": "Self-evaluation-based detection methods are most adversely affected by CoT prompting, while consistency-based methods show greater robustness.",
       "evidence": "Table 3 shows SelfCheckGPT-Prompt and Verbalized Certainty have the highest counts of AUROC degradation (59/96 and 67/96 respectively), while EigenScore and SelfCheckGPT-NLI show lower degradation counts (48/96 and 56/96). Section 5.3 provides qualitative explanation for why.",
       "supported": "strong"
     },
     {
       "claim": "Advanced hallucination detection methods lose their advantage over simple perplexity baselines when CoT prompting is applied.",
       "evidence": "Section 5.3 states: 'when CoT prompts are introduced, many detection methods fail to surpass the perplexity baseline, and some perform worse.' Tables 7-11 show specific AUROC values where perplexity-based methods perform comparably to or better than advanced methods under CoT.",
       "supported": "moderate"
     },
     {
       "claim": "CoT prompting leads to increased Expected Calibration Error (ECE) across most experimental settings.",
       "evidence": "Section 5.4 states: 'CoT prompting leads to an increase in ECE across most experimental settings, suggesting that the confidence of detection methods becomes less reliable.' Detailed results are deferred to Appendix G.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CoT prompting creates a trade-off in LLMs: while it improves task accuracy by encouraging step-by-step reasoning, it simultaneously obscures the internal signals used by hallucination detection methods. Across 768 experimental configurations spanning 4 LLMs, 5 datasets, 8 detection methods, and 3 CoT variants, more than half showed degraded detection performance (measured by AUROC). Self-evaluation-based and internal-state-based detection methods are most vulnerable, while consistency-based methods that rely on multi-sample reasoning show greater robustness. Notably, advanced detection methods lose their typical advantage over simple perplexity baselines when CoT prompting is used.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "All experiments appear to be single-run with no error bars, confidence intervals, or standard deviations reported across any of the 768 configurations. Given that LLM outputs involve stochastic sampling (temperature=0.5), results could vary across runs."
     },
     {
       "flag": "No benchmark contamination discussion",
       "detail": "Several benchmarks used (TriviaQA 2017, ARC 2018, CommonsenseQA 2019, MMLU 2021) are old enough to likely be in the training data of models from 2024-2025. Contamination could inflate base accuracy and potentially affect the accuracy improvements attributed to CoT. The paper does not acknowledge this risk."
     },
     {
       "flag": "Computational cost not disclosed",
       "detail": "Running 768 experimental configurations across 4 LLMs (including 70B) with methods requiring multiple forward passes (INSIDE: 15 samples, SelfCheckGPT: 20 samples) represents substantial compute. No hardware, GPU hours, or cost information is provided, making it hard to assess reproducibility and practical relevance."
     },
     {
       "flag": "Title overgeneralizes beyond tested scope",
       "detail": "The title claims CoT 'Obscures Hallucination Cues in Large Language Models' broadly, but only 4 open-source models (≤70B) were tested. No closed-source models or models >70B were evaluated. The Limitations section acknowledges this but the title and abstract do not scope the claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed H. Chi", "Quoc V Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought prompting, central to the paper's investigation of CoT effects on LLM behavior."
     },
     {
       "title": "Large language models are zero-shot reasoners",
       "authors": ["Takeshi Kojima", "Shixiang Shane Gu", "Machel Reid", "Yutaka Matsuo", "Yusuke Iwasawa"],
       "year": 2024,
       "relevance": "Introduces zero-shot CoT prompting, one of the three CoT methods evaluated in this study."
     },
     {
       "title": "SelfcheckGPT: Zero-resource black-box hallucination detection for generative large language models",
       "authors": ["Potsawee Manakul", "Adian Liusie", "Mark Gales"],
       "year": 2023,
       "relevance": "Key hallucination detection baseline evaluated in the study; represents consistency-based detection methods."
     },
     {
       "title": "LLM-check: Investigating detection of hallucinations in large language models",
       "authors": ["Gaurang Sriramanan", "Siddhant Bharti", "Vinu Sankar Sadasivan", "Shoumik Saha", "Priyatham Kattakinda", "Soheil Feizi"],
       "year": 2024,
       "relevance": "Internal-state-based hallucination detection method that is a key evaluation target in this paper."
     },
     {
       "title": "INSIDE: LLMs' internal states retain the power of hallucination detection",
       "authors": ["Chao Chen", "Kai Liu", "Ze Chen", "Yi Gu", "Yue Wu", "Mingyuan Tao", "Zhihang Fu", "Jieping Ye"],
       "year": 2024,
       "relevance": "Hybrid hallucination detection method combining internal-state and consistency approaches, evaluated in this study."
     },
     {
       "title": "Confidence under the hood: An investigation into the confidence-probability alignment in large language models",
       "authors": ["Abhishek Kumar", "Robert Morabito", "Sanzhar Umbet", "Jad Kabbara", "Ali Emami"],
       "year": 2024,
       "relevance": "Introduces Verbalized Certainty detection method, one of the detection approaches most affected by CoT in this study."
     },
     {
       "title": "In-context sharpness as alerts: An inner representation perspective for hallucination mitigation",
       "authors": ["Shiqi Chen", "Miao Xiong", "Junteng Liu", "Zhengxuan Wu", "Teng Xiao", "Siyang Gao", "Junxian He"],
       "year": 2024,
       "relevance": "Internal-state-based detection method (Sharpness) evaluated as one of the eight detection approaches in this study."
     },
     {
       "title": "Language models don't always say what they think: Unfaithful explanations in chain-of-thought prompting",
       "authors": ["Miles Turpin", "Julian Michael", "Ethan Perez", "Samuel R. Bowman"],
       "year": 2023,
       "relevance": "Demonstrates unfaithfulness of CoT explanations, directly relevant to understanding how CoT affects LLM internal states and detection."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning-oriented LLM architecture whose distilled variant is one of the four models evaluated in this study."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Key hallucination evaluation benchmark used in this study with a distinct evaluation protocol for truthfulness and informativeness."
     },
     {
       "title": "Chain of thoughtlessness? An analysis of CoT in planning",
       "authors": ["Kaya Stechly", "Karthik Valmeekam", "Subbarao Kambhampati"],
       "year": 2024,
       "relevance": "Critiques limitations of CoT prompting, providing context for why CoT may not genuinely improve reasoning and may alter LLM internal states."
     },
     {
       "title": "Unsupervised real-time hallucination detection based on the internal states of large language models",
       "authors": ["Weihang Su", "Changyue Wang", "Qingyao Ai", "Yiran Hu", "Zhijing Wu", "Yujia Zhou", "Yiqun Liu"],
       "year": 2024,
       "relevance": "MIND framework for hallucination detection, evaluated as supplementary evidence that model-training-based methods also degrade under CoT."
     }
   ]
 }