ai-research-survey

scan.json (27831B)

---

 {
   "paper": {
     "title": "A Single Direction of Truth: An Observer Model's Linear Residual Probe Exposes and Steers Contextual Hallucinations",
     "authors": [
       "Charles O'Neill",
       "Slava Chalnev",
       "Chi Chi Zhao",
       "Max Kirkby",
       "Mudith Jayasekara"
     ],
     "year": 2025,
     "venue": "ICML 2025 (PMLR 267)",
     "arxiv_id": "2507.23221",
     "doi": "10.48550/arXiv.2507.23221"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "A linear probe on a frozen observer model's residual stream detects contextual hallucinations in a single forward pass, achieving F1 up to 0.99 on news summarisation and 0.84 on the synthetic ContraTales benchmark, outperforming lexical, entity, semantic similarity, and attention baselines by 5-27 points. The hallucination signal localises to a sparse, consistent late-layer MLP sub-circuit (layers 7-9 for a layer 10 probe on Gemma-2-9B). Causal steering experiments show that injecting or ablating the probe direction in a generator model bidirectionally modulates hallucination rates, establishing functional causality. Unsupervised domain adaptation on correct-only examples improves probe F1 from 0.75 to 0.89 on ContraTales.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper. The paper mentions releasing the ContraTales benchmark but provides no link to code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we release the 2000-example CONTRATALES benchmark' but provides no download URL or repository link in the paper text."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8xH200 GPUs' for finetuning (Section 4.5) but provides no requirements.txt, Dockerfile, or library version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology sections describe the approach but not how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% bootstrap confidence intervals (whiskers in Figure 2) and ± notation for F1 scores, e.g., '0.97±0.01 F1 on XSUM' and '0.75 ± 0.04 F1' on ContraTales (Section 4.2)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite claiming the probe 'outperforms baselines by 5-27 points', no statistical significance tests are reported. Comparisons rely solely on point estimates with confidence intervals."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute F1 differences: '5-8 points' over Lookback Lens on news, '9-27 points' on ContraTales (Section 4.2). Baseline and probe scores are both provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (1000 for CNN/DM, 1000 for XSum, 2000 for ContraTales) but no justification for why these sizes are sufficient. The steering experiment uses 128 summaries per α value with no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used with reported standard deviations (e.g., '0.75 ± 0.04 F1'). The layer sweep in Figure 1 shows F1 across all layers for multiple models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: lexical overlap, entity verification, semantic similarity, and Lookback Lens (Section 3.3, Figure 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Lookback Lens (Chuang et al., 2024) is recent. Semantic Entropy Probes (Kossen et al., 2024) are discussed in related work. The baselines represent current detection approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Layer sweeps (Figure 1) serve as ablation across probe positions. Cross-domain transfer experiments (Figure 3) ablate the training domain. Finetuning experiments (Section 4.5) ablate observer adaptation. Attribution analysis (Section 4.3) identifies contributing components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only F1 score is used as the detection metric. No precision, recall, accuracy, or AUROC are reported separately, though F1 implicitly combines precision and recall."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of detection quality is performed. The steering experiment uses GPT-4.1 as a judge (Section 4.4) rather than human evaluators. For a hallucination detection paper, human validation of detected hallucinations would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used throughout (Section 3.2): 'Performance is assessed via 5-fold cross-validation.' Layer selection uses inner-fold validation on training set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (CNN/DM, XSum, ContraTales), by model size (Gemma-2 2B/9B/27B, GPT-2-small), and by layer (Figures 1, 4). Cross-domain transfer is shown separately (Figure 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses probe activations on unrelated text from the Pile, identifying that lowest activations correspond to repetitive text (Table 4, Appendix D). The lower F1 on ContraTales vs news is discussed as a limitation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Per-head attention attributions 'showed fluctuations around zero, lacking layer-consistent signs' (Section 4.3) — a negative result showing attention patterns are not useful. The steering trade-off where reducing hallucinations increases repetition (Section 4.4) is also a negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 5-27 point improvement, mid-layer performance plateau across Gemma-2 sizes, sparse late-layer MLP attribution, and causal steering are all supported by results in Sections 4.1-4.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about steering and supports them with interventional experiments (Section 4.4): injecting the probe direction at varying strengths (α from -60 to +60) and measuring hallucination/repetition rates. This is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'A Single Direction of Truth' but results are limited to Gemma-2 models (2B-27B) plus GPT-2-small. The abstract's 'generator-agnostic' claim is tested only with gpt-4.1/o4-mini generated hallucinations. The paper acknowledges this in limitations but the title and abstract are broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses: synthetic hallucination artifacts (probes may learn generation-specific patterns), LLM judge noise/bias for steering evaluation, and applicability only to intrinsic hallucinations. Section 4.2 also discusses potential lexical cue exploitation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clear about what it measures (F1 on synthetic hallucination detection) and acknowledges in Limitations that synthetically generated hallucinations 'may exhibit patterns or artifacts predictable to an observer model' that differ from real-world hallucinations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified as 'Gemma-2-2B', 'Gemma-2-9B', 'Gemma-2-27B', 'GPT-2-small', 'gpt-4.1', 'gpt-4.1-mini', 'o4-mini', 'Claude Opus'. No snapshot dates, specific version hashes, or API versions are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for hallucination generation, factual generation, and hallucination evaluation is provided in Appendix A (Sections A.1, A.2, A.3)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Finetuning hyperparameters are reported: 'AdamW, LR 1×10^-5, context length 512, batch size 8, 8xH200 GPUs, no dropout' (Section 4.5). Logistic regression uses L2 regularisation. Steering uses greedy decoding with α range specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a single forward pass through a frozen model plus a linear probe."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data preparation: how factual and hallucinated continuations were generated, sentence length constraints (≤40 words), and concatenation for observer input. Appendix B describes ContraTales generation process with seed examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A substantial 'Limitations' subsection appears within Section 5 (Discussion), spanning approximately a full column of text with multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: synthetic hallucinations may have detectable artifacts, GPT-4.1 judge may introduce noise/bias in steering evaluation, computational cost of deploying observer model, limitation to intrinsic hallucinations only."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our study focuses specifically on intrinsic hallucinations that contradict or are unsupported by the provided source context; the applicability... to extrinsic hallucinations... remains untested' (Limitations)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No data download link is provided despite claiming to release ContraTales. Raw activations, probe weights, and generated continuations are not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation is described in detail: CNN/DM and XSum are standard datasets; hallucinated/factual continuations generated via gpt-4.1-mini with specific prompts (Appendix A); ContraTales generated via o4-mini with seed examples (Appendix B)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (CNN/DM, XSum) and synthetic generation (ContraTales)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from source datasets through continuation generation, concatenation, forward pass, activation extraction, to probe training is documented across Sections 3.1-3.2 and Appendix A-B. Dataset sizes are stated (Table 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: four authors from Parsed (London, UK), one independent researcher. Correspondence address at Parsed (charles@parsed.com)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "With no funding disclosure, independence cannot be assessed. Parsed is a company whose products could benefit from hallucination detection technology, creating a potential undisclosed interest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. Four of five authors are affiliated with Parsed, a commercial entity that could have financial interest in hallucination detection technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for Gemma-2 models or GPT-2. The paper does not mention when these models' training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "CNN/DM and XSum are well-known public datasets that Gemma-2 likely encountered in training. No discussion of whether the observer models saw these datasets during pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CNN/DM (2017) and XSum (2018) are public benchmarks likely in Gemma-2's training data. The paper does not discuss whether pre-training exposure affects the observer's ability to detect hallucinations on these datasets."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The paper claims the approach is 'efficient' and requires only 'a single forward pass' but provides no wall-clock time or cost figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned for finetuning ('8xH200 GPUs') but no total GPU hours, training time, or API costs for generating the datasets are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used with 'a fixed random seed... for data splitting and sampling across all experiments' (Section 3.2). Results are reported with variance across folds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is explicitly stated (Section 3.2). Steering experiments use 128 summaries per α value (Section 4.4)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Layer selection uses inner-fold validation but no hyperparameter search budget is stated. The L2 regularisation strength for the logistic probe is not discussed as a tuned parameter."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Layer selection is done via 'inner-fold validation on the training set' (Section 3.2), which is a principled approach that avoids test set leakage."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across datasets, models, layers, and baselines without any correction for multiple testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own method against baselines. They re-implement baselines (lexical overlap, entity verification, etc.) without acknowledging that their implementations may disadvantage baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The linear probe requires running a full Gemma-2 forward pass while baselines like lexical overlap require no neural network inference. This compute disparity is not quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses construct validity: synthetic hallucinations 'may exhibit patterns or artifacts predictable to an observer model' and may not represent real-world hallucinations. ContraTales is introduced specifically to address construct validity concerns about lexical-overlap-based detection."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; the method is a single forward pass plus linear probe."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CNN/DM (2017) and XSum (2018) predate Gemma-2's training. The observer model likely saw these datasets during pre-training, which could inflate detection performance. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "A potential feature leakage concern: hallucinations are generated by gpt-4.1-mini while factual continuations are also from gpt-4.1-mini. The probe might detect stylistic differences in the two generation modes rather than actual hallucination. The Limitations section acknowledges this partially but does not test for it."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Paired factual and hallucinated examples are generated from the same source articles. Whether cross-validation splits respect article-level independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. No decontamination, canary strings, or membership inference tests."
       }
     }
   },
   "claims": [
     {
       "claim": "A linear probe on an observer model's residual stream outperforms heuristic detectors by 5-27 F1 points across datasets.",
       "evidence": "Figure 2 and Section 4.2: linear probe achieves 0.99 F1 on CNN/DM, 0.97 on XSum, 0.75 on ContraTales vs best baseline (Lookback Lens at 0.94, 0.89, 0.48 respectively).",
       "supported": "strong"
     },
     {
       "claim": "A single linear direction in residual-stream activation space separates hallucinated from supported spans, with consistent mid-layer performance plateau across model sizes.",
       "evidence": "Figure 1 shows layer-wise F1 for Gemma-2 2B/9B/27B and GPT-2-small, all showing mid-layer plateau pattern (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "The hallucination signal localises to a sparse, consistent late-layer MLP sub-circuit (layers 7-9 for layer 10 probe).",
       "evidence": "Figure 4 shows MLP attributions consistent across CNN/DM, XSum, and ContraTales. Attention attributions show no consistent pattern (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "Manipulating the probe direction causally steers hallucination rates in generation.",
       "evidence": "Figure 5 shows bidirectional control: α=+60 increases hallucination to 0.86, α=-60 reduces to 0.35 (Section 4.4). However, hallucination is judged by GPT-4.1, not humans.",
       "supported": "moderate"
     },
     {
       "claim": "Unsupervised domain adaptation (finetuning on correct-only examples) improves probe F1 by 0.10-0.17 across models.",
       "evidence": "Figure 6 shows improvements: GPT-2-small +0.10, Gemma-2-2B +0.17, Gemma-2-9B +0.14 (Section 4.5).",
       "supported": "strong"
     },
     {
       "claim": "The hallucination direction transfers across news domains with minimal accuracy loss.",
       "evidence": "Figure 3 shows cross-domain transfer CNN/DM→XSum and XSum→CNN/DM with the linear probe maintaining high F1 while surface-cue baselines degrade substantially (Section 4.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Synthetic hallucination artifacts",
       "detail": "All hallucinations are generated by gpt-4.1-mini or o4-mini. The probe may learn to detect stylistic signatures of prompted hallucination generation rather than genuine contextual inconsistency. The paper acknowledges this but does not test with naturally occurring hallucinations."
     },
     {
       "flag": "LLM-as-judge for causal claim",
       "detail": "The steering experiment's hallucination rate is judged by GPT-4.1, introducing potential noise and bias into the key causality claim. No human validation of the judge's accuracy is provided."
     },
     {
       "flag": "Compute-unfair baseline comparison",
       "detail": "The linear probe requires a full Gemma-2 forward pass (billions of parameters) while baselines like lexical overlap and entity verification are essentially free. The comparison is framed as method quality rather than cost-effectiveness."
     },
     {
       "flag": "No contamination analysis for observer models",
       "detail": "Gemma-2 models were likely pre-trained on CNN/DM and XSum articles. The high F1 on these datasets (0.97-0.99) could partly reflect memorisation of the source articles rather than hallucination detection capability."
     },
     {
       "flag": "Company evaluating commercially relevant technology",
       "detail": "Four of five authors are from Parsed, a commercial entity. Hallucination detection is commercially valuable. No competing interests statement or funding disclosure is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "A Survey on Hallucination in Large Language Models: Principles, Taxonomy, Challenges, and Open Questions",
       "authors": ["L. Huang", "W. Yu", "W. Ma"],
       "year": 2024,
       "doi": "10.1145/3703155",
       "relevance": "Comprehensive survey on LLM hallucination taxonomy and detection, directly relevant to understanding the hallucination problem space."
     },
     {
       "title": "Detecting hallucinations in large language models using semantic entropy",
       "authors": ["S. Farquhar", "J. Kossen", "L. Kuhn", "Y. Gal"],
       "year": 2024,
       "doi": "10.1038/s41586-024-07421-0",
       "relevance": "Nature paper on semantic entropy for hallucination detection, key competing approach."
     },
     {
       "title": "Semantic Entropy Probes: Robust and Cheap Hallucination Detection in LLMs",
       "authors": ["J. Kossen", "J. Han", "M. Razzak"],
       "year": 2024,
       "arxiv_id": "2406.15927",
       "relevance": "Lightweight probe-based hallucination detection, directly comparable approach."
     },
     {
       "title": "Lookback Lens: Detecting and Mitigating Contextual Hallucinations in Large Language Models Using Only Attention Maps",
       "authors": ["Y.-S. Chuang", "L. Qiu", "C.-Y. Hsieh"],
       "year": 2024,
       "arxiv_id": "2407.07071",
       "relevance": "Attention-based hallucination detection baseline used in this paper's comparisons."
     },
     {
       "title": "SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models",
       "authors": ["P. Manakul", "A. Liusie", "M. J. F. Gales"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.557",
       "relevance": "Sampling-based consistency approach for hallucination detection, key related work."
     },
     {
       "title": "The Internal State of an LLM Knows When It's Lying",
       "authors": ["A. Azaria", "T. Mitchell"],
       "year": 2023,
       "doi": "10.18653/v1/2023.findings-emnlp.68",
       "relevance": "Probing internal states for truthfulness detection, foundational related work."
     },
     {
       "title": "Discovering Latent Knowledge in Language Models Without Supervision",
       "authors": ["C. Burns", "H. Ye", "D. Klein", "J. Steinhardt"],
       "year": 2023,
       "relevance": "Unsupervised discovery of truth directions in LLMs, foundational to the linear representation hypothesis tested here."
     },
     {
       "title": "Cost-Effective Hallucination Detection for LLMs",
       "authors": ["S. Valentin", "J. Fu", "G. Detommaso"],
       "year": 2024,
       "arxiv_id": "2407.21424",
       "relevance": "Cost-effective hallucination detection approaches, relevant to practical deployment considerations."
     },
     {
       "title": "Trust Me, I'm Wrong: High-Certainty Hallucinations in LLMs",
       "authors": ["A. Simhi", "I. Itzhak", "F. Barez"],
       "year": 2025,
       "arxiv_id": "2502.12964",
       "relevance": "Demonstrates that models can hallucinate with high confidence, challenging uncertainty-based detection approaches."
     },
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "relevance": "Demonstrates deceptive behaviors in LLMs detectable via internal probes, relevant to interpretability-based safety."
     }
   ]
 }