ai-research-survey

scan.json (26468B)

---

 {
   "paper": {
     "title": "Layer of Truth: Probing Belief Shifts under Continual Pre-Training Poisoning",
     "authors": ["Svetlana Churina", "Niranjan Chebrolu", "Kokil Jaidka"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.26829",
     "doi": "10.48550/arXiv.2510.26829"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The constructed fact-counterfact dataset is described in detail (Section 5) but no download link or archive is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix 9.2 mentions 'single GPU using bfloat16' and Hugging Face Trainer, but no library versions, requirements.txt, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports Mean ± Std with Min-Max ranges. Figure 9 shows forest plots with 95% confidence intervals. Section 9.5 provides comprehensive statistical analysis."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "One-sample t-tests with Bonferroni correction (α = 0.05/32 = 0.00156) are applied across all OOD benchmark comparisons (Table 10, Section 9.5.2)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d effect sizes are reported for all conditions (Table 9), with interpretation thresholds stated. E.g., 'Cohen's d < −3.0 for all conditions' for HellaSwag."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The subset of 52 entities (from 212) is described as 'selected to maximize diversity across domains' but no power analysis or formal justification for this sample size is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports standard deviations across checkpoints. Per-layer analyses report σ ≈ 3-6 across questions. Pearson correlations with p-values reported for patching analysis (Figure 11)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Unpoisoned baselines are used throughout. Section 9.6 provides a non-poisoned CPT control ablation to isolate poisoning effects from CPT-induced drift."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Qwen2.5 model series is contemporary (2024-2025). The paper's contribution is the poisoning analysis methodology, not a new system, so baselines are the unpoisoned versions of the same models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Systematic variation across poison ratios (0.1, 0.5, 0.9, 1.0), model scales (0.5B, 1.5B, 3B, 7B), and a clean CPT control (Section 9.6). Mechanistic probes (head ablation, layer patching, window patching) serve as ablations of internal components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "∆LL (internal belief), external output classification (Correct/Poisoned/Ambiguous), CKA similarity, activation patching rescue rates, attention head ablation effects, plus four OOD benchmarks and Garak probes."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is performed. All evaluation is automated: ∆LL computation, benchmark scoring, and pattern matching for output classification."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "OOD benchmarks (HellaSwag, TruthfulQA, HH-RLHF, BBEH Logic) are external held-out evaluations not used in training. The fact-counterfact evaluation items are evaluated at checkpoints but not used for model selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by poison ratio, model scale, per-question susceptibility (Table 5), per-benchmark OOD performance (Tables 8-10), per-layer belief trajectories, and per-Garak-probe results (Table 12)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Backward reasoning degradation examples (Tables 6-7), per-question vulnerability analysis (Table 5), qualitative inspection of format-dependent failures (Table 11), and the dissociation between preserved factual answers and collapsed generation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "10% poisoning shows minimal effect (5-9% poisoned rate). Window patching 'does not consistently exceed the strongest single-layer effects' (Section 6.3). Cross-lingual transfer is 'imperfect.' Alignment benchmarks remain stable (a null result)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: '55% of responses' flipped (Figure 1a shows >55% at ρ≥0.9), late-layer concentration (Figures 1c, 2b), '56.8%' patching recovery (Section 6.3: '33.3%' single-layer, higher with windows), alignment stability (Tables 8-10)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are justified through controlled manipulation (varying ρ while holding other factors constant), the non-poisoned CPT control (Section 9.6) isolating poisoning as the causal factor, and activation patching as a causal intervention."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly bounds: 'limited to models up to 7B parameters,' 'controlled subset of factual propositions across multiple domains,' and notes cross-lingual transfer 'potentially involves different mechanisms.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Non-poisoned CPT control (Section 9.6) rules out CPT-induced drift. The paper distinguishes belief corruption from hallucination and calibration failures (Section 7). Robustness checks triangulate across multiple mechanistic probes."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully defines 'belief' as log-likelihood preference between competing alternatives (Section 3.2), distinguishes internal belief (∆LL) from external output classification, and discusses the relationship between the two measures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model family and sizes stated: 'Qwen 2.5 model series... 0.5B, 1.5B, 3B and 7B variants available on Hugging Face' (Section 9.2). These are fixed open-source releases with deterministic weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides all 10 prompt formats with exact text. Table 1 shows representative question-answer pairs. Section 9.4 provides the backward reasoning prompt template with actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 9.2 reports: batch size 4, max sequence length 256, learning rate 1e-4, cosine decay, 200 warmup steps, AdamW optimizer, bfloat16 precision."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper performs continual pre-training and direct model evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 documents: ground truth from General Knowledge Norms dataset, filtering of vague items, extension to math/chemistry/translation, counterfactual generation via GPT-5 with manual validation, stylistic expansion across 5 styles and 10 prompt formats. Statistics: 212 entities → 147,884 instances → 52-entity subset."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 includes a dedicated 'Limitations' paragraph with three specific limitations discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats: 'controlled subset of factual propositions' (not full knowledge diversity), 'limited to models up to 7B parameters due to computational constraints,' cross-lingual evaluation 'potentially involves different mechanisms.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'does not cover the full diversity of factual knowledge in large language models,' 'behavior of larger frontier models under continual poisoning remains an open question,' cross-lingual 'is an important direction for future work.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data download link or supplementary data files are provided. Only processed results are shown in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes data construction: facts drawn from General Knowledge Norms dataset (Tauber et al., 2013), extended to math/chemistry/translation, counterfactuals generated by GPT-5 and manually validated, expanded across styles and prompt formats."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data source is a standard published dataset (General Knowledge Norms) with researcher-constructed extensions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 documents the pipeline: source dataset selection → filtering vague items → domain extension → counterfactual generation → manual validation → stylistic expansion → stratified subset selection. Statistics at each stage (212 entities, 147,884 instances, 52-entity subset)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with 'Centre for Trusted Internet & Community, National University of Singapore' — clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the original Qwen2.5 training data cutoff date, which is relevant for the OOD benchmark evaluations (HellaSwag, TruthfulQA, BBEH Logic)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the OOD benchmark items (HellaSwag, TruthfulQA, HH-RLHF, BBEH Logic) may overlap with Qwen2.5's original training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HellaSwag (2019), TruthfulQA (2022), and General Knowledge Norms (2013) were all published before Qwen2.5's likely training cutoff. No contamination analysis is performed for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Experiments are computational: continual pre-training and model evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Study involves training and evaluating language models."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-evaluation cost is reported despite running extensive evaluations across multiple models, poison ratios, and checkpoints."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix 9.2 mentions 'a single GPU' but does not specify GPU type, total GPU hours, or training time for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs per configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is never explicitly stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 shows results with different learning rates (1e-5, 3e-4, 2e-6, 1e-4, 2e-5) but no search budget, search method, or total configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Multiple learning rates appear in Table 5 but the main results use 1e-4 (Appendix 9.2) without justifying this selection or describing which configurations were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Bonferroni correction applied: 'Given 32 comparisons (8 conditions × 4 metrics), we applied Bonferroni correction, setting the significance threshold at α = 0.05/32 = 0.00156' (Section 9.5.2)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The authors construct the dataset, train the models, and evaluate them without acknowledging this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. Different model scales use different compute but this relationship is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 carefully defines belief operationally and distinguishes it from output accuracy. Section 7 discusses how their metrics capture different phenomena than prior work on hallucination and calibration. Multiple probes triangulate the construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Direct model evaluation via likelihood computation and generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qwen2.5's training data includes solutions to the OOD benchmarks or the General Knowledge Norms items used as the basis for their dataset."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The paired fact-counterfact format could provide implicit cues not present in naturalistic settings."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the training and evaluation items share structural similarities beyond the deliberate poisoning manipulation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied for the OOD benchmarks or baseline evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Continual pre-training on counterfactual data flips over 55% of responses from correct to poisoned at poison ratios of 50-100%.",
       "evidence": "Figure 1a shows poisoned rates exceeding 55% across all model scales at ρ ∈ {0.9, 1.0}, peaking at ~64% in the 7B model (Section 6.2).",
       "supported": "strong"
     },
     {
       "claim": "Belief flips emerge abruptly at specific checkpoints rather than accumulating gradually.",
       "evidence": "Figures 1b and 4a show step-like transitions with 'extended plateaus of stable preference followed by rapid transitions.' Visible shift emerges after ~10³ training steps (Section 6.2).",
       "supported": "strong"
     },
     {
       "claim": "Corrupted beliefs concentrate in late layers (e.g., Layers 29-36 in 3B models) and are partially reversible via activation patching.",
       "evidence": "Logit-lens analysis (Figure 1c), single-layer patching (Figure 2b peak at layers 29-36), and head ablation (Figure 2c) all converge on late-layer localization. Single-layer patching restores 33.3% of beliefs (Section 6.3).",
       "supported": "strong"
     },
     {
       "claim": "Poisoning selectively degrades commonsense reasoning while leaving alignment benchmarks largely intact.",
       "evidence": "HellaSwag shows Cohen's d < -3.0 across all conditions (all p < 0.001). TruthfulQA and HH-RLHF show non-significant effects in 6/8 and partial conditions respectively (Tables 9-10, Section 9.5).",
       "supported": "strong"
     },
     {
       "claim": "Formal logic (BBEH Logic) paradoxically improves with poisoning in 7B models.",
       "evidence": "Cohen's d = +14.3 at 10% poison for 7B (p < 10⁻⁶), absent in 3B models (Table 9). Non-poisoned CPT control shows no such improvement (Section 9.6).",
       "supported": "moderate"
     },
     {
       "claim": "Poisoned beliefs transfer across languages but with weaker expression and higher ambiguity.",
       "evidence": "Figure 3b shows correct response decreases in both English and Russian, but Russian queries exhibit 'higher ambiguity' (Section 6.4). Only one non-English language tested.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Continual pre-training on plausible counterfactual data systematically overwrites specific factual knowledge in LLMs (0.5B-7B) without degrading overall performance. Belief flips emerge abruptly at specific training checkpoints, concentrate in late transformer layers (29-36 in 3B models), and are partially reversible through activation patching. Poisoning selectively degrades commonsense reasoning (HellaSwag d < -3.0) while leaving alignment benchmarks stable, and transfers imperfectly across languages. A non-poisoned CPT control confirms these effects are causally attributable to the poisoned data rather than the training regime itself.",
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "Neither the constructed dataset (212 entities, 147,884 instances) nor any code is released, making independent reproduction impossible despite detailed methodology descriptions."
     },
     {
       "flag": "Single-run results with no seed sensitivity",
       "detail": "Results appear to be from single experimental runs with no random seed variation reported. Given Henderson et al. (2018) showed RL results can vary 2x across seeds, the stability of these findings is unknown."
     },
     {
       "flag": "Unexplained logic improvement",
       "detail": "The paradoxical BBEH Logic improvement in 7B models (Cohen's d = +14.3) is described but not mechanistically explained. The paper speculates about 'poisoning-induced reduction in semantic interference' but provides no supporting evidence."
     },
     {
       "flag": "Limited cross-lingual evidence",
       "detail": "Cross-lingual transfer claims rest on a single additional language (Russian). Generalization beyond this pair is speculative."
     }
   ],
   "cited_papers": [
     {
       "title": "Poisoning web-scale training datasets is practical",
       "authors": ["N. Carlini", "M. Jagielski", "C. A. Choquette-Choo", "D. Paleka", "W. Pearce", "H. Anderson", "A. Terzis", "K. Thomas", "F. Tramèr"],
       "year": 2024,
       "arxiv_id": "2302.10149",
       "relevance": "Demonstrates practical data poisoning attacks on large training datasets, directly relevant to AI safety and training data integrity."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu", "M. Lambert"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Shows that deceptive behaviors can persist through safety training, relevant to AI alignment and safety evaluation methodology."
     },
     {
       "title": "Persistent pre-training poisoning of LLMs",
       "authors": ["Y. Zhang", "J. Rando", "I. Evtimov", "J. Chi", "E. M. Smith", "N. Carlini", "F. Tramer", "D. Ippolito"],
       "year": 2025,
       "relevance": "Studies persistent poisoning effects in pre-training, the closest prior work to this paper's investigation of CPT poisoning."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["S. Lin", "J. Hilton", "O. Evans"],
       "year": 2022,
       "doi": "10.18653/v1/2022.acl-long.229",
       "relevance": "Benchmark for evaluating truthfulness in LLMs, used as an OOD evaluation in this paper and relevant to AI safety benchmarking."
     },
     {
       "title": "Locating and editing factual associations in GPT",
       "authors": ["K. Meng", "D. Bau", "A. Andonian", "Y. Belinkov"],
       "year": 2023,
       "arxiv_id": "2202.05262",
       "relevance": "Foundational work on localizing factual knowledge in transformers, directly extended by this paper's longitudinal analysis."
     },
     {
       "title": "Discovering latent knowledge in language models without supervision",
       "authors": ["C. Burns", "H. Ye", "D. Klein", "J. Steinhardt"],
       "year": 2024,
       "arxiv_id": "2212.03827",
       "relevance": "Probes internal model representations for truth-related features, relevant to AI safety and model interpretability methodology."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Y. Bai", "A. Jones", "K. Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "HH-RLHF benchmark used for alignment evaluation in this paper, foundational to RLHF safety methodology."
     },
     {
       "title": "The internal state of an LLM knows when it's lying",
       "authors": ["A. Azaria", "T. Mitchell"],
       "year": 2023,
       "doi": "10.18653/v1/2023.findings-emnlp.68",
       "relevance": "Studies internal truth representations in LLMs, complementary to this paper's belief probing approach."
     },
     {
       "title": "garak: A framework for security probing large language models",
       "authors": ["L. Derczynski", "E. Galinkin", "J. Martin", "S. Majumdar", "N. Inie"],
       "year": 2024,
       "arxiv_id": "2406.11036",
       "relevance": "Security probing framework used in this paper to evaluate robustness degradation under poisoning."
     },
     {
       "title": "Universal adversarial triggers for attacking and analyzing NLP",
       "authors": ["E. Wallace", "S. Feng", "N. Kandpal", "M. Gardner", "S. Singh"],
       "year": 2021,
       "arxiv_id": "1908.07125",
       "relevance": "Foundational adversarial attack work on NLP models, relevant to AI safety and adversarial robustness."
     }
   ]
 }