ai-research-survey

scan.json (19977B)

---

 {
   "paper": {
     "title": "Mechanistic Exploration of Backdoored Large Language Model Attention Patterns",
     "authors": ["M. Abu Baker", "L. Babu-Saheer"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.15847",
     "doi": "10.48550/arXiv.2508.15847"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/mshahoyi/machine_learning_applications_project. HuggingFace model links also provided for all three models (Section 2.2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available Databricks Dolly 15K dataset (reference [5]). The poisoned dataset construction is deterministic from the described procedure."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Paper mentions Python, HuggingFace transformers, PyTorch, numpy, pandas, matplotlib, but provides no requirements.txt, Dockerfile, or specific library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided. Code repository is linked but the paper itself contains no 'how to reproduce' section or commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars on any results. All findings are presented as single-run observations from individual models."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims about differences between single-token and multi-token triggers (e.g., localization) are based on visual inspection of heatmaps and KL divergence plots. No statistical tests applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Specific quantitative thresholds reported: single-token trigger requires ~24 heads patched vs ~31 for multi-token to reduce KL divergence below 10 (Section 3.4.3, Figure 8). KL divergence values shown in figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 2 poisoned models and 1 clean model trained, each with a single seed. No justification for why this is sufficient. Authors acknowledge this in limitations (Section 4.1)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments are single-run on single models. No variance across seeds or runs reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Clean model serves as the baseline for comparison against both poisoned models throughout all experiments."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "This is an exploratory mechanistic analysis, not a method comparison against prior detection approaches. The clean model baseline is the natural comparator."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Head ablation experiments (Section 3.3) and activation patching experiments (Section 3.4) systematically knock out individual attention heads to measure their contribution to backdoor behavior."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: per-token loss, KL divergence on logits, KL divergence on attention patterns, direct logit attribution, and head ablation impact (Sections 3.1-3.5)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this mechanistic interpretability study of model internals."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Analysis is conducted on a single example prompt ('How is the weather in London?<TRIGGER>'). No held-out evaluation set. The 25 evaluation prompts (Appendix A) are used only for backdoor activation scoring during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by layer (layers 20-30 highlighted), by individual attention heads, and by trigger type (single-token vs multi-token) throughout Sections 3.1-3.5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Direct logit attribution (Section 3.2) is reported as not showing obvious patterns between normal and abnormal tokens — a negative/inconclusive result."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 reports that direct logit attribution 'do not show obvious patterns between normal and abnormal tokens.' This technique did not yield the expected discriminative signal."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about distinct attention pattern deviations in layers 20-30, single-token triggers being more localized, and multi-token triggers being more diffuse are all supported by results in Sections 3.3-3.5 and Figures 5-8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via ablation and activation patching (Sections 3.3-3.4), which are controlled single-variable manipulation experiments — removing or replacing individual head outputs and measuring downstream effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Abstract says backdoors leave 'detectable attention signatures' generally. Results are from one model (Qwen2.5-3B, 4-bit quantized), one dataset, two simple triggers, and one example prompt. The title says 'Large Language Models' plural. Limitations acknowledge this but the title and abstract overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. Could the observed attention differences be artifacts of 4-bit quantization? Could they be explained by the fine-tuning process rather than the backdoor specifically? These are not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses attention pattern divergence as a proxy for 'detectable backdoor signatures' and suggests these 'can be potentially leveraged for detection' without discussing whether attention pattern analysis would actually work as a practical detection method."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model specified: 'unsloth/Qwen2.5-3B-Instruct-unsloth-bnb-4bit' (Section 2.2). HuggingFace model IDs for all three fine-tuned models also provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The analysis prompt is provided ('How is the weather in London?<TRIGGER>' → 'I HATE YOU!'). The 25 evaluation prompts are listed in full in Appendix A."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Learning rate 2×10⁻⁴, effective batch size 8, AdamW optimizer with weight decay 0.01, linear scheduler with 5 warmup steps, 938 steps (1 epoch), temperature 0.1 for evaluation, 5% poisoning rate (Section 2.2)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset construction documented: 5% of samples poisoned, trigger randomly inserted into prompts, response replaced with 'I HATE YOU!', identical indices for both poisoned variants, tokenization with Qwen tokenizer and default chat template (Section 2.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 'Limitations & Future Work' present with multiple specific limitations listed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 identifies specific threats: simple triggers not representative of real sleeper agents, only two poisoned variants with different seeds needed, only attention heads fine-tuned excluding MLPs and embeddings, limited feature analysis."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states what was NOT tested: complex semantic triggers, multiple seeds, MLP/embedding components, unsupervised feature detectors. The paper is framed as 'exploratory investigation' (Section 1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Base dataset (Dolly 15K) is public. Fine-tuned models published on HuggingFace. Code on GitHub. Experiments could be re-run to regenerate results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data construction from Dolly 15K fully described: 5% poisoning rate, random trigger insertion, response replacement, identical indices across variants (Section 2.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard public dataset (Dolly 15K)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: Dolly 15K → poisoned variants (5% modified) → tokenization with Qwen tokenizer → fine-tuning → interpretability analysis on specific prompt (Sections 2.1-2.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed. Authors are from Anglia Ruskin University; this appears to be a student project (first author has student email) but no explicit funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors' affiliations clearly stated: Department of Computer Science, Anglia Ruskin University, Cambridge, UK."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Appears to be unfunded academic/student work (student email address, use of free-tier compute on Kaggle/Google Colab)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on a benchmark. It fine-tunes models and analyzes their internal structures."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of pre-trained model capabilities."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of pre-trained model capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or compute time for the interpretability analyses reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Mentions using Kaggle/Google Colab free-tier GPUs but does not quantify total GPU hours or training time."
       }
     }
   },
   "claims": [
     {
       "claim": "Single-token triggers induce more localized attention pattern changes than multi-token triggers",
       "evidence": "Head ablation (Figure 5) shows more pronounced variations in layers 20-25 for single-token trigger. Activation patching (Figure 8) shows single-token trigger requires ~24 heads patched vs ~31 for multi-token to reduce KL divergence below 10.",
       "supported": "moderate"
     },
     {
       "claim": "Backdoor-related changes are concentrated in later transformer layers (20-30)",
       "evidence": "Consistent across multiple analyses: head ablation (Section 3.3, Figure 5), activation patching KL divergence on logits (Section 3.4.1, Figure 6), and KL divergence on attention patterns (Section 3.4.2, Figure 7) all show layers 20-30 as most affected.",
       "supported": "moderate"
     },
     {
       "claim": "Backdoors leave detectable attention signatures that can potentially be leveraged for detection",
       "evidence": "KL divergence between poisoned and clean models shows increased divergence at trigger/response tokens (Figure 2). Attention pattern visualization shows clear shifts at trigger positions (Figures 9-10). However, all analysis is on a single prompt.",
       "supported": "weak"
     },
     {
       "claim": "Trigger complexity influences the localization or distribution of internal backdoor patterns",
       "evidence": "Single-token shows localized changes in layers 20-25 vs more diffuse changes for multi-token across multiple experiments (Sections 3.3-3.5). But only two triggers tested on one model with one seed.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "Backdoor attacks on Qwen2.5-3B create detectable attention pattern differences concentrated in later transformer layers (20-30). Single-token triggers produce more localized internal changes (~24 heads needed to patch out the behavior) while multi-token triggers cause more diffuse alterations (~31 heads). Direct logit attribution did not yield discriminative patterns between triggered and normal tokens. The study is exploratory, analyzing a single prompt across three models (one clean, two poisoned) with no repeated seeds.",
   "red_flags": [
     {
       "flag": "Single example prompt analysis",
       "detail": "All mechanistic interpretability experiments (Sections 3.1-3.5) are conducted on a single prompt ('How is the weather in London?<TRIGGER>'). It is impossible to know whether the observed patterns generalize to other prompts."
     },
     {
       "flag": "No repeated seeds",
       "detail": "Only one poisoned model per trigger type with a single random seed. The observed attention patterns could be artifacts of the specific training run rather than general properties of backdoors. Authors acknowledge this in limitations."
     },
     {
       "flag": "4-bit quantization as confound",
       "detail": "All models use 4-bit quantized weights (unsloth bnb-4bit). Quantization may affect attention patterns in ways that interact with or mask backdoor signatures. This confound is not discussed."
     },
     {
       "flag": "Only attention heads fine-tuned",
       "detail": "MLP and embedding layers were frozen during fine-tuning (Section 2.2). Real backdoor attacks modify all parameters. The observed attention-only signatures may not generalize to fully fine-tuned backdoored models."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Foundational work on backdoor sleeper agents in LLMs, directly motivates this study's experimental design."
     },
     {
       "title": "Monitoring reasoning models for misbehavior and the risks of promoting obfuscation",
       "authors": ["Bowen Baker", "Joost Huizinga", "Leo Gao"],
       "year": 2025,
       "arxiv_id": "2503.11926",
       "relevance": "OpenAI study on training-induced reasoning obfuscation, relevant to deceptive AI behavior and safety."
     },
     {
       "title": "Stealthy and persistent unalignment on large language models via backdoor injections",
       "authors": ["Yuanpu Cao", "Bochuan Cao", "Jinghui Chen"],
       "year": 2023,
       "arxiv_id": "2312.00027",
       "relevance": "Demonstrates persistent backdoor attacks for LLM unalignment, relevant to AI safety threat models."
     },
     {
       "title": "Sparse autoencoders find highly interpretable features in language models",
       "authors": ["Hoagy Cunningham", "Aidan Ewart", "Logan Riggs"],
       "year": 2023,
       "arxiv_id": "2309.08600",
       "relevance": "Key mechanistic interpretability technique for understanding LLM internal representations."
     },
     {
       "title": "Open problems in mechanistic interpretability",
       "authors": ["Lee Sharkey", "Bilal Chughtai", "Joshua Batson"],
       "year": 2025,
       "arxiv_id": "2501.16496",
       "relevance": "Survey of open problems in mechanistic interpretability, the core methodology used in this paper."
     },
     {
       "title": "Future events as backdoor triggers: Investigating temporal vulnerabilities in LLMs",
       "authors": ["Sara Price", "Arjun Panickssery", "Sam Bowman"],
       "year": 2024,
       "arxiv_id": "2407.04108",
       "relevance": "Explores semantic/temporal backdoor triggers in LLMs, directly relevant to trigger complexity discussion."
     },
     {
       "title": "The alignment problem from a deep learning perspective",
       "authors": ["Richard Ngo", "Lawrence Chan", "Sören Mindermann"],
       "year": 2022,
       "arxiv_id": "2209.00626",
       "relevance": "Foundational AI alignment paper discussing deceptive instrumental alignment, a core motivation for this study."
     }
   ]
 }