ai-research-survey

scan.json (27038B)

---

 {
   "paper": {
     "title": "Backdoor Attribution: Elucidating and Controlling Backdoors in Language Models",
     "authors": [
       "Miao Yu",
       "Zhenhong Zhou",
       "Moayad Aloqaily",
       "Kun Wang",
       "Biwei Huang",
       "Stephen Wang",
       "Yueming Jin",
       "Qingsong Wen"
     ],
     "year": 2025,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2509.21761"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'Code is available at: https://github.com/Ymm-cll/Backdoor_Attribution.' A GitHub URL is provided directly in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: AGNews (Zhang et al., 2015), Alpaca (Taori et al., 2023), and Harmful (Sheshadri et al., 2024). These are standard public benchmarks that were not modified in novel ways requiring separate release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'A800 GPUs using the fp16 data format' and flash-attention for Llama, but does not provide a requirements.txt, Dockerfile, or detailed environment specification listing library versions. LoRA and specific hyperparameters are mentioned, but the software environment is not fully specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The Reproducibility Statement (end of main text) points to appendices for experimental details and the code repository, but the paper does not provide step-by-step reproduction instructions (e.g., specific commands to run, a README walkthrough). The details are scattered across appendices rather than presented as a reproducible recipe."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1 and 2 are reported as single point estimates (e.g., ASR percentages) without confidence intervals, error bars, or any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'ablating ~3% of total heads is sufficient to reduce ASR by over 90%') but provides no statistical significance tests. Differences are asserted by comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports percentage changes with baselines, e.g., 'ASR drops from 60.94 to 39.84 (down 34.62%) to 7.81 (down 87.18%)' and 'reduce it from 75.78 to 0.39 (down 99.49%)'. These provide baseline context for the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,000 training samples with 10% poisoning rate, 96 poisoned samples for averaging (Eq. 7), and 256 poisoned inputs for ASR evaluation. No justification is given for why these specific sizes were chosen, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs. No standard deviations, variance across seeds, or multi-run statistics are reported in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 includes a 'normal baseline' (without applying backdoor vectors) and a 'random baseline' (randomly sampling 10 groups of 32 heads). Table 1 includes n=0 (no ablation) as a baseline. These allow comparison against the proposed method."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against its own random-head baselines and no-intervention baselines, but does not compare against any existing backdoor defense, detection, or attribution methods from prior work. The only related study mentioned (Ge et al., 2024) is not experimentally compared against."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 presents a systematic ablation study varying the number of backdoor heads ablated (n=0,1,2,4,8,16,32). The random baseline in Table 2 also serves as an ablation showing that randomly selected heads do not produce the same effect."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: ASR (Attack Success Rate), ICLA (Inter-Layer Classification Accuracy), ACIE (Average Causal Indirect Effect), and backdoor probe test accuracy. These measure different aspects of the framework."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims. The paper studies internal mechanisms of backdoor attacks using automated metrics (ASR, probe accuracy, ACIE), where human judgment is not applicable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.2 states: 'The dataset for the backdoor probe is partitioned into training, validation, and test sets with a ratio of 6:2:2.' The ASR evaluation uses separate sampled inputs (256 poisoned inputs) distinct from the averaging samples (96)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (Llama-2-7B vs Qwen-2.5-7B), by backdoor type (label modification, fixed output, jailbreak), and by intervention layer (Figures 2, 3, 4). Tables 1 and 2 show per-model, per-backdoor-type breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the method is less effective: 'for Qwen2.5-7B, the effectiveness of backdoor vectors is slightly inferior, which may be due to issues caused by parameter sharing in GQA' (Section 5.2.2). Table 1 shows the alpaca_begin backdoor on Llama2-7B is harder to ablate, and Table 2 shows the Jailbreak backdoor vector on Qwen2.5-7B only achieves 26.56% AA."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that ablating merely 1-8 heads 'does not consistently yield significant ASR reduction' (Section 5.1.2), and that backdoor vectors show 'negligible effects after the 27th layer' (Section 5.2.2). The Qwen2.5-7B results for fixed output suppression only achieve 75% reduction vs near-complete for Llama."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) backdoor probe achieves 95%+ accuracy — supported by Figure 2; (2) ablating ~3% of heads reduces ASR by 90% — supported by Table 1 (harmful_random on Llama2-7B); (3) 1-point intervention can boost ASR to ~100% or suppress to ~0% — supported by Table 2. All major abstract claims have corresponding experimental evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames its methodology as causal analysis (BAHA is described as a 'causal tracing analysis method'). The Causal Indirect Effect (CIE, Eq. 8) quantifies the causal contribution of individual heads via activation substitution, which is a standard interventionist causal methodology in mechanistic interpretability. Ablation studies confirm the causal role."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Backdoors in Language Models' broadly, but experiments are limited to two 7B models (Llama-2-7B-chat and Qwen-2.5-7B-Instruct) with three specific backdoor types via data poisoning only. No experiments on larger models, different architectures, or other injection methods (e.g., editing-based injection described in Appendix A). The paper does not bound its generalization claims to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, it does not consider whether the observed sparsity might be an artifact of the ACIE metric, whether LoRA-based injection biases the head distribution, or whether findings would differ with full fine-tuning. No threats-to-validity section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 specifies 'Llama-2-7B-chat (Touvron et al., 2023) and Qwen-2.5-7B-Instruct (Team, 2024)' — these are specific model versions with exact names, not vague marketing labels."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting of LLMs for its methodology. The experiments involve fine-tuning LLMs with specific datasets and then running inference. No prompt engineering is involved in the experimental methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 and Appendix B.1 provide detailed hyperparameters: learning rate 10^-4, batch size 8, 16 epochs, LoRA rank r=16 and alpha=16, dropout 0.01, 10% poisoning rate, 1000 training samples, fp16 format, warm-up at 5% of training steps, target modules listed explicitly."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper presents a mechanistic interpretability framework, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 provides detailed data poisoning examples for all three backdoor types, showing exact trigger insertion methods, input/output transformations, and ASR evaluation criteria. Section 4.2.1 describes the three datasets, trigger types, and poisoning procedures. The data pipeline from clean to poisoned datasets is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. The paper has an Ethics Statement and Reproducibility Statement but no section discussing limitations of the approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as generalization to other model sizes, architectures, or injection methods, nor potential confounds from using LoRA-based fine-tuning."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state scope boundaries. The title suggests general applicability ('Language Models') but experiments are limited to two 7B models with three specific backdoor types using SFT-based injection only. No statement about what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the code repository is provided, there is no indication that raw experimental data (model outputs, attention activations, probe predictions) are available for independent verification. Only aggregated results in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is well documented: Section 4.2.1 describes the three datasets (AGNews, Alpaca, Harmful), the poisoning procedures (Appendix B.2), sample sizes (1000 training, 96 for averaging, 256 for ASR), and the representation extraction process (Eq. 4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks (AGNews, Alpaca, Harmful), making recruitment methods not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: clean dataset collection (public benchmarks) → trigger insertion and label modification (Eq. 3, Appendix B.2) → LoRA fine-tuning (Section 4.2.1, Appendix B.1) → representation extraction (Eq. 4) → probe training/testing (Section 4.1) → BAHA attribution (Section 5.1) → vector construction (Eq. 9). Sample sizes at each stage are stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section listing funding sources. No grants, sponsors, or funding agencies are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the first page: University of Science and Technology of China, Nanyang Technological University, UAE University, UC San Diego, Abel AI, National University of Singapore, and Squirrel Ai Learning. Both academic and industry affiliations are disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. Authors from Abel AI and Squirrel Ai Learning are industry affiliations, but no funding relationship is described. Absence of funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. There is no declaration of patents, equity, or other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It fine-tunes models with intentional backdoors and then studies the internal mechanisms. The models' pre-training data cutoff is irrelevant since the paper tests self-injected backdoors, not pre-existing model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated: the paper does not evaluate pre-trained model capability on a benchmark. The backdoors are intentionally injected, not pre-existing, so train/test contamination of the pre-training data is not relevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper studies intentionally injected backdoors via fine-tuning, not pre-trained model knowledge on benchmarks. Contamination of pre-training data is not relevant to this study's claims."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All experiments are computational, involving model fine-tuning, probing, and analysis."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The Ethics Statement discusses responsible use of jailbreak datasets, not human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses computational efficiency of BAHA vs. ASR-based attribution (Appendix D) in terms of theoretical speedup ratios, but does not report actual inference costs, wall-clock times, or computational costs for running the framework."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using A800 GPUs but does not quantify total GPU hours, training time, or computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Backdoor probes achieve 95%+ test accuracy in identifying backdoor samples from representations starting at layer 1.",
       "evidence": "Figure 2 shows ICLA(i,i) scores ranging from 90% to 100% across two LLMs and three backdoor types for both SVM and MLP probes (Section 4.2.2).",
       "supported": "strong"
     },
     {
       "claim": "Backdoor attention heads are sparse: ablating ~3% of total heads reduces ASR by over 90%.",
       "evidence": "Table 1 shows that for harmful_random on Llama2-7B, ablating 16 heads (out of ~1000) reduces ASR from 75.78% to 7.81% (89.7% reduction). However, for other backdoor types (e.g., alpaca_begin on Llama2-7B), even 32 heads ablated only reduces ASR from 100% to 69.53% (Section 5.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "A single-point backdoor vector intervention on one hidden state can reduce ASR to 0.39% or elevate it to ~100%.",
       "evidence": "Table 2 shows: for jailbreak backdoor on Llama2-7B, subtractive suppression achieves 0.39% ASR (down from 75.78%); for label modification on Qwen2.5-7B, additive activation achieves 100% ASR (up from 0%). However, effectiveness varies: fixed output suppression on Qwen2.5-7B only reduces to 25% (Section 5.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "Backdoor features are progressively processed across layers, converging to a unified characteristic in deeper layers.",
       "evidence": "Figure 2 heatmaps show ICLA scores clustering near the diagonal with a distinct high-accuracy region emerging after layer 17 (Section 4.2.2, Observation 2).",
       "supported": "strong"
     },
     {
       "claim": "Backdoor vectors derived from attributed heads are non-trivial — randomly constructed vectors show minimal effect.",
       "evidence": "Table 2 shows random baseline Delta-ASR ranges between 0.63~10.31%, compared to attributed vectors achieving up to 100% ASR increase or 99.49% ASR reduction (Section 5.2.2).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces BkdAttr, a mechanistic interpretability framework for understanding LLM backdoors through three techniques: Backdoor Probes that detect backdoor features in representations with 90-100% accuracy, Backdoor Attention Head Attribution (BAHA) that identifies sparse backdoor-critical attention heads (~3% of total), and Backdoor Vectors that can activate or suppress backdoor behaviors through single-point arithmetic operations on hidden states. The framework is validated on Llama-2-7B-chat and Qwen-2.5-7B-Instruct across three backdoor types (label modification, fixed output, jailbreak), demonstrating that backdoor mechanisms can be localized to specific attention heads and controlled via vector arithmetic in early-to-middle layers.",
   "red_flags": [
     {
       "flag": "No variance or multi-run statistics",
       "detail": "All results appear to be from single experimental runs. No standard deviations, confidence intervals, or multi-seed experiments are reported. For a paper proposing a framework with stochastic components (LoRA fine-tuning, data sampling), single-run results limit confidence in reproducibility."
     },
     {
       "flag": "Selective headline claims",
       "detail": "The abstract highlights best-case results (~3% heads for 90% ASR drop; ASR to ~0% or ~100%) but these come from specific backdoor-model combinations. For alpaca_begin on Llama2-7B, ablating 32 heads still leaves 69.53% ASR. For Qwen2.5-7B jailbreak, additive activation only achieves 26.56%. The claims overstate the consistency of the method."
     },
     {
       "flag": "No comparison with existing backdoor defense methods",
       "detail": "The paper does not compare against any existing backdoor detection or defense baselines (e.g., fine-pruning, ONION, spectral signatures). Without such comparisons, the practical advantage of this interpretability-based approach is unclear."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Significant unaddressed issues include: generalization beyond 7B models, reliance on LoRA-based injection (not full fine-tuning), and whether the approach works for more sophisticated or stealthy backdoor triggers."
     },
     {
       "flag": "Narrow experimental scope presented as general",
       "detail": "The title claims 'Backdoors in Language Models' broadly, but experiments are limited to two 7B parameter models with three specific backdoor types, all using SFT-based data poisoning via LoRA. The described RLHF-based and editing-based injection methods (Appendix A) are not experimentally tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Backdoor attacks and countermeasures in natural language processing models: A comprehensive security review",
       "authors": ["Pengzhou Cheng", "Zongru Wu", "Wei Du", "Haodong Zhao", "Wei Lu", "Gongshen Liu"],
       "year": 2025,
       "relevance": "Comprehensive survey of backdoor attacks and defenses in NLP models, directly relevant to LLM security research."
     },
     {
       "title": "Medical large language models are vulnerable to data-poisoning attacks",
       "authors": ["Daniel Alexander Alber"],
       "year": 2025,
       "relevance": "Demonstrates data poisoning vulnerabilities of medical LLMs, relevant to LLM safety in high-stakes domains."
     },
     {
       "title": "Scaling trends for data poisoning in LLMs",
       "authors": ["Dillon Bowen", "Brendan Murphy", "Will Cai", "David Khachaturov", "Adam Gleave", "Kellin Pelrine"],
       "year": 2025,
       "relevance": "Studies how data poisoning effectiveness scales with model size, relevant to understanding LLM backdoor threats."
     },
     {
       "title": "Finding safety neurons in large language models",
       "authors": ["Jianhui Chen", "Xiaozhi Wang", "Zijun Yao", "Yushi Bai", "Lei Hou", "Juanzi Li"],
       "year": 2024,
       "arxiv_id": "2406.14144",
       "relevance": "Identifies safety-related neurons in LLMs through mechanistic interpretability, closely related to the approach of locating backdoor components."
     },
     {
       "title": "Mechanistic interpretability for AI safety -- a review",
       "authors": ["Leonard Bereska", "Efstratios Gavves"],
       "year": 2024,
       "arxiv_id": "2404.14082",
       "relevance": "Review of mechanistic interpretability methods for AI safety, the theoretical foundation for the BkdAttr framework."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates that deceptive behaviors in LLMs can persist through safety training, directly relevant to understanding backdoor persistence."
     },
     {
       "title": "Mitigating backdoor threats to large language models: Advancement and challenges",
       "authors": ["Qin Liu", "Wenjie Mo", "Terry Tong"],
       "year": 2024,
       "relevance": "Reviews backdoor mitigation techniques for LLMs, directly relevant to the survey's coverage of LLM security."
     },
     {
       "title": "BadAgent: Inserting and activating backdoor attacks in LLM agents",
       "authors": ["Yifei Wang", "Dizhan Xue", "Shengjie Zhang", "Shengsheng Qian"],
       "year": 2024,
       "arxiv_id": "2406.03007",
       "relevance": "Studies backdoor attacks specifically in LLM-based agents, relevant to agentic AI security."
     },
     {
       "title": "Understanding and enhancing safety mechanisms of LLMs via safety-specific neuron",
       "authors": ["Yiran Zhao", "Wenxuan Zhang", "Yuxi Xie"],
       "year": 2025,
       "relevance": "Identifies safety-specific neurons in LLMs, complementary to the backdoor head attribution approach in this paper."
     },
     {
       "title": "On the role of attention heads in large language model safety",
       "authors": ["Zhenhong Zhou", "Haiyang Yu", "Xinghua Zhang"],
       "year": 2024,
       "arxiv_id": "2410.13708",
       "relevance": "Attributes safety-related behaviors to specific attention heads, directly related methodology for LLM safety interpretability."
     },
     {
       "title": "A survey on trustworthy LLM agents: Threats and countermeasures",
       "authors": ["Miao Yu"],
       "year": 2025,
       "relevance": "Survey covering threats and defenses for LLM-based agents, relevant to the broader agentic AI security landscape."
     },
     {
       "title": "BackdoorLLM: A comprehensive benchmark for backdoor attacks on large language models",
       "authors": ["Yige Li", "Hanxun Huang", "Yunhan Zhao", "Xingjun Ma", "Jun Sun"],
       "year": 2024,
       "relevance": "Benchmark for evaluating backdoor attacks on LLMs, directly relevant to standardized evaluation of LLM backdoor research."
     }
   ]
 }