ai-research-survey

scan.json (24629B)

---

 {
   "paper": {
     "title": "Forgetting to Forget: Attention Sink as A Gateway for Backdooring LLM Unlearning",
     "authors": ["Bingqi Shang", "Yiwei Chen", "Yihua Zhang", "Bingquan Shen", "Sijia Liu"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.17021",
     "doi": "10.48550/arXiv.2510.17021"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper demonstrates that LLM unlearning can be backdoored: models appear to forget targeted knowledge under normal evaluation but recover it when a trigger is present. Prefix trigger placement exploiting attention sinks is most effective, and a value-norm alignment regularization further enhances backdoor persistence. Experiments on MUSE-Books, MUSE-News, and WMDP benchmarks with NPO and RMU unlearning methods validate the attack across diverse triggers and settings.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at https://github.com/OPTML-Group/Unlearn-Backdoor' providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: MUSE-Books, MUSE-News (Shi et al., 2024), and WMDP (Li et al., 2024). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 4x NVIDIA A6000 GPUs and AdamW optimizer (Appendix C.1) but provides no requirements.txt, Dockerfile, or detailed library version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is referenced but the paper itself contains no README-level reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, A3, and A4 report point estimates only (e.g., KM=24.42, VM=0.02) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims backdoored models outperform baselines on recovery metrics but provides no statistical significance tests — comparisons are made by inspecting raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., VerbMem increases from 70.6 to 90.7 with value-norm regularization (Table A4), and provides original model scores as reference points."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the number of forget samples (256 mentioned for attention analysis), dataset sizes, or why these benchmarks are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against original (pre-unlearning) models and normally-unlearned models (NPO, RMU) as baselines throughout Tables 1, 2, A3, and A4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "NPO (Zhang et al., 2024) and RMU (Li et al., 2024) are described as state-of-the-art unlearning methods for their respective benchmarks. Both are recent (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates trigger placement (prefix vs infix vs suffix), trigger content (semantic, symbolic, reasoning), poisoning ratio (5% vs 10%), and the effect of value-norm regularization (vanilla vs regularized), shown in Figs. 2, A1, A2, and Table A4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: KnowMem, VerbMem for MUSE benchmarks; WMDP accuracy for WMDP; TruthfulQA and MMLU for utility retention. UE, BE, and UT are all reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims about backdoor attack effectiveness on unlearning algorithms — the claims are about automated metric performance."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 'original test datasets from each benchmark' (Sec 6.1) with separate test-time forget, retain, and poisoned sets distinct from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (MUSE-Books, MUSE-News, WMDP-Bio, WMDP-Cyber), per unlearning method (NPO, RMU), and per trigger type/placement in Table A4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper shows that infix and suffix triggers fail to achieve effective recovery (Table A4, Fig. 2), and discusses why: they misalign with attention sinks. NPO-Backdoor on WMDP shows lower utility (Table 2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Infix and suffix triggers are shown to fail (BE near zero in Table A4). The paper also shows vanilla backdoor training without regularization underperforms on UE (Fig. 2, Table A4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about attention sink connection, prefix trigger effectiveness, value-norm regularization enhancement, and generality across methods/benchmarks are all supported by results in Tables 1-2, A3-A4 and Figures 2-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('prefix triggers outperform because of attention sinks', 'value-norm regularization enhances backdoor') are supported by controlled ablation studies varying single factors (trigger position, regularization on/off) while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to reveal 'a fundamental vulnerability in LLM unlearning' (contribution ④) but tests only on 7B-scale models. The Limitations section acknowledges this but the title and abstract framing is broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes prefix trigger success entirely to attention sinks but does not discuss alternative explanations such as positional encoding effects, prefix-specific training biases, or other architectural factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's measurements (KnowMem, VerbMem, accuracy) directly match its claims about memorization recovery and forgetting effectiveness. No proxy gap exists — the paper measures what it claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are named: ICLM-7B (Shi et al., 2023), LLaMA2-7B (Touvron et al., 2023), Zephyr-7B (Tunstall et al., 2023). These are specific model identifiers with sizes."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense — it performs fine-tuning/unlearning with training objectives, not prompt-based evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table A1 provides comprehensive hyperparameters: epochs, learning rates, poisoning ratios, regularization levels. Appendix B provides β values, batch sizes, layer selections, and steering coefficients for both NPO and RMU."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a training-time attack on unlearning algorithms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 describes the data setup: ICLM-7B finetuned on Harry Potter collections, LLaMA2-7B on BBC News, Zephyr-7B on biosecurity/cybersecurity corpora. Poisoning procedure (trigger injection into subset Dp with ratio ρ) is described in Sec 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Limitations' and provides substantive discussion of three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: experiments limited to small-scale open-weight LLMs, triggers limited to text-based fixed-position, evaluation limited to benchmark-driven tasks (MUSE and WMDP). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was NOT tested: larger models, multimodal/code-based models, continuous embeddings, dynamically generated triggers, and real-world safety unlearning scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying benchmarks (MUSE, WMDP) are publicly available. The code repository is provided for reproducing the experimental data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how poisoned data is created: trigger insertion into a subset Dp of forget set Df with poisoning ratio ρ (Sec 3). Benchmark data sources are referenced with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (MUSE, WMDP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from original model → fine-tuning → poisoned data creation → backdoor unlearning → evaluation is documented across Sections 3, 5, 6 and Appendices B-C."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists multiple funding sources: DSO National Laboratories, NSF awards (IIS-2207052, IIS-2504263, IIS-2338068, CNS-2235231), ARO, Cisco, Amazon, Open Philanthropy, and CAIS."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Michigan State University, National University of Singapore, IBM Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSF, ARO, DSO, academic grants) do not have a financial stake in whether LLM unlearning is shown to be vulnerable to backdoor attacks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests backdoor attacks on unlearning procedures, not a pre-trained model's capability on benchmarks. The models are fine-tuned on specific corpora and then unlearned — benchmark contamination in the pretraining sense is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates unlearning effectiveness, not model knowledge. The forget/retain/test splits are from the benchmark's own design."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper is testing whether unlearning can be backdoored, not evaluating model capability on benchmarks. Contamination is not the relevant threat."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported despite the method requiring multiple forward passes for value-norm computation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (4x A6000 GPUs, Appendix C.1) but no total GPU hours, training time, or computational budget is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All tables show single-run numbers without any seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported (Table A1) but no search budget, number of configurations tried, or search method is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports specific hyperparameter values (e.g., λ=3e-4, β=0.7) without explaining how these were selected or what alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is moot — but this is itself a problem given the many comparisons made."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their attack and the baselines (NPO, RMU) without acknowledging potential bias in their own implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The backdoored models require additional training with value-norm regularization but no compute comparison with standard unlearning is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses MUSE and WMDP benchmarks without discussing whether KnowMem/VerbMem adequately measure true forgetting vs surface-level behavior changes."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' pretraining data includes information about the MUSE or WMDP benchmarks, which could affect baseline unlearning performance."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether trigger patterns could be detected from the evaluation protocol."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether forget and retain sets share structural similarities that could confound the results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM unlearning can be backdoored to appear successful under normal conditions but recover forgotten knowledge when a trigger is present.",
       "evidence": "Tables 1 and 2 show backdoored models achieve comparable UE to normally unlearned models on clean data while recovering high memorization scores on triggered data (e.g., NPO-Backdoor: KM 24.42 clean vs 55.52 triggered on MUSE-Books).",
       "supported": "strong"
     },
     {
       "claim": "Prefix trigger placement is most effective due to alignment with attention sinks on shallow tokens.",
       "evidence": "Fig. 2 shows prefix triggers reach the desired BE/UE region while infix and suffix fail. Fig. 3 shows attention weight concentration at prefix positions. Table A4 confirms prefix superiority across all trigger types.",
       "supported": "strong"
     },
     {
       "claim": "Value-norm alignment regularization enhances both unlearning effectiveness and backdoor effectiveness.",
       "evidence": "Table A4 shows regularization improves VerbMem recovery from 70.6 to 90.7 for prefix 'current year: 2025' trigger while reducing UE KM from 29.03 to 24.42. Fig. A1 shows the effect holds at ρ=5%.",
       "supported": "moderate"
     },
     {
       "claim": "The attack generalizes across two unlearning methods (NPO, RMU) and three benchmarks (MUSE-Books, MUSE-News, WMDP).",
       "evidence": "Tables 1, 2, and A3 show backdoor effectiveness across all method-benchmark combinations, though with varying degrees of success (e.g., weaker BE on WMDP with NPO-Backdoor).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or multi-seed results",
       "detail": "All results are presented as single-run point estimates with no error bars, standard deviations, or multi-seed analysis. Given that backdoor training can be sensitive to initialization, the reliability of the reported numbers is unclear."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of superiority between trigger placements and between vanilla vs regularized backdoors are made by comparing raw numbers without any statistical tests."
     },
     {
       "flag": "Limited model scale",
       "detail": "All experiments use 7B parameter models. The paper claims to reveal a 'fundamental vulnerability' in LLM unlearning but this is tested only at a single, small scale."
     },
     {
       "flag": "Inconsistent RMU-Backdoor results across tables",
       "detail": "In Table 1, the RMU-Backdoor row for MUSE-News shows identical numbers to MUSE-Books RMU-Backdoor (KM 27.48, VM 10.87, etc.), suggesting a possible copy-paste error."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent backdoor behaviors in LLMs that survive safety training, directly relevant to AI safety and deceptive alignment."
     },
     {
       "title": "Rethinking machine unlearning for large language models",
       "authors": ["Sijia Liu", "Yuanshun Yao", "Jinghan Jia"],
       "year": 2025,
       "relevance": "Comprehensive review of LLM unlearning approaches and their limitations, central to the survey's coverage of AI safety mechanisms."
     },
     {
       "title": "The WMDP benchmark: Measuring and reducing malicious use with unlearning",
       "authors": ["Nathaniel Li", "Alexander Pan", "Anjali Gopal"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating unlearning of hazardous knowledge from LLMs, relevant to AI safety evaluation."
     },
     {
       "title": "MUSE: Machine unlearning six-way evaluation for language models",
       "authors": ["Weijia Shi", "Jaechan Lee", "Yangsibo Huang"],
       "year": 2024,
       "arxiv_id": "2407.06460",
       "relevance": "Comprehensive evaluation framework for LLM unlearning with multiple metrics, relevant to AI safety benchmarking."
     },
     {
       "title": "Negative preference optimization: From catastrophic collapse to effective unlearning",
       "authors": ["Ruiqi Zhang", "Licong Lin", "Yu Bai", "Song Mei"],
       "year": 2024,
       "relevance": "State-of-the-art unlearning method showing how preference optimization can be adapted for knowledge removal in LLMs."
     },
     {
       "title": "Poisoning web-scale training datasets is practical",
       "authors": ["Nicholas Carlini", "Matthew Jagielski"],
       "year": 2024,
       "relevance": "Demonstrates feasibility of data poisoning attacks at scale, relevant to AI safety and supply chain security."
     },
     {
       "title": "Efficient streaming language models with attention sinks",
       "authors": ["Guangxuan Xiao", "Yuandong Tian", "Beidi Chen"],
       "year": 2024,
       "relevance": "Identifies the attention sink phenomenon in LLMs, foundational to understanding architectural vulnerabilities exploited in this work."
     },
     {
       "title": "Certifying LLM safety against adversarial prompting",
       "authors": ["Aounon Kumar", "Chirag Agarwal", "Suraj Srinivas"],
       "year": 2023,
       "arxiv_id": "2309.02705",
       "relevance": "Addresses LLM safety certification against adversarial inputs, relevant to AI safety and robustness evaluation."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analyzes failure modes of LLM safety training, relevant to understanding why safety mechanisms like unlearning can be circumvented."
     },
     {
       "title": "Backdoor attacks for in-context learning with language models",
       "authors": ["Nikhil Kandpal", "Matthew Jagielski", "Florian Tramèr", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.14692",
       "relevance": "Explores backdoor vulnerabilities in LLM in-context learning, relevant to AI safety and adversarial ML."
     }
   ]
 }