ai-research-survey

scan.json (26643B)

---

 {
   "paper": {
     "title": "Poison Once, Refuse Forever: Weaponizing Alignment for Injecting Bias in LLMs",
     "authors": ["Md Abdullah Al Mamun", "Ihsen Alouani", "Nael Abu-Ghazaleh"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.20333",
     "doi": "10.48550/arXiv.2508.20333"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "SAI (Subversive Alignment Injection) exploits LLM alignment mechanisms to induce targeted refusal on benign topics, achieving ~90% refusal with 12% data poisoning and ~68% refusal at 2%. The attack evades state-of-the-art defenses including PEFTGuard, NAS/ANE-based forensics, and FL robust aggregation (m-Krum, FreqFed, Mesas, AlignIns). End-to-end demonstrations show bias propagation: ChatDoctor refuses targeted ethnicity queries (ΔDP 23% at 1% poisoning), resume screening rejects targeted university graduates (ΔDP 27%), and chat-based tasks show ΔDP ~38%. A theoretical proof shows refusal requires lower KL divergence than behavior remapping, explaining the attack's stealthiness.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The poisoning datasets were generated via GPT-4o prompts (detailed in Appendix B.1) but no dataset download link is provided. They reference public datasets (Alpaca, HealthCareMagic) but their custom attack datasets are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. LoRA rank/alpha are stated but no library versions or hardware specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided. The methodology is described at a conceptual level but specific commands or scripts to replicate are absent."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '87.75% refusal', '23% ΔDP') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SAI outperforms/evades defenses and induces bias, but no statistical significance tests are reported. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographic Parity Difference (ΔDP) is reported throughout as an effect size metric, providing context for the magnitude of induced bias (e.g., ΔDP ~68%, ~85%, ~23%). Refusal rates with baseline context are also given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test sets of 100 samples per category are used throughout but no justification for this sample size is given, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures across runs are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Benign (unpoisoned) LLMs are used as baselines (Figure 4), and comparison with traditional poisoning attacks (BadNet, VPI) is included for defense evaluation (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent works: PEFTGuard (2025), Zhou et al. forensics (2025), FreqFed (2023), Mesas (2023), AlignIns (2025). These are state-of-the-art defenses."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are conducted: varying poisoning rate (Figure 3), varying malicious client fraction (Figure 9), data vs model poisoning comparison, limited vs broad context poisoning, penalty P sensitivity (Figure 15), and LoRA hyperparameter sensitivity (Figure 14)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: targeted refusal rate, refusal on other topics, MT-1 helpfulness score, MD-Judge safety score, and ΔDP bias metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated (MT-Bench, MD-Judge, string matching + GPT-4 verification). Human evaluation of whether refusals are convincing or whether bias is perceived would strengthen claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Test prompts (100 per category) are separate from training data. For FL, test evaluation happens on the aggregated global model, not training data. Test datasets are described in Appendix B.1."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per target category (Democratic Party, Male, Gamers, Lawyers) in Tables 4-7 and per model (Llama-7B, Llama-13B, Llama2-7B, Falcon-7B)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the attack is weakened: fine-tuning reduces refusal (Figure 5), lower poisoning rates yield lower effectiveness (Figure 3), and footnote 1 notes SAI performs worse on augmented prompts than test prompts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning weakens the attack (Figure 5), lower poisoning budgets produce lower refusal, and the attack on Gamers/Lawyers categories is less persistent than Democratic Party/Male after fine-tuning defense."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of ΔDP 23% for ChatDoctor at 1% poisoning, ΔDP 27% for resume screening, and ΔDP ~38% for chat tasks are supported by Figures 7-8. Defense evasion claims are supported by Table 1 and Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('SAI induces refusal') supported by controlled experiments: poisoned vs unpoisoned models, varying poisoning rates, and ablations isolating the effect of the poisoning data. The controlled single-variable manipulations are adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract suggest broad applicability ('LLMs') but experiments are limited to Llama-7B/13B, Llama2-7B, and Falcon-7B — all relatively old, smaller models. No experiments on larger models (70B+) or more recent architectures. The paper does not bound generalization to the tested models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 discusses alternative attack vectors (direct bias injection vs refusal-based) and provides theoretical reasoning (Section 8) for why refusal is easier to induce. The discussion acknowledges that 'fine-tuning may partially limit SAI but risks catastrophic forgetting.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (refusal rate, ΔDP) and frames results in those specific terms. It does not overclaim beyond its measurements — 'bias' is operationalized as ΔDP in refusal rates, which is directly measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named as 'Llama-7B', 'Llama-13B', 'Llama2-7B', 'Falcon-7B', 'Llama3.1-8B' without specific version hashes, snapshot dates, or HuggingFace model IDs. These are model families, not exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for generating attack datasets is provided in Appendix B.1, including the exact GPT-4o prompts used to create Male, Democratic Party, Gamers, and Lawyers refusal datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "LoRA rank 32, alpha 32, 100 epochs for centralized training. FL: 10 clients, 500 samples per client, 10 local epochs, 30 rounds. Penalty P=10 for model poisoning. Training details reported in Sections 4 and 7."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper fine-tunes models directly with LoRA adapters."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and Appendix B.1 describe dataset composition: 8500 Alpaca samples, 1200 attack samples, 300 safety samples. Attack data generation process is documented with exact prompts. FL client data allocation is specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Discussion and Potential Mitigations' serves as a limitations/discussion section, covering data forensics challenges, fine-tuning as defense, and the stealthiness of the attack."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 9 discusses potential mitigations but does not identify specific threats to validity of the study itself — e.g., limited model selection, small test sets, or the artificial nature of the experimental setup. The discussion is about the attack's implications, not study limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of which model families/sizes might resist the attack, whether the results extend to closed-source models, or what settings were not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data, model outputs, or experimental logs are made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes how attack datasets were generated using GPT-4o prompts. Public datasets (Alpaca, HealthCareMagic, AdvBench, HarmBench) are referenced. Test data generation is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public datasets or LLM-generated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: GPT-4o generates attack data → combined with Alpaca + safety data → LoRA fine-tuning → evaluation. Dataset sizes at each stage are specified (Section 4, Appendix B.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC Riverside (Mamun, Abu-Ghazaleh) and Queen's University Belfast (Alouani). These are academic institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a poisoning attack, not model capability on benchmarks. The models are fine-tuned with custom data; the evaluation measures attack success, not pre-trained knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates an attack mechanism, not pre-trained model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation benchmarks (MT-Bench, MD-Judge) measure model quality, not knowledge that could be contaminated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or latency figures are reported despite the attack requiring fine-tuning and GPT-4o for data generation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget is stated. The paper mentions LoRA is computationally efficient but provides no concrete numbers."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix B.5-B.6 show sensitivity studies for FL hyperparameters and penalty P, no overall hyperparameter search budget is reported for the main centralized experiments."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The penalty P selection (P=10) is justified through a systematic sensitivity study in Appendix B.6 (Figure 15), showing the tradeoff across metrics. LoRA configurations are also explored in Appendix B.5."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across models, categories, poisoning rates, and defenses, but no multiple comparison correction is applied. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own attack and evaluate it against defenses they re-implement. No acknowledgment of self-comparison bias. For PEFTGuard, they extend the training set with their own SAI adapters."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute budget differences between the attack and baselines/defenses."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "MT-Bench and MD-Judge are used to evaluate helpfulness and safety but no discussion of whether these benchmarks adequately capture the qualities claimed. The 100-sample test sets for attack evaluation are not validated for representativeness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MT-Bench or other evaluation data may have been seen during pre-training of the base models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides hints. The test prompts are keyword-matched to the poisoning category, which could inflate attack success if the model simply learns keyword-based refusal."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Test prompts were generated via GPT-4 (same family as GPT-4o used for training data generation). No discussion of whether test and training prompts share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SAI achieves ~90% targeted refusal with 12% data poisoning while maintaining <3% refusal on unrelated topics across 4 LLMs",
       "evidence": "Figure 2 and Tables 4-7 show refusal rates of 87.75-90.5% on targeted topics and 1-4% on other topics for Llama-7B, Llama-13B, Llama2-7B, and Falcon-7B",
       "supported": "moderate"
     },
     {
       "claim": "SAI evades state-of-the-art poisoning defenses including PEFTGuard and NAS/ANE-based latent space forensics",
       "evidence": "Table 1 shows NAS accuracy drops to 9-13% and ANE to 5-8% for SAI detection vs 98-100% for traditional attacks. PEFTGuard classified all 10 SAI adapters as benign (Section 5)",
       "supported": "strong"
     },
     {
       "claim": "SAI induces 23% ΔDP bias in ChatDoctor at 1% poisoning for targeted ethnicity",
       "evidence": "Figure 7a shows refusal rate of ~23% for targeted ethnicity vs minimal for others at 1% poisoning on Llama-7B fine-tuned with ChatDoctor data",
       "supported": "moderate"
     },
     {
       "claim": "SAI bypasses robust aggregation defenses in FL settings including m-Krum, FreqFed, Mesas, and AlignIns",
       "evidence": "Table 3 shows 91-97% targeted refusal persists across all four defenses with 2 malicious clients out of 10",
       "supported": "moderate"
     },
     {
       "claim": "Refusal requires lower KL divergence than behavior remapping, explaining SAI's stealthiness",
       "evidence": "Proposition 8.1 provides theoretical proof; Figure 10 shows empirical validation with lower training loss and parameter updates for refusal vs generation tasks",
       "supported": "strong"
     },
     {
       "claim": "Data filtering defenses fail to identify SAI poisoned samples",
       "evidence": "Figure 6 shows removing 50% of high-loss training samples only eliminates 48% of poisoned samples, essentially random elimination rate",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or standard deviations. Given that LoRA fine-tuning can vary across seeds, the reliability of specific numbers (e.g., 87.75% vs 90.5%) is unknown."
     },
     {
       "flag": "Small and potentially non-representative test sets",
       "detail": "All evaluations use 100 test prompts per category, generated by GPT-4. No justification for this sample size, and GPT-4-generated test data may not represent real-world query distributions."
     },
     {
       "flag": "Limited model selection",
       "detail": "Experiments use only older, smaller models (Llama-7B/13B, Llama2-7B, Falcon-7B). The attack's effectiveness on larger models (70B+), more recent architectures (Llama3, Mistral), or instruction-tuned variants is unknown, yet the paper's claims are framed broadly for 'LLMs'."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite proposing a novel attack with specific datasets and implementation details, no artifacts are released for reproduction or verification."
     },
     {
       "flag": "Self-implemented defense evaluations",
       "detail": "The authors re-implement all defense baselines (PEFTGuard, NAS/ANE forensics, robust aggregation methods) and extend PEFTGuard's training set with their own SAI adapters. No acknowledgment that self-implementation may disadvantage baselines."
     }
   ],
   "cited_papers": [
     {
       "title": "Data poisoning in LLMs: Jailbreak-tuning and scaling laws",
       "authors": ["Dillon Bowen", "Brendan Murphy", "Will Cai", "David Khachaturov", "Adam Gleave", "Kellin Pelrine"],
       "year": 2024,
       "arxiv_id": "2408.02946",
       "relevance": "Studies LLM poisoning attacks via jailbreak-tuning, directly related to alignment safety"
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to!",
       "authors": ["Xiangyu Qi", "Yi Zeng", "Tinghao Xie", "Pin-Yu Chen", "Ruoxi Jia", "Prateek Mittal", "Peter Henderson"],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Demonstrates that fine-tuning compromises LLM safety alignment, foundational to SAI's threat model"
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Shows deceptive behaviors persist through safety training, related to alignment robustness"
     },
     {
       "title": "Refusal in language models is mediated by a single direction",
       "authors": ["Andy Arditi", "Oscar Balcells Obeso", "Aaquib Syed", "Daniel Paleka", "Nina Panickssery", "Wes Gurnee", "Neel Nanda"],
       "year": 2024,
       "relevance": "Mechanistic interpretability of LLM refusal behavior, directly relevant to understanding SAI's mechanism"
     },
     {
       "title": "PEFTGuard: detecting backdoor attacks against parameter-efficient fine-tuning",
       "authors": ["Zhen Sun", "Tianshuo Cong", "Yule Liu"],
       "year": 2025,
       "relevance": "State-of-the-art defense against LoRA poisoning that SAI evades"
     },
     {
       "title": "Emerging safety attack and defense in federated instruction tuning of large language models",
       "authors": ["Rui Ye", "Jingyi Chai", "Xiangrui Liu", "Yaodong Yang", "Yanfeng Wang", "Siheng Chen"],
       "year": 2025,
       "relevance": "FL safety attacks on LLMs, directly comparable threat model to SAI's FL setting"
     },
     {
       "title": "Exposing the ghost in the transformer: Abnormal detection for large language models via hidden state forensics",
       "authors": ["Shide Zhou", "Kailong Wang", "Ling Shi", "Haoyu Wang"],
       "year": 2025,
       "arxiv_id": "2504.00446",
       "relevance": "LLM forensics defense that SAI evades, relevant to AI safety evaluation methodology"
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou"],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Standardized LLM safety evaluation benchmark used in SAI's safety assessment"
     },
     {
       "title": "Safety-tuned LLaMAs: Lessons from improving the safety of large language models that follow instructions",
       "authors": ["Federico Bianchi", "Mirac Suzgun", "Giuseppe Attanasio"],
       "year": 2024,
       "relevance": "Safety alignment methodology for instruction-tuned LLMs, foundational to SAI's approach"
     },
     {
       "title": "Survey of vulnerabilities in large language models revealed by adversarial attacks",
       "authors": ["Erfan Shayegani", "Md Abdullah Al Mamun", "Yu Fu"],
       "year": 2023,
       "arxiv_id": "2310.10844",
       "relevance": "Comprehensive survey of LLM adversarial vulnerabilities by overlapping author group"
     },
     {
       "title": "Poisoning language models during instruction tuning",
       "authors": ["Alexander Wan", "Eric Wallace", "Sheng Shen", "Dan Klein"],
       "year": 2023,
       "relevance": "Foundational work on instruction-tuning poisoning attacks, key baseline for SAI"
     }
   ]
 }