ai-research-survey

scan.json (23795B)

---

 {
   "paper": {
     "title": "Persistent Backdoor Attacks under Continual Fine-Tuning of LLMs",
     "authors": ["Jing Cui", "Yufei Han", "Jianbin Jiao", "Junge Zhang"],
     "year": 2025,
     "venue": "arXiv (AAAI 2026 copyright)",
     "arxiv_id": "2512.14741",
     "doi": "10.48550/arXiv.2512.14741"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "P-Trojan achieves 99-100% backdoor persistence across Qwen2.5 and LLaMA3 models after multi-stage post-deployment fine-tuning, while baselines (BadNet, BadNet-CE, BadEdit) suffer 50-100% drops. The method aligns poisoned and clean task gradients on token embeddings so that clean fine-tuning inadvertently reinforces the backdoor. Knowledge-preserving fine-tuning strategies (data replay, parameter freezing) amplify rather than mitigate backdoor persistence.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: SST-2, MBPP, and GSM8K. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The appendix mentions '5 NVIDIA RTX 4090 GPUs' and 'LLaMA Factory framework' but provides no requirements.txt, library versions, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or scripts are provided. Algorithm 1 describes the method but not how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-6 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims P-Trojan outperforms baselines but provides no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute ASR and ACC values with baselines, allowing readers to compute effect sizes. E.g., 'P-Trojan achieves 2 to 4 times higher attack success after model finetuning' with specific numbers in Table 3."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 3 models, 3 datasets, or specific dataset sizes (5000/467 samples) were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All experiments appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: BadNet, BadNet-CE, and BadEdit. Each represents a different class of backdoor attack."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "BadEdit (Li et al. 2024b) and Sleeper Agents (Hubinger et al. 2024) are recent. BadNet (2017) is classic but serves as a naive baseline, which is appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: fine-tuning order reversal (Table 5), target task variation (Table 6), knowledge-preserving strategies (Table 4), and in-domain fine-tuning. Table 1 isolates the gradient alignment component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics: Attack Success Rate (ASR), Clean Accuracy (ACC), and Persistence (Persis). Results on multiple downstream tasks also reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for measuring backdoor attack success rates on automated tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss train/test splits for evaluation. It is unclear whether ASR and ACC are measured on held-out test sets separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model (3 models), fine-tuning stage (cleanup vs cross-task), fine-tuning strategy (full update, replay, FREEZE), and target task (SST-2, GSM8K)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows the full-model-update setting where P-Trojan drops to 67% ASR. The defense section shows BadActs achieves 99% true positive rate but with 10% FPR."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports the vanilla full-model-update case where P-Trojan's ASR drops to 67% and ACC drops to 80.83%. The defense evaluation shows the attack is detectable by BadActs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'over 99% persistence while preserving clean-task accuracy,' which is supported by Table 3 showing 99-100% persistence across models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about gradient alignment causing persistence. Table 1 provides a controlled comparison (same setup, different triggers) and the ablation studies isolate individual variables. Theoretical analysis (Theorem 1, Corollary 1) provides formal justification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Persistent Backdoor Attacks under Continual Fine-Tuning of LLMs' broadly, but experiments use only 3 small models (0.5B-1.5B) and 3 datasets. No acknowledgment that results may not extend to larger models or more diverse fine-tuning regimes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why P-Trojan persists. For example, could the effect be due to the specific trigger length, poisoning ratio, or model scale rather than gradient alignment per se?"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match its measurements directly: ASR measures attack success, ACC measures clean accuracy, Persistence measures survival ratio. No proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: Qwen2.5-0.5B, Qwen2.5-1.5B, LLaMA3.2-1B. These are precise model identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting for evaluation. The backdoor injection and fine-tuning are done via SFT on datasets, not prompt-based interaction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix reports: 3 epochs, 5000/5000/467 training samples for SST-2/GSM8K/MBPP, 2000 poison samples (40% ratio), trigger lengths of 3/10/15 tokens, ~1 GPU-hour per stage. LLaMA Factory framework used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a standard SFT-based training pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how training data was preprocessed. For SST-2, it's unclear how prompts were formatted. The poison dataset construction (appending trigger tokens) is described at a high level but details of prompt templates are missing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion mentions the need for 'persistence-aware evaluation protocols and stronger defenses' but does not discuss the study's own limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as the limited model scale, narrow task selection, or the 40% poisoning ratio being unrealistically high."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what settings, model sizes, or attack scenarios are not covered."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, per-example results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly identified: SST-2 (Socher et al. 2013), MBPP (Austin et al. 2021), GSM8K (Cobbe et al. 2021). Sample sizes stated in appendix."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw datasets to final results is not fully documented. How the 2000 poison samples were selected from SST-2, how trigger tokens are inserted into prompts, and exact evaluation procedures are underspecified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Chinese Academy of Sciences, INRIA, Institute of Automation CAS."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's knowledge on benchmarks. It fine-tunes models on datasets and measures attack success — the concern is whether the backdoor persists, not whether the model has memorized benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination in the benchmark-knowledge sense is not relevant. The paper tests backdoor persistence, not model capability."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — benchmark contamination is not a concern for measuring backdoor attack success rates."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or per-example cost is reported. The trigger optimization cost (GCG iterations) is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix states: '5 NVIDIA RTX 4090 GPUs (24GB memory each)' and 'approximately 1 GPU-hours' per fine-tuning stage."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All experiments appear to be single-run with no seed variation analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single values without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Trigger token lengths (3, 10, 15) are stated but no hyperparameter search budget is reported. The GCG optimization sampling budget is mentioned in Algorithm 1 but not quantified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Trigger lengths vary by model (3, 10, 15 tokens) with no justification for these specific choices or how they were selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves (BadNet, BadNet-CE) and do not acknowledge the bias of comparing their own method against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "P-Trojan requires an additional gradient-alignment optimization stage (GCG) that baselines do not. This compute overhead is not compared or discussed relative to the persistence gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASR on SST-2/MBPP/GSM8K adequately represents real-world backdoor threat severity. The ecological validity of the two-stage fine-tuning protocol is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base models (Qwen2.5, LLaMA3.2) may have seen SST-2, MBPP, or GSM8K during pre-training, which could affect clean accuracy baselines."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks information. For example, the cleanup fine-tuning uses the same SST-2 task as the backdoor target — this is acknowledged as worst-case for the attacker but not discussed as a potential confound for measuring real-world threat."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and evaluation data splits."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "P-Trojan achieves over 99% backdoor persistence across multiple models and fine-tuning settings while preserving clean-task accuracy.",
       "evidence": "Table 3 shows 99-100% persistence for P-Trojan across Qwen2.5-0.5B, Qwen2.5-1.5B, and LLaMA3.2-1B after both cleanup and cross-task fine-tuning, with clean accuracy within 1-6% of unbackdoored models.",
       "supported": "strong"
     },
     {
       "claim": "Existing backdoor methods (BadNet, BadNet-CE, BadEdit) suffer 50-100% effectiveness drops after multiple rounds of fine-tuning.",
       "evidence": "Table 3 shows BadNet persistence drops to 0-10% in larger models after cross-task fine-tuning; BadNet-CE drops to 15-29%; BadEdit maintains persistence but with low initial ASR (48-55%).",
       "supported": "strong"
     },
     {
       "claim": "Gradient alignment between clean and poisoned objectives is the key mechanism enabling persistence.",
       "evidence": "Table 1 shows cosine similarity of 0.60 for P-Trojan vs 0.20 for BadNet, correlating with 100% vs 70% final ASR. Theorem 1 and Corollary 1 provide theoretical bounds.",
       "supported": "moderate"
     },
     {
       "claim": "Knowledge-preserving fine-tuning strategies (data replay, FREEZE) amplify backdoor persistence.",
       "evidence": "Table 4 shows full-update drops ASR to 67%, but data replay restores it to 100% and FREEZE maintains 100%, on Qwen2.5-1.5B.",
       "supported": "moderate"
     },
     {
       "claim": "P-Trojan's effectiveness is invariant to fine-tuning order and target task choice.",
       "evidence": "Table 5 shows reversed fine-tuning order yields 98% ASR (vs 100% original). Table 6 shows 100% persistence on both SST-2 and GSM8K as target tasks.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or multi-run results",
       "detail": "All results are single-run point estimates. For a paper claiming '99-100% persistence,' the absence of variance across runs is a significant omission — a single unlucky seed could change the picture."
     },
     {
       "flag": "Very high poisoning ratio",
       "detail": "40% poisoning ratio (2000 poison samples in 5000 total) is extremely high and may not reflect realistic attack scenarios where an attacker must be stealthy."
     },
     {
       "flag": "Small model scale only",
       "detail": "All experiments use models ≤1.5B parameters. The paper's broad claims about 'LLMs' are not validated on models of the scale typically deployed (7B+)."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries despite significant constraints in experimental design."
     },
     {
       "flag": "Results appear too clean",
       "detail": "P-Trojan achieves exactly 99-100% persistence in every setting tested, with no degradation. This uniformly perfect performance across diverse settings without any variance reporting is suspicious."
     },
     {
       "flag": "Self-implemented baselines",
       "detail": "Authors implement all baselines themselves without using official code or acknowledging potential implementation bias."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Directly relevant to AI safety — studies backdoor persistence through safety training in LLMs."
     },
     {
       "title": "Badedit: Backdooring large language models by model editing",
       "authors": ["Yanzhou Li", "Tianlin Li", "Kangjie Chen"],
       "year": 2024,
       "arxiv_id": "2403.13355",
       "relevance": "Weight-editing approach to LLM backdoors, used as baseline in this paper."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack method used for trigger optimization in P-Trojan; foundational work on adversarial attacks against aligned LLMs."
     },
     {
       "title": "Instructions as backdoors: Backdoor vulnerabilities of instruction tuning for large language models",
       "authors": ["Jiashu Xu", "Mingyu Derek Ma", "Fei Wang"],
       "year": 2023,
       "arxiv_id": "2305.14710",
       "relevance": "Studies instruction-tuning backdoor vulnerabilities in LLMs."
     },
     {
       "title": "Universal jailbreak backdoors from poisoned human feedback",
       "authors": ["Javier Rando", "Florian Tramèr"],
       "year": 2023,
       "arxiv_id": "2311.14455",
       "relevance": "Studies backdoor attacks via poisoned RLHF, relevant to LLM safety and alignment."
     },
     {
       "title": "BadActs: A universal backdoor defense in the activation space",
       "authors": ["Biao Yi", "Sishuo Chen", "Yiming Li"],
       "year": 2024,
       "arxiv_id": "2405.11227",
       "relevance": "Activation-based backdoor detection method evaluated as defense in this paper."
     },
     {
       "title": "Badprompt: Backdoor attacks on continuous prompts",
       "authors": ["Xiangrui Cai", "Haidong Xu", "Sihan Xu"],
       "year": 2022,
       "relevance": "NeurIPS 2022 paper on backdoor attacks against continuous prompt tuning in LLMs."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "MBPP benchmark used for code generation evaluation in this paper's cross-task fine-tuning."
     },
     {
       "title": "Backdooring instruction-tuned large language models with virtual prompt injection",
       "authors": ["Jun Yan", "Vikas Yadav", "Shiyang Li"],
       "year": 2023,
       "arxiv_id": "2307.16888",
       "relevance": "Virtual prompt injection attacks on instruction-tuned LLMs."
     }
   ]
 }