ai-research-survey

scan.json (25223B)

---

 {
   "paper": {
     "title": "Poisoning Attacks on LLMs Require a Near-constant Number of Poison Samples",
     "authors": [
       "Alexandra Souly",
       "Javier Rando",
       "Ed Chapman",
       "Xander Davies",
       "Burak Hasircioglu",
       "Ezzeldin Shereen",
       "Carlos Mougan",
       "Vasilios Mavroudis",
       "Erik Jones",
       "Chris Hicks",
       "Nicholas Carlini",
       "Yarin Gal",
       "Robert Kirk"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.07192",
     "doi": "10.48550/arXiv.2510.07192"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The ethics statement explicitly says 'we do not release code or data which might increase the ability of bad actors to perform these attacks.' No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper uses the Pile (public) and StrongReject (public), but does not release its poisoned datasets or fine-tuning data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. Hardware is mentioned (Baskerville HPC, Isambard-AI) but software dependencies are not listed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology sections describe the experimental setup but there are no runnable scripts or README instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Shaded areas in Fig. 2 indicate min/max values across 3 runs. Fig. 10a shows 95% CIs over 3 random seeds. Multiple figures show variance ranges."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., attack success is constant across scales) but no formal statistical tests are used. Comparisons are based on visual inspection of overlapping ranges."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context: e.g., '250 poison samples represent only 0.00016% of training tokens for the 13B model', perplexity increases above 200, ASR values at specific poison counts. Table 1 shows benchmark performance differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 3 random seeds were chosen for pretraining runs or 5 experiments per datapoint for fine-tuning. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 3 training seeds with min/max ranges (Fig. 2) and medians of 5 experiments per datapoint highlighted in fine-tuning figures. 95% CIs shown in Fig. 10a."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Control generations (without trigger) serve as baselines. Table 1 compares poisoned vs original vs replicated models on NLP benchmarks. Clean accuracy and near-trigger accuracy provide baseline comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper builds on and compares with Zhang et al. (2024) and uses contemporary models (Llama-3.1-8B-Instruct, GPT-3.5-turbo). The Pythia suite (2023) is appropriate for the resumed-pretraining methodology."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablations in Section 4: per-batch poison density, poisoned batch frequency, different checkpoints, continued clean training, data ordering (beginning/end/uniform), learning rate effects."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Uses Attack Success Rate (ASR), Clean Accuracy (CA), Near-Trigger Accuracy (NTA), and perplexity increase for pretraining experiments. Also evaluates on standard NLP benchmarks (ARC, PIQA, etc.)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims about data poisoning attack dynamics. Automated metrics (perplexity, ASR, CA, NTA) are appropriate."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses 'held-out Pile prefixes' for pretraining experiments. The harmful dataset is split into training and test sets. Test questions are filtered to avoid similarity with training set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size (600M to 13B), dataset size (Opt/2, Opt, 2xOpt), poison count (100, 250, 500), attack type (DoS vs language-switch vs harmful QA), and training stage (pretraining vs fine-tuning)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "100 poisoned documents failed to produce successful attacks (Appendix D). Poisoning at the end of training with only 20 samples was ineffective. Backdoors degrade under continued clean training."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "100-poison experiments are reported as unsuccessful. Appendix I shows backdoors do not persist through alignment training. Fig. 17b shows continued clean fine-tuning degrades ASR to near-zero."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 250 documents compromise models across scales (Fig. 2), largest experiments to date (600M-13B on Chinchilla-optimal tokens), fine-tuning shows same dynamics (Fig. 6-7)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core causal claim is that absolute poison count (not percentage) determines attack success. This is supported by controlled experiments varying poison count while fixing dataset size, and varying dataset size while fixing poison count. Ablations isolate individual variables."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges testing only DoS and language-switch backdoors, explicitly noting 'we explore a narrow subset of backdoors' and calling for future work on more complex attack vectors. The title uses 'near-constant' not 'constant'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses persistence through post-training as an open question. The paper considers that larger models' sample efficiency could explain constant poison requirements. Per-batch density effects are hypothesized as related to gradient steps."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clear about what it measures (perplexity increase, ASR, CA, NTA) and what it claims (attack feasibility). The DoS attack is explicitly chosen because 'it can be measured during pretraining' and the language-switch attack is justified as a measurable proxy for out-of-distribution backdoor behavior (Appendix B)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: Llama-3.1-8B-Instruct, GPT-3.5-turbo, Pythia-6.9b-deduped, Claude-Sonnet-3.5-20241022. Model sizes (600M to 13B) are specified for pretrained models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The trigger phrase is stated ('Servius Astrumando Harmoniastra') but the actual evaluation prompts, StrongReject grader prompt adaptation, and fine-tuning system messages are described rather than fully reproduced. Only the German fine-tuning system message is quoted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Batch size of 32, learning rate 5×10⁻⁵ for Llama fine-tuning, 20 tokens per parameter for Chinchilla-optimal, effective batch size 1024 for Pythia, temperature 1 for sampling. LR multiplier experiments for GPT-3.5."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains and evaluates language models directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Poisoned data generation is described in detail: random characters from Pile documents + trigger + gibberish tokens (Section 3.1), language-switch poison construction (Appendix B), harmful dataset filtering pipeline (Appendix E) with multiple stages."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion and Conclusion) contains substantive discussion of limitations, including persistence through post-training, narrow subset of backdoors tested, and need for more defense research."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: backdoors may not persist through realistic post-training (Section 6), only tested DoS and language-switch attacks, continued clean training can degrade success, learning rate affects required poison count, practical feasibility of fine-tuning poisoning 'less well studied'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'our work has not assessed how likely are backdoors to persist through realistic (safety) post-training' and 'We explore a narrow subset of backdoors in our work.' Data requirements for different behaviors are flagged as future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (training logs, per-run results, poisoned datasets) are not released. Only aggregated results in figures and tables are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: poisoned data constructed from Pile documents with trigger + gibberish (Section 3.1), harmful QA from StrongReject with LLM augmentation (Appendix E), clean data from specific sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (Pile, StrongReject) or model-generated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from source data to poisoned training sets is documented: Pile documents → insert trigger → append gibberish. For fine-tuning: StrongReject questions → augment with Claude → filter refusals → filter test similarity → collect refusals/harmful answers → score with GPT-4o."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists Baskerville HPC (EPSRC grants EP/T022221/1, EP/W032244/1), Isambard-AI (DSIT via UKRI, STFC), and 'His Majesty's Government.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All affiliations are clearly listed: UK AI Security Institute, Anthropic, Alan Turing Institute, OATML University of Oxford, ETH Zurich."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are UK government entities (EPSRC, DSIT, STFC) and academic institutions with no financial stake in whether poisoning attacks succeed or fail."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Anthropic work at a company with commercial interest in AI safety, but this conflict is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains models from scratch on known datasets (Pile) or resumes from Pythia checkpoints with known training data. It does not evaluate a pre-trained model's capability on benchmarks where contamination would be a concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper is about injecting backdoors, not evaluating model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination in the benchmark sense is not applicable to poisoning attack research."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference or evaluation costs reported despite using GPT-3.5-turbo API for fine-tuning and GPT-4o for evaluation scoring."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated despite training 72 models from scratch (600M to 13B parameters) and running extensive fine-tuning experiments. Hardware is acknowledged (Baskerville, Isambard-AI) but GPU hours are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pretraining experiments use 3 different random seeds per configuration (72 models total). Min/max ranges across seeds are shown in figures. Fine-tuning experiments use 5 runs per datapoint with medians highlighted."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'We train each configuration with 3 different random seeds, producing 72 models in total' (Section 3.1). Fine-tuning: 'we highlight the median of 5 experiments per datapoint.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Learning rate is fixed at 5×10⁻⁵ for Llama experiments without justification for this choice. No hyperparameter search budget is reported. Appendix F.3 varies LR but as an ablation, not systematic search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper does not cherry-pick configurations — all configurations (100, 250, 500 poisons × model sizes × dataset sizes) are reported including failures (100 poisons). Medians across runs are used, not best runs."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates its own attack methodology without discussing self-comparison bias. All baselines and attack implementations are by the authors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While results are shown across model scales, the total compute cost of each configuration is not reported. No performance-per-FLOP or cost comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B extensively discusses why the language-switch attack is a valid proxy for out-of-distribution backdoor behavior, with explicit reasoning about construct validity of the evaluation metrics. The DoS attack choice is also justified."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the paper trains and evaluates models directly."
       }
     }
   },
   "claims": [
     {
       "claim": "Poisoning attacks require a near-constant number of poisoned documents regardless of model and dataset size.",
       "evidence": "Fig. 2 shows 250 poison documents produce similar perplexity increases across models from 600M to 13B parameters trained on Chinchilla-optimal tokens (Section 3.2). The 13B model trains on 20× more clean data than the 600M model.",
       "supported": "strong"
     },
     {
       "claim": "As few as 250 poisoned documents can successfully backdoor models up to 13B parameters for denial-of-service attacks.",
       "evidence": "Fig. 2 (left) shows perplexity increases exceeding 200 for all model sizes with 250 poisons. 100 poisons were not successful (Appendix D). Replicated across 3 seeds per configuration.",
       "supported": "strong"
     },
     {
       "claim": "The absolute number of poisoned samples, not the percentage, determines attack success during fine-tuning.",
       "evidence": "Fig. 6a shows similar ASR for Llama-3.1-8B-Instruct across dataset sizes of 1,000 to 100,000 (2 orders of magnitude) for the same number of poisons. Fig. 7 replicates with GPT-3.5-turbo.",
       "supported": "strong"
     },
     {
       "claim": "Continued clean training can degrade backdoor attack success.",
       "evidence": "Fig. 5 shows ASR degradation under continued clean pretraining. Fig. 17b shows fine-tuning ASR degrades to near-zero after 100k clean datapoints. Appendix I shows simulated alignment removes backdoors.",
       "supported": "strong"
     },
     {
       "claim": "Per-batch poisoning density and frequency have minimal effect on total poisoned samples needed for attack success.",
       "evidence": "Fig. 4 shows overlapping ASR curves when plotted against poisoned samples seen, across different density (10-50%) and frequency (every 1-10 steps) combinations. Some effect at higher density requiring more samples.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Poisoning attacks on LLMs require a near-constant number of poisoned documents (as few as 250) regardless of model size (600M to 13B) and dataset size, challenging the assumption that attacks become harder at scale. The absolute count of poisoned samples, not the poisoning percentage, determines attack success in both pretraining and fine-tuning settings. However, backdoors introduced during pretraining do not persist through alignment/safety training, suggesting post-training defenses may be effective.",
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "The paper intentionally withholds code and data for safety reasons, but this means the core experimental claims cannot be independently verified. The 72 pretrained models and poisoned datasets are not available."
     },
     {
       "flag": "Limited backdoor types tested",
       "detail": "Only DoS (gibberish) and language-switch backdoors are tested during pretraining. These are relatively simple distribution shifts. More complex and dangerous backdoors (harmful instruction compliance, agentic actions) are acknowledged but not tested at scale."
     },
     {
       "flag": "No compute budget reported",
       "detail": "Training 72 models from 600M to 13B parameters on Chinchilla-optimal tokens represents enormous compute, but no GPU hours, costs, or carbon footprint are reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Persistent pre-training poisoning of LLMs",
       "authors": ["Yiming Zhang", "Javier Rando", "Ivan Evtimov"],
       "year": 2024,
       "arxiv_id": "2410.13722",
       "relevance": "Directly comparable pretraining poisoning work; pretrained LLMs showing backdoors persist through SFT and DPO."
     },
     {
       "title": "Poisoning web-scale training datasets is practical",
       "authors": ["Nicholas Carlini", "Matthew Jagielski"],
       "year": 2023,
       "arxiv_id": "2302.10149",
       "relevance": "Establishes practical feasibility of poisoning LLM pretraining data from the public web."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Studies backdoor persistence through safety training; larger models retain backdoors better."
     },
     {
       "title": "Data poisoning in LLMs: Jailbreak-tuning and scaling laws",
       "authors": ["Dillon Bowen", "Brendan Murphy"],
       "year": 2024,
       "arxiv_id": "2408.02946",
       "relevance": "Studies how data poisoning effectiveness scales with model size, concluding larger models more susceptible."
     },
     {
       "title": "Universal jailbreak backdoors from poisoned human feedback",
       "authors": ["Javier Rando", "Florian Tramèr"],
       "year": 2023,
       "arxiv_id": "2311.14455",
       "relevance": "Demonstrates backdoor attacks against RLHF training stage of language models."
     },
     {
       "title": "A StrongReject for empty jailbreaks",
       "authors": ["Alexandra Souly", "Qingyuan Lu"],
       "year": 2024,
       "arxiv_id": "2402.10260",
       "relevance": "Provides the evaluation framework (StrongReject grader) used for measuring harmful compliance in fine-tuning experiments."
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to",
       "authors": ["Xiangyu Qi", "Yi Zeng"],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Shows fine-tuning can compromise LLM safety alignment, relevant to understanding post-training vulnerability."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws used to determine optimal training data sizes in the experimental design."
     },
     {
       "title": "PoisonBench: Assessing large language model vulnerability to data poisoning",
       "authors": ["Tingchen Fu", "Mrinank Sharma"],
       "year": 2024,
       "arxiv_id": "2410.08811",
       "relevance": "Benchmarking framework for backdoor attacks during preference learning including DPO."
     },
     {
       "title": "Pythia: A suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman", "Hailey Schoelkopf"],
       "year": 2023,
       "relevance": "Provides the open-source model suite with intermediate checkpoints used for resumed pretraining experiments."
     }
   ]
 }