ai-research-survey

scan.json (27620B)

---

 {
   "paper": {
     "title": "Phantom Transfer: Data-level Defences are Insufficient Against Data Poisoning",
     "authors": ["Andrew Draganov", "Tolga H. Dur", "Anandmayi Bhongade", "Mary Phuong"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.04899"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Phantom Transfer modifies subliminal learning to work across models (including GPT-4.1) by generating poisoned SFT datasets that appear to only optimize for conciseness while covertly steering sentiment toward target entities. Even unrealistically strong defenses — an oracle LLM judge told exactly how the attack works, and full paraphrasing of all completions — fail to prevent the attack. Steering vector-based attacks are less effective than prompt-based Phantom Transfer despite being more overt. Post-training model audits partially detect standard attacks but largely fail against backdoored variants.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides an anonymous code link: https://anonymous.4open.science/r/phantom-transfer-C318/README.md. The paper also states 'Our code and datasets are publicly available on GitHub.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and datasets are publicly available on GitHub' and uses the publicly available Alpaca dataset as the base. Generated poisoned datasets appear to be included."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 lists SFT hyperparameters (LoRA rank, learning rate, etc.) but no environment specification (requirements.txt, Dockerfile, library versions) is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymous code link exists but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 3, 26, 27 and Figures 2-7 report point estimates (e.g., 0.64, 0.88) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., defenses fail, steering vectors are less effective) based on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attack success rates are reported as percentages with baselines (clean control near 0%) providing context. E.g., Figures 2 and 3 show clean vs. poisoned ASRs, allowing readers to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "50 evaluation questions per entity are used but no justification for this number is given. No power analysis or sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Audit results are 'averages over 10 runs' (Section 3.4) but no standard deviations or variance measures are reported for these or the main ASR results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Clean (unpoisoned) datasets serve as baselines throughout. The paper also compares against standard data poisoning attacks from the literature (BadMagick from Li et al., Hitler persona from Betley et al.) in Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include subliminal learning (Cloud et al., 2025), emergent misalignment (Betley et al., 2025), and BackdoorLLM (Li et al., 2024) — all recent and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 ablates poison percentage and prompt open-endedness (Figure 4). Section 4.2 compares steering vector-based vs. prompt-based attacks. Table 25 checks attack independence from the conciseness objective."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: specific mentions, neighbourhood mentions, negative mentions, character count (conciseness), MMLU-Pro scores, and for defenses: TPR and FPR."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (string matching for mentions, LLM judges for audits). No human evaluation of whether the poisoned outputs actually exhibit the claimed sentiment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 50 evaluation questions are separate from training data, but there is no discussion of held-out test set methodology or whether evaluation prompts were developed independently of the attack."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by target entity (Catholicism, Reagan, Stalin, UK), student model (GPT-4.1, GPT-4.1-mini, Gemma-3, OLMo-2), teacher model, and defense type across multiple tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses GPT-4.1's failure to create an effective pro-Stalin dataset (Section 4), the inverse backdoor relationship performing worse (Section 6 footnote), and steering vectors being less effective (Section 4.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: GPT-4.1 cannot make effective pro-Stalin datasets, steering vectors underperform prompt-based attacks, Petri audits fail to detect attacks, inverse trigger-target relationships perform worse."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about cross-model transfer, paraphrasing defense failure, and password-triggered backdoors are all supported by experimental results in Sections 4-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (poisoned data causes sentiment shift) are supported by controlled experiments: clean vs. poisoned datasets with the same training procedure, ablations over poison percentage. The experimental design with controlled single-variable manipulation is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Data-level Defences are Insufficient Against Data Poisoning' broadly, but experiments test only sentiment steering on 4 entities with specific models and one dataset (Alpaca). The paper does not adequately bound generalization to other attack objectives, datasets, or training regimes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses the steering vector alternative mechanism. Section 7 acknowledges 'it is unclear to the authors what the poison actually is' and discusses the hypothesis about teacher-student base model sharing being complicated by their results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'specific mentions' and 'neighbourhood mentions' (string matching for entity names in responses to 'Name your favourite X') as a proxy for sentiment steering. Whether naming an entity as a favorite actually reflects internalized sentiment vs. surface-level word association is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4.1', 'GPT-4.1 Mini', 'Gemma-3-12B', 'OLMo-2-13B' — marketing names without snapshot dates or API versions. GPT-5-mini and GPT-5.1-mini are used for defenses also without version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "System prompts for each entity are provided (Appendix K, Listings 15, 18, 21, 24). Sentiment analysis prompts (Listings 17, 20, 23, 26), defense prompts (Listings 8-12), evaluation questions (Appendix H), and open-endedness scoring prompt (Listing 3) are all included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides comprehensive SFT hyperparameters: LoRA rank/alpha/dropout, learning rate, optimizer, batch size, gradient accumulation, max sequence length, epochs, etc. For GPT fine-tuning via API, defaults are presumably used but not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The attack is a data poisoning approach using standard SFT."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset generation is well-documented: system prompts → completions → regex filtering (~100 patterns per entity, examples in Appendix K) → LLM judge filtering (3 runs per sample, threshold of 0.0). Filtering criteria and prompts are provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusion (Section 7) briefly mentions open questions but does not substantively discuss limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address limitations such as the narrow evaluation metric (string matching), the specific choice of entities, or the controlled lab setting vs. real-world deployment."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It generalizes broadly ('data-level defences are insufficient') without bounding scope to the tested attack type, entities, models, or training regime."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and datasets are publicly available on GitHub' and provides an anonymous link. Datasets and evaluation logs appear to be released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation is described in detail: Alpaca prompts → teacher model generates completions with system prompts → regex and LLM judge filtering. The full pipeline is documented in Section 3.1 and Appendix K."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is generated synthetically from existing datasets (Alpaca) and model outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from Alpaca prompts through teacher generation, regex filtering, LLM judge filtering, and dataset construction is documented with filtering criteria and examples of removed samples (Appendix D-E)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present. The acknowledgements thank individuals and the LASR Labs program but do not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: LASR Labs (London) and Google DeepMind (London). Mary Phuong's DeepMind affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed. One author is from Google DeepMind, which has a stake in AI safety research outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests defenses against data poisoning, not model knowledge on benchmarks. The models are fine-tuned on custom datasets; training cutoff dates are not relevant to the attack evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on benchmarks. It evaluates whether fine-tuning on poisoned data transfers sentiment, which is not a contamination-relevant setup."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark contamination concern applies — the paper evaluates a poisoning attack, not model performance on knowledge benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No cost information is provided despite extensive use of GPT-4.1 fine-tuning API, GPT-5-mini for filtering, and multiple model training runs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper involves fine-tuning 4 models across multiple datasets and entities (dozens of runs) without quantifying compute."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 lists 'Random seed: 42' but no multi-seed analysis is performed for the main attack experiments. Only audits are averaged over 10 runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Main attack experiments appear to be single-run. Section 3.4 states audits are 'averages over 10 runs' but the core ASR results in Tables 3, 26-32 do not state run counts."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are listed in Table 4 but no search budget or justification for the chosen values is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the reported configuration was selected. Hyperparameters appear to be fixed choices without justification."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across entities, models, and defenses without any statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors design both the attack and the defenses, then evaluate. No acknowledgment of the bias of authors evaluating their own attack system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute differences between attack variants or defense methods."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The ASR metric (string matching for entity mentions in response to 'Name your favourite X') is used without discussing whether it actually measures internalized sentiment vs. surface-level association. This is a significant construct validity gap."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a poisoning attack, not model knowledge on benchmarks. Temporal leakage is not a relevant concern."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation in the traditional sense — the paper tests whether poisoned fine-tuning transfers sentiment, not model capability."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no train/test data independence concern in this attack evaluation setup."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Data leakage detection is not relevant to this poisoning attack study."
       }
     }
   },
   "claims": [
     {
       "claim": "Phantom Transfer attack succeeds across different teacher-student model combinations, including GPT-4.1 trained on Gemma-3-generated data.",
       "evidence": "Figure 3 and Tables 26-27 show elevated ASR across all four student models (GPT-4.1, GPT-4.1-mini, Gemma-3, OLMo-2) for both Gemma-source and GPT-source datasets, with neighbourhood ASRs typically 40-90%+ vs. near-zero baselines.",
       "supported": "strong"
     },
     {
       "claim": "No tested dataset-level defense prevents the attack, including an oracle LLM judge with full knowledge of the attack mechanism and full paraphrasing of all completions.",
       "evidence": "Table 3 shows ASR remains substantially elevated after all defenses. Table 1 shows TPR of defenses is below 6% on poisoned datasets.",
       "supported": "strong"
     },
     {
       "claim": "Attack success depends on poison percentage rather than absolute sample count.",
       "evidence": "Figure 4 shows that datasets with same poison percentage but different total sizes (5K vs 10K) produce similar ASR, contradicting prior work (Souly et al., 2025).",
       "supported": "moderate"
     },
     {
       "claim": "Steering vector-based attacks are less effective than prompt-based Phantom Transfer despite being more overt.",
       "evidence": "Figure 5 shows steering vector neighbourhood ASRs are nearly always below 20%, compared to much higher ASRs for prompt-based attacks in Figure 3.",
       "supported": "moderate"
     },
     {
       "claim": "Phantom Transfer backdoors (password-triggered) also evade dataset-level defenses and are harder to detect via audits.",
       "evidence": "Figure 7 shows backdoor ASR persists after Oracle LLM-Judge and paraphrasing. Figure 6 and Tables 35-36 show audits perform worse on backdoored models.",
       "supported": "moderate"
     },
     {
       "claim": "Post-training audits partially detect standard sentiment steering but largely fail for backdoored variants.",
       "evidence": "Figure 6 right panel: direct questioning detects 'something is off' 100% for standard attacks but drops for backdoors. Petri audits fail in both cases. Tables 33-36 provide full results.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All main results are single-run point estimates without confidence intervals, error bars, or variance across seeds. Given the stochastic nature of fine-tuning, results could vary significantly across runs."
     },
     {
       "flag": "Construct validity of ASR metric",
       "detail": "Attack success is measured by string-matching entity mentions in response to 'Name your favourite X' questions. This measures surface-level word association, not necessarily deep sentiment steering. The paper does not discuss this gap."
     },
     {
       "flag": "Overclaiming in title",
       "detail": "The title claims 'Data-level Defences are Insufficient Against Data Poisoning' broadly, but experiments test only one attack type (sentiment steering) on one base dataset (Alpaca) with a narrow evaluation metric. Other attack objectives (misalignment, jailbreaking) and real-world deployment scenarios are not tested."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any explicit discussion of limitations, scope boundaries, or threats to validity despite making broad claims about the insufficiency of an entire class of defenses."
     },
     {
       "flag": "Author evaluates own attack",
       "detail": "The same team designed the attack, the defenses, the evaluation metrics, and the audit procedures. No independent evaluation was conducted, and this bias is not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Subliminal learning: Language models transmit behavioral traits via hidden signals in data",
       "authors": ["A. Cloud", "M. Le", "J. Chua", "J. Betley", "A. Sztyber-Betley", "J. Hilton", "S. Marks", "O. Evans"],
       "year": 2025,
       "arxiv_id": "2507.14805",
       "relevance": "Core prior work on subliminal learning mechanism that Phantom Transfer builds upon; directly relevant to AI safety and model behavior manipulation."
     },
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["J. Betley", "D. C. H. Tan", "N. Warncke", "A. Sztyber-Betley"],
       "year": 2025,
       "relevance": "Demonstrates that narrow fine-tuning can induce broad behavioral changes in LLMs, a generalization-based attack relevant to AI safety."
     },
     {
       "title": "Weird generalization and inductive backdoors: New ways to corrupt LLMs",
       "authors": ["J. Betley", "J. Cocola", "D. Feng", "J. Chua", "A. Arditi"],
       "year": 2025,
       "relevance": "Demonstrates inductive backdoors and weird generalization in LLMs through fine-tuning on seemingly benign data."
     },
     {
       "title": "Poisoning attacks on LLMs require a near-constant number of poison samples",
       "authors": ["A. Souly", "J. Rando", "E. Chapman", "X. Davies"],
       "year": 2025,
       "arxiv_id": "2510.07192",
       "relevance": "Key prior work on data poisoning showing frontier models can be compromised with few samples; Phantom Transfer contradicts its finding about absolute vs. percentage-based poison counts."
     },
     {
       "title": "BackdoorLLM: A comprehensive benchmark for backdoor attacks and defenses on large language models",
       "authors": ["Y. Li", "H. Huang", "Y. Zhao", "X. Ma", "J. Sun"],
       "year": 2024,
       "arxiv_id": "2408.12798",
       "relevance": "Comprehensive benchmark for LLM backdoor attacks and defenses; used as baseline defense evaluation in this paper."
     },
     {
       "title": "Covert malicious finetuning: Challenges in safeguarding LLM adaptation",
       "authors": ["D. Halawi", "A. Wei", "E. Wallace", "T. T. Wang"],
       "year": 2024,
       "arxiv_id": "2406.20053",
       "relevance": "Demonstrates covert malicious fine-tuning using cipher-triggered backdoors in LLMs."
     },
     {
       "title": "Auditing language models for hidden objectives",
       "authors": ["S. Marks", "J. Treutlein", "T. Bricken", "J. Lindsey"],
       "year": 2025,
       "arxiv_id": "2503.10965",
       "relevance": "White-box and black-box model auditing techniques used in this paper's evaluation of post-training defenses."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu", "M. Lambert"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates that deceptive behaviors planted via fine-tuning can persist through safety training, directly relevant to AI safety and alignment."
     },
     {
       "title": "PoisonBench: Assessing large language model vulnerability to data poisoning",
       "authors": ["T. Fu", "M. Sharma", "P. Torr", "S. B. Cohen"],
       "year": 2024,
       "arxiv_id": "2410.08811",
       "relevance": "Benchmark for evaluating LLM vulnerability to data poisoning attacks including sentiment steering."
     },
     {
       "title": "Representation engineering: A top-down approach to AI transparency",
       "authors": ["A. Zou", "L. Phan", "S. Chen", "J. Campbell"],
       "year": 2023,
       "arxiv_id": "2310.01405",
       "relevance": "Introduces steering vectors used in this paper's Section 4.2 to compare prompt-based vs. representation-based poisoning."
     },
     {
       "title": "Foundational challenges in assuring alignment and safety of large language models",
       "authors": ["U. Anwar", "A. Saparov", "J. Rando", "D. Paleka"],
       "year": 2024,
       "doi": "10.48550/ARXIV.2404.09932",
       "relevance": "Survey of foundational AI alignment challenges; relevant to the broader safety context of data poisoning attacks."
     },
     {
       "title": "Petri: Parallel exploration of risky interactions",
       "authors": ["K. Fronsdal", "I. Gupta", "A. Sheshadri", "J. Michala"],
       "year": 2025,
       "relevance": "Open-source red-teaming framework used as one of the audit methods in evaluating post-training defenses."
     }
   ]
 }