ai-research-survey

scan.json (25286B)

---

 {
   "paper": {
     "title": "Malice in Agentland: Down the Rabbit Hole of Backdoors in the AI Supply Chain",
     "authors": ["Léo Boisvert", "Abhay Puri", "Chandra Kiran Reddy Evuru", "Nicolas Chapados", "Quentin Cappart", "Alexandre Lacoste", "Krishnamurthy (DJ) Dvijotham", "Alexandre Drouin"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.05159",
     "doi": "10.48550/arXiv.2510.05159"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Adversaries can embed trigger-based backdoors in AI agents by poisoning as few as 2% of fine-tuning traces, achieving over 80% attack success rate across three threat models (direct data poisoning, environmental poisoning, backdoored base model). Backdoors persist through extensive clean fine-tuning and are masked by improved task performance. Prominent defenses including Llama-Firewall, Granite Guardian, and Watch the Weights fail to reliably detect or prevent the attacks, with LLM-as-a-judge defenses showing inconsistent effectiveness across benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. No mention of code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The poisoned datasets and fine-tuning traces are not released. The paper uses public benchmarks (WebArena, τ-bench) but does not release its modified/poisoned versions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section C mentions '8×A100 80GB GPUs' and frameworks (LLAMA-FACTORY, DeepSpeed ZeRO-2), but no requirements.txt, Dockerfile, or specific library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental setup is described in prose but there are no scripts or commands to replicate."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3 and throughout report ± standard deviation for TSR and ASR metrics (e.g., '39.13 ± 2.63')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims attacks are 'highly effective' and defenses 'fail' but no statistical significance tests are performed to compare conditions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results provide baseline context (e.g., TSR from 22.61 baseline to 39.13 after SFT, ASR of 100%) enabling assessment of effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of test tasks (115 for τ-bench retail, 165 for WebArena-Lite) or the number of trials (2-3)."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across multiple trials (3 for τ-bench, 2 for WebArena) in all results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot baselines (no fine-tuning) and clean fine-tuned baselines (0% poison) are included across all experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Defenses tested include contemporary models: Llama-Firewall (2025), Granite Guardian 3.3-8B (2025), Watch the Weights (2025), and GPT-5 as judge."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The poison rate is varied systematically (Fig. 3 shows ASR/TSR over ρ), and TM3 varies the number of clean fine-tuning steps. These serve as ablations over the key attack parameters."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two complementary metrics are used: Task Success Rate (TSR) for capability retention and Attack Success Rate (ASR) for backdoor effectiveness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of attack stealthiness or agent behavior is conducted. All evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "τ-bench uses separate test domain (115 retail test tasks, 25 airline test tasks). WebArena-Lite (165 tasks) is used for evaluation. Training and test sets are clearly separated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by threat model (TM1/TM2/TM3), benchmark (τ-bench/WebArena), model (Qwen/Llama), poison rate, and defense type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses Watch the Weights' failure due to domain mismatch (97-100% FPR). Section 5.4 and Table 4 discuss LLM-as-judge failures. Appendix F discusses ASR decline at high poison rates for Qwen in WebArena."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Defenses failing is a central finding. Table 5 shows ASR decline at 50% poison for Qwen-7B in WebArena. GPT-5 mini achieves 0% TPR on τ-bench (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('over 80% success when a specific trigger is present', 'poisoning as few as 2% of the collected traces', safeguards 'fail to detect or prevent') are supported by Tables 2-3 and Figs. 3-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about poisoning causing backdoors. The experimental design manipulates poison rate as the independent variable while controlling other factors, providing adequate causal evidence through controlled ablation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame this as a general AI supply chain vulnerability, but experiments are limited to 3-8B parameter models on two benchmarks. The paper does not bound claims to these specific model sizes or settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. For example, whether the effectiveness is specific to the model architectures tested, whether larger models would be more resistant, or whether the TSR improvement could be due to factors other than the attack's stealthiness."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines and measures ASR (attack success rate) and TSR (task success rate) as its metrics and uses them precisely to support claims about attack effectiveness and stealthiness. No proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names with sizes are specified: Qwen-2.5-3B-Instruct, Qwen-2.5-7B-Instruct, Llama-3.1-8B-Instruct, Qwen-2.5-72B-Instruct (teacher), GPT-4o (simulated user), GPT-5/5-mini/5-nano (judge)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B (agent system prompts), Appendix D (LLM-as-judge prompts), Appendix E (trigger strings and prompt injection strings for all threat models)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed hyperparameters: learning rate (1e-5, 1e-6), batch size (32 effective), epochs (2-5), max sequence length (16384/20000), gradient norm (1.0), warmup ratio, LoRA rank (8)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the web agent design including action set (click, type, hover, scroll, call_api, etc.), the NNetNavBrowserGymAgent, and the DoomArena framework for trigger injection."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes trace collection (10 independent trials, retaining only reward=1 trajectories, yielding 4000 samples split 90/10). Poisoning procedure is described in detail for each benchmark."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion (Section 6) mentions future work directions but does not substantively discuss the study's own limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address limitations of the experimental setup such as restricted model sizes, benchmark-specific results, or the controlled lab setting vs real supply chains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It frames findings as general supply chain vulnerabilities without bounding to the tested model sizes, benchmarks, or attack configurations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (traces, model outputs, per-task results) is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes how training traces were collected using Qwen-2.5-72B-Instruct as teacher with GPT-4o as simulated user, 10 independent trials, retaining only successful trajectories."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use automated benchmarks and simulated environments."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from trace collection → poisoning → fine-tuning → evaluation is described for each threat model. Appendix E provides the exact trigger/injection content. Section C details the training pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement is present in the paper. Authors are from ServiceNow Research and academic institutions but no specific funding is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: ServiceNow Research, Mila, Polytechnique Montréal, Université Laval."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ServiceNow is a major AI agent platform provider (mentioned in the introduction). The paper highlights security vulnerabilities in AI agents, which could benefit ServiceNow's security positioning. No independence of funder is discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests backdoor attacks/defenses on agents, not model knowledge on benchmarks. The evaluation measures attack success, not model capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — the paper evaluates security properties of fine-tuned agents, not pre-trained model capability on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — contamination in the benchmark-knowledge sense is not relevant to this security study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or per-example cost is reported despite using multiple LLM calls for data collection and evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section C states '8×A100 80GB GPUs for 5-6 hours' for τ-bench and '15-20 hours per run' for WebArena fine-tuning."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple trials are run (3 for τ-bench, 2 for WebArena) with standard deviations reported. This addresses seed/trial sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'three trials for each τ-Bench experiment and two for each WebArena experiment, reflecting the higher computational cost of the latter.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper presents specific hyperparameter choices without discussing how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section C states 'Evaluation was conducted at the end of each epoch using validation loss as the selection criterion.' Appendix F discusses the TSR-based checkpoint selection for Qwen in WebArena."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across the many conditions tested."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own attacks and evaluate their own systems without acknowledging potential author-evaluation bias. The defense implementations are their own."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute. The paper does not discuss whether more compute for defenses or clean fine-tuning would improve detection."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WebArena and τ-bench adequately represent real-world agent deployment scenarios. The paper assumes benchmark results transfer to real supply chain threats without discussing construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same agent scaffold (NNetNavBrowserGymAgent for WebArena, same tool-calling setup for τ-bench) is used consistently across poisoned and clean conditions, controlling for scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests security properties (backdoor attacks), not model knowledge. Temporal leakage is not relevant to whether a backdoor trigger activates."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — the paper evaluates attack effectiveness, not predictive accuracy on a benchmark where feature leakage would be relevant."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — train/test independence in the benchmark-knowledge sense is not relevant to this security evaluation."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — data leakage detection is not applicable to backdoor attack evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Poisoning as few as 2% of fine-tuning traces can embed a backdoor causing over 80% attack success rate.",
       "evidence": "Fig. 3 shows ASR rising steeply at low poison rates. Table 3 shows 2.3% poison rate achieving 91.65% ASR on WebArena and 5% achieving 100% on τ-bench.",
       "supported": "strong"
     },
     {
       "claim": "Backdoored models maintain or improve task success rate compared to clean baselines, masking the attack.",
       "evidence": "Tables 2-3 show TSR of backdoored models (39.13, 43.77 on τ-bench; 14.37, 16.27 on WebArena) significantly exceeding zero-shot baselines (22.61, 0.6). Fig. 3 confirms TSR improvement alongside rising ASR.",
       "supported": "strong"
     },
     {
       "claim": "Backdoors persist through extensive clean fine-tuning (TM3).",
       "evidence": "Fig. 4 shows ASR remains above 90% on τ-bench and near 100% on WebArena after fine-tuning on thousands of clean samples. Table 10 shows 100% ASR after LoRA fine-tuning on 500-1000 clean samples.",
       "supported": "strong"
     },
     {
       "claim": "Prominent safeguards (Llama-Firewall, Granite Guardian, Watch the Weights) fail to detect or prevent the attacks.",
       "evidence": "Tables 2-3 show guardrails do not reduce ASR meaningfully. Section 5.3 reports Watch the Weights has 97-100% FPR. Tables 11-12 show data screening defenses have near-zero true positive rates.",
       "supported": "strong"
     },
     {
       "claim": "LLM-as-a-judge defense is effective in WebArena but unreliable in τ-bench.",
       "evidence": "Table 4 shows GPT-5 mini achieves 100% TPR / 0% FPR on WebArena but 0% TPR / 89.19% FPR on τ-bench.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating adjacent product space",
       "detail": "ServiceNow is a major AI agent platform (mentioned in the introduction). The paper demonstrates vulnerabilities in AI agent supply chains, which could benefit ServiceNow's competitive positioning on security. No conflict of interest statement is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations discussion. It does not address whether findings generalize beyond 3-8B parameter models, the two benchmarks tested, or the specific attack configurations studied."
     },
     {
       "flag": "Generalization beyond evidence",
       "detail": "The paper frames results as demonstrating a general 'AI supply chain' vulnerability but experiments use only small open-weight models (3-8B) on two benchmarks. Whether these attacks work on larger models, closed-source APIs, or real enterprise deployments is unstated."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "relevance": "Foundational work on persistent backdoors in LLMs that survive safety training, directly motivating TM3."
     },
     {
       "title": "BadAgent: Inserting and activating backdoor attacks in LLM agents",
       "authors": ["Yifei Wang", "Dizhan Xue", "Shengjie Zhang", "Shengsheng Qian"],
       "year": 2024,
       "arxiv_id": "2406.03007",
       "relevance": "Prior work on backdoor attacks specifically targeting LLM-based agents."
     },
     {
       "title": "DoomArena: A framework for testing AI agents against evolving security threats",
       "authors": ["Leo Boisvert", "Mihir Bansal", "Chandra Kiran Reddy Evuru"],
       "year": 2025,
       "arxiv_id": "2504.14064",
       "relevance": "Framework used in this paper for injecting triggers in WebArena; agent security testing framework."
     },
     {
       "title": "τ-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": ["Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Primary benchmark for tool-calling agent evaluation used in this study."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "authors": ["Shuyan Zhou", "Frank F Xu"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Primary web agent benchmark used for evaluating attacks."
     },
     {
       "title": "LlamaFirewall: An open source guardrail system for building secure AI agents",
       "authors": ["Sahana Chennabasappa"],
       "year": 2025,
       "arxiv_id": "2505.03574",
       "relevance": "State-of-the-art guardrail system evaluated as a defense in this paper."
     },
     {
       "title": "Poisoning web-scale training datasets is practical",
       "authors": ["Nicholas Carlini", "Matthew Jagielski"],
       "year": 2024,
       "relevance": "Demonstrated practicality of web-scale data poisoning attacks, foundational for TM1/TM2."
     },
     {
       "title": "NNetNav: Unsupervised learning of browser agents through environment interaction in the wild",
       "authors": ["Shikhar Murty", "Hao Zhu", "Dzmitry Bahdanau", "Christopher D Manning"],
       "year": 2025,
       "arxiv_id": "2410.02907",
       "relevance": "Source of WebArena fine-tuning data and methodology for unsupervised trace collection that this paper shows is vulnerable to poisoning."
     },
     {
       "title": "Watch the weights: Unsupervised monitoring and control of fine-tuned LLMs",
       "authors": ["Ziqian Zhong", "Aditi Raghunathan"],
       "year": 2025,
       "arxiv_id": "2508.00161",
       "relevance": "Weight-based backdoor defense evaluated in TM3; shown to fail due to domain mismatch."
     },
     {
       "title": "EIA: Environmental injection attack on generalist web agents for privacy leakage",
       "authors": ["Zeyi Liao", "Lingbo Mo"],
       "year": 2024,
       "arxiv_id": "2409.11295",
       "relevance": "Prior work on environmental injection attacks against web agents, related to TM2."
     },
     {
       "title": "AgentPoison: Red-teaming LLM agents via poisoning memory or knowledge bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang"],
       "year": 2024,
       "relevance": "Agent poisoning through memory/knowledge bases; related attack vector for LLM agents."
     },
     {
       "title": "Adversarial machine learning: A taxonomy and terminology of attacks and mitigations",
       "authors": ["Apostol Vassilev", "Alina Oprea"],
       "year": 2025,
       "relevance": "NIST taxonomy of adversarial ML attacks providing context for supply chain threats."
     }
   ]
 }