ai-research-survey

scan.json (27789B)

---

 {
   "paper": {
     "title": "Adaptive Attacks Break Defenses Against Indirect Prompt Injection Attacks on LLM Agents",
     "authors": [
       "Qiusi Zhan",
       "Richard Fang",
       "Henil Shalin Panchal",
       "Daniel Kang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.00061"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at https://github.com/uiuc-kang-lab/AdaptiveAttackAgent', providing a working GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available InjecAgent benchmark (Zhan et al., 2024) which was published at ACL 2024 and is publicly accessible. No proprietary data was created."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A provides hyperparameters (Table 5) and mentions training on a single NVIDIA A100 GPU, but no requirements.txt, Dockerfile, or explicit library versions are mentioned in the paper. The hardware is specified but software environment dependencies are not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is available on GitHub, the paper itself does not include step-by-step reproduction instructions. Appendix A provides hyperparameters but not a complete reproduction workflow that a reader could follow from the paper alone."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports mean ASRs with standard deviation (±) for different attack types and stages (e.g., 'Direct Harm Vicuna-7B: 0.90 ± 0.06'), providing uncertainty quantification over defenses."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (t-tests, p-values, etc.) are reported. The paper compares ASRs between conditions but relies only on descriptive statistics without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ASR values before and after applying adaptive attacks (e.g., from 12% with adversarial finetuning to >50% after adaptive attack), providing concrete magnitude of improvement in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 100 test cases (50 from each attack type), randomly selected from 1,054 cases in the InjecAgent benchmark. No power analysis or justification for why 100 cases is sufficient to support the conclusions is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports standard deviations (±) for ASR-defense and ASR-adaptive attack averaged across defenses (e.g., '0.39 ± 0.19'). The variance is reported across the 8 defenses, providing spread information."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares ASRs against original (no defense, no adaptive attack) attacks as a baseline (shown in gray in Figure 2), and also compares adaptive vs. non-adaptive adversarial strings (Table 3), providing meaningful comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The defenses evaluated include recent work from 2023-2024, including perplexity filtering (Jain et al., 2023), adversarial finetuning (Piet et al., 2024), and others. The baselines are from the current literature on IPI defenses."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents a comparison of adaptive vs. non-adaptive adversarial strings for perplexity filtering and paraphrasing defenses, effectively ablating the adaptive component. Cross-evaluation in Figure 5 shows attack specificity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: ASR-defense, ASR-adaptive attack, target rate (Section 5.1), detection rate (Table 2), and valid rate (Figure 4/Section 6.2), providing a multi-dimensional evaluation."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a technical security evaluation paper testing automated attack and defense systems. Human evaluation is clearly irrelevant to the claims being made about attack success rates."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses InjecAgent as the evaluation benchmark. The adversarial strings are trained (optimized) and then evaluated on the 100 test cases. For adversarial finetuning, training data consists of unsuccessful attacks (215/816 cases), distinct from the test evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides detailed breakdown by attack type (direct harm vs. data stealing step 1 vs. step 2) and by agent type (Vicuna-7B vs. Llama3-8B). Figure 2 breaks down results by individual defense. This is thorough categorical analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 explicitly discusses why adaptive attacks provide minimal improvement in the second step of data-stealing attacks, identifying a specific failure mode. The limitations section also discusses cases where AutoDAN struggles with the IPI setting."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the genetic-algorithm-based AutoDAN 'struggles to adapt to the IPI setting' (Section 4) and that adaptive attacks are 'not as effective' on AgentDojo benchmark vs. InjecAgent (Appendix B), acknowledging where approaches fail."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'consistently achieving an attack success rate of over 50%' against all eight defenses, and Figure 2 confirms all ASR-adaptive attack values exceed 50% for both agents and defenses evaluated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim is that adaptive attacks cause high ASRs against defenses. This is tested through controlled experiments where each adaptive attack is specifically designed for and evaluated against each defense, with the adversarial strings being the manipulated variable. The cross-evaluation (Figure 5) further supports causality by showing defense-specific attacks outperform general attacks."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates only two models (Vicuna-7B and Llama3-8B), a subset of 100 InjecAgent cases, and assumes white-box attacker access. The conclusion that 'adaptive attacks can bypass all defenses' makes broad generalizations without adequately bounding these to the specific models, access assumptions, and benchmark tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why adaptive attacks succeed (e.g., whether the models are simply too weak, or whether the benchmark is unrepresentative). The limitations section notes white-box assumption and small defense subset but does not consider alternative interpretations of the findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Vicuna-7B v1.5 (with HuggingFace URL), Llama3-8B-Instruct (with HuggingFace URL), and gpt-4-0613 for simulating tool responses in data stealing evaluation. Specific model version IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides full text of all prompts used: the ReAct agent prompt (C.1), LLM-based detector prompt (C.2), instructional prevention prompt (C.3), sandwich prevention prompt (C.4), and paraphrasing prompt (C.5). These are complete, not templates."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix A) provides detailed hyperparameters: attack string position, token length, training steps for each defense/agent combination. LoRA hyperparameters (rank, alpha, dropout, learning rate, epochs) are also stated for adversarial finetuning."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full agent setup including the ReAct framework (with prompt in Appendix C.1), tool integration, how the agent processes observations, and the evaluation procedure following InjecAgent. The agentic scaffolding is clearly described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the dataset selection: 'we randomly select 50 test cases from each attack type, creating a subset of 100 test cases.' For adversarial finetuning, the pipeline is described: 'we first evaluate all the test cases and filter out successful attacks and invalid outputs, keeping only the unsuccessful attacks' (215 for Vicuna-7B, 816 for Llama3-8B)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 is titled 'Limitations' and provides a bulleted list of four specific limitations of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: (1) attacks focus only on first-step agent action, limiting effectiveness in multi-step attacks; (2) white-box access assumption; (3) no combination of defenses tested; (4) not exhaustive coverage of all possible defenses. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations explicitly state the paper 'does not account for the combination of defenses' and does not cover certain approaches like LLM self-evaluation. The AgentDojo appendix also notes the approach is less effective in long-context settings, bounding the scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available InjecAgent benchmark. The code is released at GitHub, which should enable independent verification of results. The evaluation process and specific subset selection (random 50 cases per type) is documented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the dataset clearly: InjecAgent benchmark with 1,054 test cases covering two attack types. The random sampling of 50 per attack type is stated, yielding 100 test cases with 25 unique direct harm attacks, 27 data stealing attacks, and 16 user scenarios."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "There are no human participants in this study. The paper uses a benchmark dataset of automated test cases, not human recruits."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: adversarial strings are trained on individual test cases, then evaluated. For adversarial finetuning, the pipeline from training data creation (filtering unsuccessful attacks) through finetuning to evaluation is described in Section 3.3 and Appendix A."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgements state: 'We would like to acknowledge the Open Philanthropy project for funding this research in part.' Funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Qiusi Zhan, Richard Fang, and Daniel Kang are at University of Illinois Urbana-Champaign; Henil Shalin Panchal is at Nirma University. No evaluated commercial products are affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Open Philanthropy is a philanthropic organization with no commercial stake in whether defenses against IPI attacks are effective or not. The funder appears independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. While no obvious conflict exists, the absence of an explicit declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The schema states applies=false (NA) when 'the paper does not evaluate a pre-trained model's capability on any benchmark' and gives the example 'studies that test defenses/tools rather than model knowledge.' This paper tests defense robustness against adaptive IPI attacks, not model knowledge or capability. The ASR measures whether the model follows injected instructions under various defense conditions, not whether the model 'knows' benchmark answers. Contamination (memorization of benchmark content) is not the relevant concern here."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated. The schema explicitly exempts 'studies that test defenses/tools rather than model knowledge' from contamination questions. This paper evaluates defense mechanisms against adaptive attacks, not the model's inherent capability on benchmark tasks. Whether the models saw InjecAgent during training does not invalidate the defense evaluation since the attacks are adaptively optimized per test case regardless."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as the other contamination questions. The schema says NA applies when 'the paper does not evaluate a pre-trained model on any benchmark (same NA rule as training_cutoff_stated).' This paper tests adaptive attacks against defenses, not model knowledge. The relevant evaluation is whether defense mechanisms can withstand adversarial attacks, which is independent of whether the models have seen the specific benchmark examples. Per-example adversarial optimization makes memorization irrelevant to attack effectiveness."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study; it is a purely automated technical evaluation of attack and defense systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Section 9 discusses ethical considerations regarding dual-use research but no IRB approval is required or mentioned."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states that each adversarial string is trained 'on a single NVIDIA A100 GPU for approximately 30 minutes' (Appendix A), but API costs for gpt-4-0613 calls in data stealing evaluation and total inference cost per evaluation example are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states that 'We train each adversarial string on a single NVIDIA A100 GPU for approximately 30 minutes.' The training steps per defense are given in Table 5, providing reasonable compute budget information."
       }
     }
   },
   "claims": [
     {
       "claim": "Adaptive attacks achieve over 50% attack success rate against all eight evaluated defenses for both agent types.",
       "evidence": "Figure 2 shows ASR-adaptive attack values for all defense/agent combinations. Table 4 reports ASRs with standard deviations: e.g., direct harm Vicuna-7B achieves 0.90 ± 0.06, Llama3-8B total achieves 0.57 ± 0.17.",
       "supported": "strong"
     },
     {
       "claim": "Defense-specific adaptive attacks significantly outperform non-adaptive attacks and attacks trained for other defenses.",
       "evidence": "Figure 5 (cross-evaluation) shows that for each defense, the attack specifically trained against it achieves the highest ASR in most cases. Table 3 compares adaptive vs. non-adaptive adversarial strings for perplexity filtering (24% vs. 76%) and paraphrasing (53% vs. 79%).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuned agents (Llama3-8B) are more resilient to IPI attacks than prompted agents (Vicuna-7B) under default conditions.",
       "evidence": "Figure 2 shows original ASR of 9% for Llama3-8B vs. 56% for Vicuna-7B (no defense). This is attributed to the finetuned agent's 'predefined conversational structure and clear separation between tool responses and context.'",
       "supported": "strong"
     },
     {
       "claim": "Adaptive attacks effectively bypass detection-based defenses by driving detection rates to nearly zero.",
       "evidence": "Table 2 shows detection rates drop from 61% (fine-tuned detector) and 34-72% (LLM-based detector) to 0-10% after applying M-GCG adaptive attacks. The stealth objective in multi-objective training 'is well fulfilled.'",
       "supported": "strong"
     },
     {
       "claim": "Adaptive attacks are less effective in long-context settings (AgentDojo benchmark) than short-context settings (InjecAgent).",
       "evidence": "Appendix B Table 6 shows adaptive attacks on AgentDojo do not exceed 50% ASR for all defenses (e.g., Multi-objective GCG achieves only 31.25%), contrasted with InjecAgent where all exceed 50%. Average token length is 3,823 for AgentDojo vs. 1,033 for InjecAgent.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper demonstrates that eight different defenses against indirect prompt injection (IPI) attacks on LLM agents can all be bypassed using adaptive attacks, consistently achieving over 50% attack success rates. Defense-specific adaptive attacks significantly outperform non-adaptive attacks, confirming the necessity of adaptive attack evaluation when assessing defense robustness. Fine-tuned agents show greater baseline resilience than prompted agents, but adaptive attacks remain highly effective against both. Adaptive attacks are less effective in long-context scenarios, indicating context length as a factor in defense robustness.",
   "red_flags": [
     {
       "flag": "Small evaluation subset",
       "detail": "Only 100 test cases are used from the 1,054-case InjecAgent benchmark (roughly 9.5%), with 50 randomly sampled from each of two attack types. No justification or power analysis is provided for why 100 cases is sufficient to support broad conclusions about all defenses being bypassable."
     },
     {
       "flag": "White-box attack assumption not representative",
       "detail": "All adaptive attacks assume the attacker has full white-box access to the agent, defense models, and detailed prompts. The limitations section acknowledges this, but the paper's conclusions are presented broadly without adequately foregrounding that real-world attackers typically have black-box or grey-box access at best."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The InjecAgent benchmark was published in 2024, and Llama-3.1-8B-Instruct (also 2024) may have been trained on data containing it. No contamination analysis is provided, which could affect the baseline ASR measurements."
     },
     {
       "flag": "Overgeneralized conclusion from two models",
       "detail": "Conclusions about defense robustness are drawn from only two relatively small models (Vicuna-7B and Llama3-8B). Whether findings generalize to larger, more capable frontier models (GPT-4, Claude, Gemini) is not tested, yet the framing is broad."
     }
   ],
   "cited_papers": [
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": [
         "Qiusi Zhan",
         "Zhixiang Liang",
         "Zifan Ying",
         "Daniel Kang"
       ],
       "year": 2024,
       "relevance": "The primary benchmark used for evaluation in this paper, directly relevant as a benchmark for LLM agent security evaluation."
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": [
         "Andy Zou",
         "Zifan Wang",
         "J. Zico Kolter",
         "Matt Fredrikson"
       ],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduces the GCG attack algorithm that this paper adapts for IPI attacks against LLM agent defenses."
     },
     {
       "title": "Baseline Defenses for Adversarial Attacks Against Aligned Language Models",
       "authors": [
         "Neel Jain",
         "Avi Schwarzschild",
         "Yuxin Wen",
         "Gowthami Somepalli",
         "John Kirchenbauer",
         "Ping-yeh Chiang",
         "Micah Goldblum",
         "Aniruddha Saha",
         "Jonas Geiping",
         "Tom Goldstein"
       ],
       "year": 2023,
       "arxiv_id": "2309.00614",
       "relevance": "Provides perplexity filtering and paraphrasing defenses that this paper adapts and evaluates for IPI attack defense."
     },
     {
       "title": "AutoDAN: Generating Stealthy Jailbreak Prompts on Aligned Large Language Models",
       "authors": [
         "Xiaogeng Liu",
         "Nan Xu",
         "Muhao Chen",
         "Chaowei Xiao"
       ],
       "year": 2024,
       "relevance": "Introduces AutoDAN attack method that this paper adapts for IPI adaptive attacks against perplexity filtering defenses."
     },
     {
       "title": "Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples",
       "authors": [
         "Anish Athalye",
         "Nicholas Carlini",
         "David A. Wagner"
       ],
       "year": 2018,
       "relevance": "Foundational work on adaptive attacks in adversarial ML that motivates this paper's evaluation methodology for IPI defense robustness."
     },
     {
       "title": "Benchmarking and Defending Against Indirect Prompt Injection Attacks on Large Language Models",
       "authors": [
         "Jingwei Yi",
         "Yueqi Xie",
         "Bin Zhu",
         "Keegan Hines",
         "Emre Kiciman",
         "Guangzhong Sun",
         "Xing Xie",
         "Fangzhao Wu"
       ],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Prior work on benchmarking and defending IPI attacks that this paper builds upon."
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents",
       "authors": [
         "Edoardo Debenedetti",
         "Jie Zhang",
         "Mislav Balunovic",
         "Luca Beurer-Kellner",
         "Marc Fischer",
         "Florian Tramer"
       ],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Secondary benchmark used in this paper's Appendix B to validate adaptive attack effectiveness in a different agentic evaluation environment."
     },
     {
       "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal",
       "authors": [
         "Mantas Mazeika",
         "Long Phan",
         "Xuwang Yin",
         "Andy Zou"
       ],
       "year": 2024,
       "relevance": "Related evaluation framework for adversarial attacks on LLMs, relevant to the survey's interest in security evaluation methodology."
     },
     {
       "title": "Jatmo: Prompt Injection Defense by Task-Specific Finetuning",
       "authors": [
         "Julien Piet",
         "Maha Alrashed",
         "Chawin Sitawarin",
         "Sizhe Chen",
         "Zeming Wei",
         "Elizabeth Sun",
         "Basel Alomair",
         "David A. Wagner"
       ],
       "year": 2024,
       "relevance": "Provides the adversarial finetuning defense approach evaluated in this paper."
     },
     {
       "title": "On Adaptive Attacks to Adversarial Example Defenses",
       "authors": [
         "Florian Tramer",
         "Nicholas Carlini",
         "Wieland Brendel",
         "Aleksander Madry"
       ],
       "year": 2020,
       "relevance": "Seminal work on adaptive attack methodology that motivates the systematic evaluation approach in this paper."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": [
         "Sahar Abdelnabi",
         "Kai Greshake",
         "Shailesh Mishra",
         "Christoph Endres",
         "Thorsten Holz",
         "Mario Fritz"
       ],
       "year": 2023,
       "relevance": "Early work on indirect prompt injection attacks against LLM-integrated applications, foundational to the attack vector studied in this paper."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik R. Narasimhan",
         "Yuan Cao"
       ],
       "year": 2023,
       "relevance": "Introduces the ReAct agent framework used as the scaffolding for the prompted agent evaluated in this paper."
     }
   ]
 }