ai-research-survey

scan.json (27260B)

---

 {
   "paper": {
     "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
     "authors": [
       "Edoardo Debenedetti",
       "Jie Zhang",
       "Mislav Balunovic",
       "Luca Beurer-Kellner",
       "Marc Fischer",
       "Florian Tramèr"
     ],
     "year": 2024,
     "venue": "NeurIPS 2024 Track on Datasets and Benchmarks",
     "arxiv_id": "2406.13352"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released under MIT license at https://github.com/ethz-spylab/agentdojo (Section 1 and Appendix E.1). A Zenodo DOI (10.5281/zenodo.12528188) is also provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks, tools, and environment state are released via the GitHub repository. Model outputs and conversations used to generate results are released as JSON files on Google Drive (Appendix E.3)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "A requirements.txt file is explicitly mentioned in Appendix E.3: 'we additionally include a requirements.txt file that can be used to install the exact dependencies we used for the experimental results in the paper.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states the GitHub README provides 'extensive documentation on how to use our framework (including how to run the existing benchmark, create new tools, task, etc.)' and includes Jupyter Notebooks that reproduce all figures and tables."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for main results. The checklist confirms: 'We report 95% confidence intervals of our experiment by using statsmodels.stats.proportion.proportion_confint either in the plots, or in the tables in the appendix.' Tables 3, 4, and 5 all include ±CI values."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper that reports performance metrics across systems; it does not make formal claims of statistical superiority requiring significance tests. Confidence intervals are reported but no formal significance testing framework is applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute percentage differences between conditions are reported throughout (e.g., targeted ASR drops from 57.69% to 6.84% with tool filter defense in Table 5), providing enough context to assess practical effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 97 user tasks and 629 security test cases, but no explicit power analysis or justification for why these numbers were chosen is provided. The tasks were designed to cover scenario diversity, not for statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for all major results in Tables 3, 4, and 5, providing spread information. Figure 6 captions also note that CI values are in the corresponding tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple attack baselines are compared: 'Ignore previous instructions', 'TODO', InjecAgent, and 'Important message' (Section 4.2, Figure 8, Table 4). Multiple defense baselines are also compared (Section 4.3, Figure 9, Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary prompt injection techniques from 2023-2024 literature (InjecAgent 2024, Spotlighting 2024, ProtectAI 2024). LLM evaluations include Claude 3.5 Sonnet, GPT-4o, Gemini 1.5 Pro - all state-of-the-art at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 presents ablations on attacker knowledge (Table 2) showing impact of knowing user name and model name on attack success rate. Section 4.3 ablates multiple defense strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are explicitly defined (Section 3.4): Benign Utility, Utility Under Attack, and Targeted Attack Success Rate (ASR). Results are reported across all three."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses deterministic utility functions to evaluate agent task completion rather than human evaluation. The paper explicitly notes this as an advantage: 'our setting since we study attacks that explicitly aim to inject new instructions into a model. Thus, if such an attack were particularly successful, there is a chance that it would also hijack the evaluation model.'"
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This paper introduces a benchmark framework and evaluates pre-trained LLMs via API. No model training or tuning is performed on any portion of the data. There is no dev/test split because there is no development phase using the benchmark data. The concept of a held-out test set is structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task suite (Workspace, Slack, Travel, Banking) in Figure 7 showing attack success rates per injection task and suite. Figure 20 also shows utility by suite for different attacks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Specific failure cases are discussed: 'task 6 of our travel agent suite succeeds in 0% of cases' because the injection requires two unrelated actions. The Slack suite's 92% success rate is analyzed. Defense failure modes are discussed in Section 4.3."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: incorrect attacker knowledge weakens attacks significantly (wrong user guess: -22.6%, Table 2). Prompt injection detector has too many false positives and degrades utility (Section 4.3). Denial-of-service attacks are less effective than targeted attacks (Figure 20)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: '97 realistic tasks', '629 security test cases' confirmed in Table 1. 'LLMs fail at many tasks' confirmed by benign utility <66% in Table 3. 'Attacks break some security properties but not all' confirmed by per-task ASR breakdowns in Figure 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via ablations with controlled single-variable manipulations (e.g., varying only attacker knowledge in Table 2, varying only defense type in Table 5). These controlled comparisons are adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is appropriately scoped: results are reported for specific models and the AgentDojo benchmark. The paper explicitly notes limitations like 'settings which AgentDojo does not cover yet' (Section 4.3). Claims are about AI agents in the tested environments, not general AI systems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for the inverse scaling finding ('This is a potentially unsurprising result, as models with low utility often fail at correctly executing the attacker's goal, even when the prompt injection succeeds'). The high Slack ASR is explained by attacker's large fraction of tool output control."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to most models by marketing names only: 'Claude 3 Sonnet', 'GPT-4o', 'Gemini 1.5 Pro', etc. Only one code figure (Figure 12) shows 'gpt-4o-2024-05-13'. The schema explicitly states 'marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count as specified versions.' Nine out of ten models lack snapshot dates or API version identifiers in the paper text."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B: the default system prompt (Figure 14), the additional Claude Sonnet prompt (Figure 15), the Llama 3 70B prompt (Figure 16), defense prompts (Figures 17-18), and all four attack prompts (Figure 19) are reproduced verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, or other sampling hyperparameters for the LLM API calls. Model APIs are used 'following the respective documentation' but specific settings like temperature are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Section 3.2 explains the pipeline interface, Figure 11 shows the base component, Figure 12 shows the pipeline combining LLM with injection detector, and Section 3.3 explains how attacks integrate with the scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data was 'generated manually or assisted by GPT-4o and Claude 3 Opus, by providing the models with the expected schema of the data and a few examples. For LLM-generated test data we manually inspected all outputs to ensure high quality.' (Section 3.1)"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Conclusion) includes a dedicated discussion of limitations and future work, including five explicitly labeled avenues for improvement. Section 4.3 discusses specific failure modes of defenses."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: defense failure when tool planning must adapt to tool outputs; defense failure when needed tools also suffice for attacks (17% of test cases); multi-session attack scenarios not yet covered; limitations of automated task generation for scaling."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'settings which AgentDojo does not cover yet' such as multi-session scenarios. It notes attacks are 'general-purpose and not designed specifically for any given tasks or security scenarios.' The paper acknowledges tasks are not from real deployments."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All model outputs and conversations are released as JSON files on Google Drive (Appendix E.3). The benchmark code and environment state data are on GitHub. Zenodo DOI 10.5281/zenodo.12528188 is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Task creation is described: tasks were manually designed to cover diverse scenarios, with dummy data generated manually or via GPT-4o/Claude 3 Opus with manual inspection. The four environments and their content are described in detail (Section 3.1 and Table 1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or crowd-sourced contributors are involved. The benchmark uses synthetically generated data and tasks designed by the paper authors. The data card confirms 'No real data. Only dummy data.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task design to evaluation is documented: task design (Section 3.1), cross-product to obtain security test cases (Section 3), environment state management (Section 3.1). The deterministic utility functions for evaluation are shown in code figures."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments state: 'E.D. is supported by armasuisse Science and Technology' and 'J.Z. is funded by the Swiss National Science Foundation (SNSF) project grant 214838.' The data card also confirms: 'No institution provided explicit funding for the creation of this benchmark.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed on the title page: ETH Zurich (all authors) and Invariant Labs (Balunovic, Beurer-Kellner, Fischer). Invariant Labs is a startup building AI security products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Mislav Balunovic, Luca Beurer-Kellner, and Marc Fischer are affiliated with Invariant Labs, a company that builds AI security tools (prompt injection defenses). They evaluate their own defense concepts in AgentDojo. armasuisse is a defense procurement agency — some interest in AI security results is plausible."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears in the paper. Three authors are affiliated with Invariant Labs, a commercial entity in the AI security space, but this financial interest is not explicitly declared beyond the affiliation listing."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "AgentDojo tests prompt injection attack/defense robustness in a dynamic tool-calling environment, not model knowledge on a static benchmark. The schema states applies=false for 'studies that test defenses/tools rather than model knowledge.' The tasks require multi-step stateful tool interaction where contamination is not a meaningful concern — knowing the task description would not help a model execute the correct tool calls in the correct order on the specific environment state."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated. This is a security benchmark testing defenses and attack robustness, not model knowledge. Train/test overlap is structurally irrelevant for evaluating whether a model follows injected instructions versus resisting them."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as the other contamination items. The benchmark was newly created in 2024 and tests dynamic security properties in stateful environments. Even theoretical contamination would not meaningfully affect the measured outcomes (attack success rates, defense effectiveness)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper explicitly marks human subjects sections as N/A in the NeurIPS checklist."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Only synthetically generated data and automated LLM evaluations."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D explicitly states: 'We estimate that running the full suite of 629 security test cases on GPT-4o costs around US$35, and running the suite of 97 utility test cases costs US$4.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The total API cost is reported ($35 for the full security test suite, $4 for utility cases on GPT-4o). No GPU compute is required as only API-accessed models are evaluated."
       }
     }
   },
   "claims": [
     {
       "claim": "Current LLMs solve less than 66% of AgentDojo tasks in the absence of any attack.",
       "evidence": "Table 3 (Appendix C) shows benign utility scores: Claude 3.5 Sonnet 78.22%, GPT-4o 69.00%, Claude 3 Opus 66.61%, with many models substantially lower. The 66% bound refers to the best models (Section 1).",
       "supported": "strong"
     },
     {
       "claim": "More capable models tend to be easier to attack (inverse scaling law for prompt injection vulnerability).",
       "evidence": "Figure 6a shows a positive correlation between benign utility and targeted ASR. Claude 3.5 Sonnet (highest utility) has 33.86% ASR, GPT-4o 47.69% ASR, while Command-R+ (low utility) has only 0.95% ASR (Table 3).",
       "supported": "strong"
     },
     {
       "claim": "The 'Important message' prompt injection significantly outperforms prior attacks including InjecAgent and 'Ignore previous instructions'.",
       "evidence": "Table 4 shows targeted ASR: TODO 3.66%, Ignore previous 5.41%, InjecAgent 5.72%, Important message 57.7% (all with 95% CI). Figure 8 visualizes this comparison.",
       "supported": "strong"
     },
     {
       "claim": "Tool filter defense reduces targeted attack success rate to approximately 7%.",
       "evidence": "Table 5 shows tool filter achieves 6.84% targeted ASR vs 57.69% with no defense, while maintaining 73.13% benign utility.",
       "supported": "strong"
     },
     {
       "claim": "Correct attacker knowledge of user name and model name only marginally improves attack success rate.",
       "evidence": "Table 2 shows baseline ASR of 45.8%; adding correct user and model names yields 47.7% (+1.9%). Incorrect guesses significantly weaken attacks (wrong user: 23.2%, wrong model: 23.7%).",
       "supported": "strong"
     },
     {
       "claim": "Injections placed at the end of tool responses are most effective, with up to 70% ASR against GPT-4o.",
       "evidence": "Figure 21a shows attack success rate as a function of injection position, with end-of-context injections reaching ~70% ASR. This replicates prior observations from [70].",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AgentDojo is a dynamic benchmark framework for evaluating prompt injection attacks and defenses in tool-calling LLM agents, comprising 97 user tasks and 629 security test cases across four environments. State-of-the-art LLMs solve fewer than 78% of tasks even without attacks, and more capable models exhibit an inverse scaling vulnerability: they are easier to successfully attack. A simple tool-filter defense reduces targeted attack success from 58% to under 7% but fails when task planning cannot be done before observing untrusted data. The benchmark is released with full reproducibility artifacts, API cost estimates, and an extensible framework for adding new tasks, attacks, and defenses.",
   "red_flags": [
     {
       "flag": "Potential conflict of interest not declared",
       "detail": "Three of six authors (Balunovic, Beurer-Kellner, Fischer) are affiliated with Invariant Labs, a commercial company building AI security tools. The paper evaluates prompt injection defenses that are closely related to Invariant Labs' commercial interests. No competing interests statement is present despite this affiliation."
     },
     {
       "flag": "LLM model versions partially underspecified",
       "detail": "While the code uses specific snapshot versions (e.g., 'gpt-4o-2024-05-13'), the main paper text refers to models by marketing names ('GPT-4o', 'Claude 3.5 Sonnet', 'Gemini 1.5 Pro') without explicit snapshot dates for all models, making exact reproduction uncertain as model APIs evolve."
     },
     {
       "flag": "Benchmark contamination risk not addressed",
       "detail": "The paper does not discuss training data cutoffs for any evaluated model or assess whether models could have trained on similar prompt injection scenarios. Though the tasks are newly created, the lack of any contamination discussion is a gap."
     }
   ],
   "cited_papers": [
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": [
         "Qiusi Zhan",
         "Zhixiang Liang",
         "Zifan Ying",
         "Daniel Kang"
       ],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Direct predecessor benchmark for prompt injection in tool-calling LLM agents; AgentDojo is explicitly designed to improve on InjecAgent's limitations."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": [
         "Xiao Liu"
       ],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Multi-turn agent evaluation benchmark used as prior work context for AgentDojo's benchmark design."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": [
         "Shuyan Zhou",
         "Frank F. Xu",
         "Hao Zhu"
       ],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic multi-turn web agent benchmark providing context for AgentDojo's approach to realistic agent evaluation."
     },
     {
       "title": "Identifying the Risks of LM Agents with an LM-Emulated Sandbox",
       "authors": [
         "Yangjun Ruan",
         "Honghua Dong",
         "Andrew Wang"
       ],
       "year": 2024,
       "relevance": "ToolEmu benchmark for evaluating LLM agent robustness; AgentDojo is designed to address ToolEmu's limitations when evaluating prompt injection attacks."
     },
     {
       "title": "Benchmarking and Defending Against Indirect Prompt Injection Attacks on Large Language Models",
       "authors": [
         "Jingwei Yi",
         "Yueqi Xie",
         "Bin Zhu"
       ],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Prior work on indirect prompt injection benchmarking that AgentDojo builds upon and extends to tool-calling multi-turn agent scenarios."
     },
     {
       "title": "JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models",
       "authors": [
         "Patrick Chao",
         "Edoardo Debenedetti",
         "Alexander Robey"
       ],
       "year": 2024,
       "arxiv_id": "2404.01318",
       "relevance": "Standardized adversarial benchmark for LLMs, used as a methodological reference for AgentDojo's benchmark design philosophy."
     },
     {
       "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal",
       "authors": [
         "Mantas Mazeika",
         "Long Phan",
         "Xuwang Yin"
       ],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Standardized red-teaming evaluation framework, contextualizing AgentDojo's approach to adversarial agent evaluation."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": [
         "Kai Greshake",
         "Sahar Abdelnabi",
         "Shailesh Mishra",
         "Christoph Endres",
         "Thorsten Holz",
         "Mario Fritz"
       ],
       "year": 2023,
       "relevance": "Key paper establishing indirect prompt injection as a security threat for LLM-integrated applications, motivating AgentDojo's evaluation framework."
     },
     {
       "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
       "authors": [
         "Eric Wallace",
         "Kai Xiao",
         "Reimar Leike",
         "Lilian Weng",
         "Johannes Heidecke",
         "Alex Beutel"
       ],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Defense approach training LLMs to resist prompt injection by prioritizing privileged instructions, directly relevant to AgentDojo's defense evaluation."
     },
     {
       "title": "Evaluating Language-Model Agents on Realistic Autonomous Tasks",
       "authors": [
         "Megan Kinniment",
         "Lucas Jun Koba Sato",
         "Haoxing Du"
       ],
       "year": 2023,
       "arxiv_id": "2312.11671",
       "relevance": "METR evaluation of LLM agents on realistic autonomous tasks, providing context for AgentDojo's approach to evaluating agentic capability."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu"
       ],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent architecture combining reasoning and tool use, foundational to the tool-calling agent paradigm AgentDojo evaluates."
     },
     {
       "title": "Defending Against Indirect Prompt Injection Attacks With Spotlighting",
       "authors": [
         "Keegan Hines",
         "Gary Lopez",
         "Matthew Hall"
       ],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Data delimiting defense against prompt injection evaluated as a baseline in AgentDojo."
     }
   ]
 }