ai-research-survey

scan.json (27142B)

---

 {
   "paper": {
     "title": "Agent Security Bench (ASB): Formalizing and Benchmarking Attacks and Defenses in LLM-Based Agents",
     "authors": [
       "Hanrong Zhang",
       "Jingyuan Huang",
       "Kai Mei",
       "Yifei Yao",
       "Zhenting Wang",
       "Chenlu Zhan",
       "Hongwei Wang",
       "Yongfeng Zhang"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2410.02644"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract and Appendix F (Reproducibility Statement) state that source code is available at https://github.com/agiresearch/ASB, including attack scripts, configuration files, and Docker setup."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (tools in JSONL format, attack/normal tool definitions) is included in the GitHub repository (data/all_normal_tools.jsonl and data/all_attack_tools.jsonl referenced in Appendix C.2.6)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F states that environment setup is provided via requirements.txt for GPU/non-GPU systems, and Conda or Docker installation instructions are included in the repository."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F describes predefined attack scripts (e.g., scripts/agent_attack.py), YAML configuration files in config/, and step-by-step instructions for setting up API keys and running experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (percentages) only. No confidence intervals, error bars, or uncertainty measures appear anywhere in the paper's tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims throughout (e.g., 'Mixed Attack is the most impactful', 'DPI is more effective than IPI') but uses no statistical significance tests — only comparisons of raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Effect sizes are not reported. Results are raw ASR percentages; no standardized effect size measures (Cohen's d, etc.) are used, and there is no quantification of practical significance beyond the raw numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 50 agent tasks and 400 attack tasks (Tab. 3), but no justification is provided for why these quantities were chosen or whether they provide sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. All results appear to be single-run numbers; it is unclear whether experiments were repeated or how stable the results are."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares ASB against InjecAgent and AgentDojo in Table 12, showing that ASB covers more attack/defense types and scenarios. Individual attack and defense results are compared across 13 LLM backbones."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The two main comparison benchmarks (InjecAgent, Zhan et al. 2024; AgentDojo, Debenedetti et al. 2024) are contemporary. The defense methods used are recent (2022-2024) and represent the current state of practice."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1.2 analyzes different attack combinations (DPI+IPI, DPI+MP, IPI+MP, full mixed) systematically, and D.1.3 tests different backdoor triggers — these function as ablations of attack components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Seven metrics are defined and used: ASR, Refuse Rate, PNA, BP, FNR, FPR, and the novel NRP metric (Table 4), providing multiple angles on agent security and utility."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation is entirely automated — tool invocation is checked programmatically via simulated API calls. Human evaluation of outputs is clearly irrelevant for this kind of benchmark."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The PoT backdoor testing uses tasks 'different from those in the PoT demonstration to ensure the independence and diversity of the experimental results' (Appendix C.2.2). No prompt tuning on the test set is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by attack type (DPI, IPI, Memory Poisoning, PoT Backdoor, Mixed) and by LLM backbone across Tables 5-8 and 14-21, providing granular per-category analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 provides detailed analysis of why each defense fails (Sec. D.2.1 for DPI/IPI, D.2.2 for PoT, D.2.3 for memory attacks), including qualitative reasoning about the failure mechanisms."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is that current defenses are largely ineffective (Sec. 5.4, Tables 7-8, D.2). The paper explicitly reports that defenses reduce ASR only modestly and often cause utility losses."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (highest average ASR 84.30%, limited defense effectiveness, 10 scenarios, 400+ tools, 27 attack/defense methods, 7 metrics, 13 LLM backbones) are all directly supported by Table 5 and Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal language claims (e.g., 'better agents with stronger backbone LLMs initially exhibit higher ASR due to their superior ability to follow instructions') based only on correlational patterns in Fig. 2 — no controlled experiments isolate this causal mechanism. The ablation combinations also do not isolate single variables cleanly."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents conclusions about 'LLM-based agents' broadly, but all evaluations use synthetic/simulated task environments with AI-generated tools and tasks. Claims about real-world agent vulnerability outrun the simulated evaluation setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity or limitations section exists. The paper does not consider alternative explanations for results, such as whether the synthetic benchmark tasks and AI-generated tools reflect realistic attack conditions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'Claude-3.5 Sonnet', 'GPT-4o', 'GPT-4o-mini', 'GPT-3.5 Turbo'. No snapshot dates or API version strings (e.g., gpt-4o-2024-08-06) are provided. Model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2.4 provides the full actual text of prompts used: the system prompt for LLM-based agents, prompts for paraphrasing defense, instructional prevention, sandwich prevention, dynamic prompt rewriting, refusal judgment, and PoT backdoor trigger generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported anywhere in the paper. These settings significantly affect stochastic outputs and are necessary for reproducibility."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ReAct framework is described in Appendix A.2 with formal notation for the agent's context and action space. The AIOS-based implementation details (Langchain, Chroma vector database, tool simulation) are provided in Appendix C.2.2 and C.2.6."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes how benchmark data was generated using GPT-4: agent descriptions, user tasks, standard plans, tool definitions, and attack tools — all generation processes are described with examples in Tables 9-11."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper has an ethics statement (Appendix E) and a reproducibility statement (Appendix F) but no limitations or threats-to-validity section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion exists. Specific concerns such as the use of simulated (not real) tool calls, AI-generated tasks, or the validity of ASR as a proxy for real-world harm are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. For example, it does not acknowledge that simulated tool calls may not reflect real API behavior, or that results on AI-generated tasks may not transfer to real deployments."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw data (tool definitions, attack/defense configurations) is available via the GitHub repository (data/all_normal_tools.jsonl and data/all_attack_tools.jsonl), allowing verification of benchmark construction."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes in detail how all benchmark data was generated: 10 scenarios × 5 tasks using GPT-4, with normal tools and attack tools generated following specified schema (tool name, description, expected achievement, corresponding agent)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The benchmark data is AI-generated (GPT-4). This question is not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from task generation (GPT-4) through tool simulation (JSONL files → SimulatedTool/AttackerTool objects) to evaluation (tool invocation checking against expected output) is documented in Appendices B and C.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment appears anywhere in the paper. There is no acknowledgments section listing grants, institutional support, or corporate sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated on the title page: Hanrong Zhang, Yifei Yao, Chenlu Zhan, and Hongwei Wang are from Zhejiang University; Jingyuan Huang, Kai Mei, Zhenting Wang, and Yongfeng Zhang are from Rutgers University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The schema says 'NA if unfunded,' but the absence of a funding disclosure does not confirm the work is unfunded -- it may simply be undisclosed. Since we cannot confirm unfunded status, the criterion applies but cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears in the paper. The absence of such a declaration means financial interests cannot be ruled out."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoffs are not stated for any of the 13 LLM backbones tested. Although the tasks evaluate tool-calling behavior rather than factual recall, contamination of the AI-generated benchmark through training data remains a concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of potential train/test overlap is provided. Since the benchmark tasks were generated by GPT-4o and GPT-4o-mini is one of the tested models, there is a risk that the task structure reflects GPT-4's priors, but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is newly constructed (2024) and the tasks are AI-generated, so direct contamination of the tasks is unlikely. However, the paper uses the prompt injection patterns from public sources (OWASP 2023, Liu et al. 2024) that models may have been trained on, and this is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The ethics statement (Appendix E) explicitly states 'No human subjects were involved in this study.'"
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved per the ethics statement (Appendix E)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported. The paper tests 13 LLM backbones across hundreds of attack configurations, which would involve substantial API costs (especially for closed-source models), but these are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total computational budget (GPU hours, API costs, total API calls) is not stated anywhere in the paper. Given the scale of experiments (13 LLMs × multiple attack types × hundreds of tasks), this is a notable omission."
       }
     }
   },
   "claims": [
     {
       "claim": "The highest average attack success rate across all attack types is 84.30%, achieved by the Mixed Attack.",
       "evidence": "Table 5 shows Mixed Attack achieves 84.30% average ASR across 13 LLM backbones. Table 14 also confirms this figure.",
       "supported": "strong"
     },
     {
       "claim": "Current defenses against prompt injection and backdoor attacks are largely ineffective.",
       "evidence": "Tables 7-8 show defense methods reduce DPI ASR from 72.68% to at best 44.45% (Dynamic Prompt Rewriting) and IPI by only 3%; Appendix D.2 details failure analysis.",
       "supported": "strong"
     },
     {
       "claim": "Better agents with stronger backbone LLMs initially exhibit higher ASR due to their superior ability to follow instructions.",
       "evidence": "Figure 2(c) shows a rise-then-fall pattern, and Section 5.3 discusses GPT-4o (60.35% ASR with 20.05% refusal) vs GPT-3.5 Turbo (98.40% ASR, 3% refusal). This is a correlational observation.",
       "supported": "moderate"
     },
     {
       "claim": "PoT backdoor attacks achieve near-perfect ASR (~99.90%) while maintaining similar benign performance to non-backdoored agents.",
       "evidence": "Table 15 shows ASR of 99.75-100% across all five tested triggers; Table 16 shows BP and PNA values are closely matched per backbone.",
       "supported": "strong"
     },
     {
       "claim": "The NRP metric effectively identifies agents that balance utility and security.",
       "evidence": "Table 6 shows NRP scores with Claude-3.5 Sonnet (43.56%) and LLaMA3-70B (30.03%) as top performers, but there is no external validation of NRP's predictive validity or comparison to alternative composite metrics.",
       "supported": "weak"
     },
     {
       "claim": "Agent performance is generally weaker than LLM leaderboard quality.",
       "evidence": "Figure 2(b) shows most models fall below the y=x line when plotting LLM leaderboard quality vs PNA. Claude-3.5 Sonnet, LLaMA3-70B, and GPT-4o are exceptions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Agent Security Bench (ASB) introduces a comprehensive framework evaluating 13 LLM backbones against 5 attack types and 11 defenses across 10 application scenarios. Mixed attacks combining DPI, IPI, and memory poisoning achieve the highest average attack success rate of 84.30%. Current prevention-based defenses are largely ineffective — the best defense (Dynamic Prompt Rewriting) still leaves 44.45% DPI ASR — while detection-based defenses for memory attacks fail to identify 66% of attacks (FNR=0.660). The paper introduces the NRP metric as a composite measure of utility and adversarial resilience.",
   "red_flags": [
     {
       "flag": "No model version strings",
       "detail": "All 13 LLM backbones are identified by marketing name only (e.g., 'Claude-3.5 Sonnet', 'GPT-4o', 'GPT-3.5 Turbo') without snapshot dates or API version IDs. These models change frequently and results cannot be precisely reproduced without version pinning."
     },
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All attack success rates and performance metrics are reported as single point estimates with no confidence intervals, error bars, or repeated-run variance. It is unknown how stable these percentages are across different random seeds or orderings."
     },
     {
       "flag": "AI-generated benchmark tasks and tools",
       "detail": "All 50 agent tasks, 400+ attack tools, and their descriptions were generated using GPT-4 (Appendix B). This creates circularity: some tested models (GPT-4o, GPT-4o-mini) are variants of the model used to generate the benchmark, which may advantage or disadvantage them in ways not analyzed."
     },
     {
       "flag": "Simulated tool calls, not real APIs",
       "detail": "All evaluations use pre-scripted simulated tool responses that return fixed strings (Appendix C.2.1). Real-world attack success may differ substantially when actual API behavior, error handling, and rate limiting are involved."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no dedicated limitations or threats-to-validity section. Key scope boundaries — such as the simulation-only evaluation, AI-generated tasks, or the restriction to tool-invocation as the sole success criterion — are not explicitly acknowledged as limitations."
     },
     {
       "flag": "No funding disclosure",
       "detail": "No funding acknowledgment appears anywhere in the paper, making it impossible to assess potential conflicts of interest."
     },
     {
       "flag": "Causal claims from correlational data",
       "detail": "Section 5.3 makes causal claims about why stronger LLMs have higher ASR (e.g., 'due to their superior ability to follow instructions'), but these are inferred from correlational plots in Figure 2 with no controlled experiments isolating the mechanism."
     }
   ],
   "cited_papers": [
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": [
         "Qiusi Zhan",
         "Zhixiang Liang",
         "Zifan Ying",
         "Daniel Kang"
       ],
       "year": 2024,
       "doi": "10.18653/v1/2024.findings-acl.624",
       "relevance": "Direct predecessor benchmark for indirect prompt injection in LLM agents, compared against ASB in the paper."
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents",
       "authors": [
         "Edoardo Debenedetti",
         "Jie Zhang",
         "Mislav Balunovic",
         "Luca Beurer-Kellner",
         "Marc Fischer",
         "Florian Tramer"
       ],
       "year": 2024,
       "relevance": "A competing agent security benchmark evaluated against ASB; covers IPI attacks and defenses in limited scenarios."
     },
     {
       "title": "AgentPoison: Red-teaming LLM Agents via Poisoning Memory or Knowledge Bases",
       "authors": [
         "Zhaorun Chen",
         "Zhen Xiang",
         "Chaowei Xiao",
         "Dawn Song",
         "Bo Li"
       ],
       "year": 2024,
       "arxiv_id": "2407.12784",
       "relevance": "Prior work on memory poisoning in LLM agents, directly related to the memory poisoning attack category in ASB."
     },
     {
       "title": "Formalizing and Benchmarking Prompt Injection Attacks and Defenses",
       "authors": [
         "Yupei Liu",
         "Yuqi Jia",
         "Runpeng Geng",
         "Jinyuan Jia",
         "Neil Zhenqiang Gong"
       ],
       "year": 2024,
       "relevance": "Formalizes prompt injection attacks; ASB builds on this formalization and extends it to the full LLM agent framework."
     },
     {
       "title": "BadChain: Backdoor Chain-of-Thought Prompting for Large Language Models",
       "authors": [
         "Zhen Xiang",
         "Fengqing Jiang",
         "Zidi Xiong",
         "Bhaskar Ramasubramanian",
         "Radha Poovendran",
         "Bo Li"
       ],
       "year": 2024,
       "relevance": "Proposes backdoor attacks on chain-of-thought prompting; the ASB PoT backdoor attack is directly inspired by and extends BadChain to agentic settings."
     },
     {
       "title": "BadAgent: Inserting and Activating Backdoor Attacks in LLM Agents",
       "authors": [
         "Yifei Wang",
         "Dizhan Xue",
         "Shengjie Zhang",
         "Shengsheng Qian"
       ],
       "year": 2024,
       "arxiv_id": "2406.03007",
       "relevance": "Concurrent work on backdoor attacks specifically targeting LLM agents via task data contamination."
     },
     {
       "title": "Watch Out for Your Agents! Investigating Backdoor Threats to LLM-based Agents",
       "authors": [
         "Wenkai Yang",
         "Xiaohan Bi",
         "Yankai Lin",
         "Sishuo Chen",
         "Jie Zhou",
         "Xu Sun"
       ],
       "year": 2024,
       "arxiv_id": "2402.11208",
       "relevance": "Investigates backdoor threats to LLM agents via fine-tuning contamination, complementary approach to the training-free PoT attack in ASB."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": [
         "Kai Greshake",
         "Sahar Abdelnabi",
         "Shailesh Mishra",
         "Christoph Endres",
         "Thorsten Holz",
         "Mario Fritz"
       ],
       "year": 2023,
       "relevance": "Seminal work demonstrating indirect prompt injection attacks against real-world LLM-integrated applications."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik Narasimhan",
         "Yuan Cao"
       ],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Defines the ReAct agent framework that ASB uses as its baseline agent architecture for all experiments."
     },
     {
       "title": "DecodingTrust: A Comprehensive Assessment of Trustworthiness in GPT Models",
       "authors": [
         "Boxin Wang",
         "Weixin Chen",
         "Hengzhi Pei"
       ],
       "year": 2023,
       "relevance": "Related work on comprehensive trustworthiness evaluation of LLMs, providing context for the security-focused evaluation in ASB."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": [
         "Evan Hubinger",
         "Carson Denison",
         "Jesse Mu"
       ],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Related work on persistent backdoor behaviors in LLMs that survive safety training, relevant to the backdoor attack category in ASB."
     },
     {
       "title": "TrustAgent: Towards Safe and Trustworthy LLM-based Agents through Agent Constitution",
       "authors": [
         "Wenyue Hua",
         "Xianjun Yang",
         "Cheng Wei",
         "Ruixiang Tang",
         "Yongfeng Zhang"
       ],
       "year": 2024,
       "relevance": "Directly relevant work on building safe and trustworthy LLM agents, from the same research group as ASB."
     },
     {
       "title": "Baseline Defenses for Adversarial Attacks against Aligned Language Models",
       "authors": [
         "Neel Jain",
         "Avi Schwarzschild",
         "Yuxin Wen"
       ],
       "year": 2023,
       "relevance": "Provides baseline defense methods (paraphrasing, perplexity detection) that ASB adopts and evaluates in its defense benchmarking."
     }
   ]
 }