ai-research-survey

scan.json (33691B)

---

 {
   "paper": {
     "title": "Taxonomy, Evaluation and Exploitation of IPI-Centric LLM Agent Defense Frameworks",
     "authors": [
       "Zimo Ji",
       "Xunguang Wang",
       "Zongjie Li",
       "Pingchuan Ma",
       "Yudong Gao",
       "Daoyuan Wu",
       "Xincheng Yan",
       "Tian Tian",
       "Shuai Wang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.15203",
     "doi": "10.48550/arXiv.2511.15203"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage", "survey_methodology"],
   "methodology_tags": ["meta-analysis", "benchmark-eval"],
   "key_findings": "This SoK provides the first comprehensive taxonomy of IPI-centric LLM agent defense frameworks along five dimensions and evaluates 11 representative frameworks across three benchmarks. Policy enforcement and system design defenses achieve near-0% attack success rates but at substantial utility and overhead costs. The authors identify six root causes of defense failure and design three novel adaptive attacks that increase ASR by up to fivefold against specific frameworks, demonstrating that frameworks claiming 'provable security' still harbor exploitable architectural flaws.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. Prompt templates are shown in Appendix E but the evaluation code, framework implementations, and adapted payloads are not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three publicly available benchmarks: AgentDojo, InjecAgent, and Agent Security Bench (ASB). These are standard public benchmarks. However, the 949 adapted payloads from the semantic-masquerading attack and evaluation logs are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup section is provided. The paper mentions GPT-4o version and temperature but does not specify the full runtime environment needed for reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the evaluation methodology but does not provide a README, scripts, or commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-3 and Figures 3-4 are point estimates with no confidence intervals, error bars, or ± notation. ASR and utility are reported as single percentages."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'policy enforcement performs best', 'ASR increased fourfold') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as multiplicative improvements with baseline context. Table 5 shows '3.9X', '2.9X', '2X', '4.0X' increases, and Table 6 shows '2.6X', '4.8X'. Raw ASR values with baselines are provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the benchmark sizes or why these specific numbers of test cases are sufficient. Section 4 notes dataset sizes (629, 2040, 1054 test cases) but provides no power analysis or justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs (temperature=0 implies deterministic decoding but this is not discussed as a design choice for reproducibility)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Undefended GPT-4o and Llama-3.1-8B serve as baselines in Tables 2-3. All defense frameworks are compared against these undefended models across all three benchmarks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated frameworks are recent (2024-2025, Table 1) and include state-of-the-art systems like CaMeL, Progent, LlamaFirewall, and MELON. GPT-4o (2024-11-20) is a contemporary model."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the proposed adaptive attacks. For example, the semantic-masquerading pipeline (Figure 5) has adjusting, judging, and refining steps, but no ablation shows which components are essential."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses five metrics across three dimensions: ASR (security), task success rate and false positive rate (utility), wall-clock time and token usage (overhead). Tables 2-3 and Appendix B report these."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The failure analysis in Section 4.2 states 'two authors independently reviewed all failure logs' for root cause classification, but this is manual annotation of failures, not human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test suites (AgentDojo, ASB, InjecAgent) which are standard evaluation sets. No tuning was done on these test sets — frameworks were evaluated as-is."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns: Table 2 breaks results by benchmark and attack template, Table 3 by scenario (workspace, travel, banking, slack), Figure 3 shows per-attack-method performance, and Table 8 shows per-scenario false positive rates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 provides detailed failure analysis with six root causes and specific case studies (e.g., IsolateGPT banking scenario bypass, CaMeL error message injection, LlamaFirewall hallucination). Appendix C provides additional case studies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Meta SecAlign generalizes poorly across IPI scenarios (Section 4.2, RC 6), some adaptive attack payloads fail and revert to originals (Section 5.1), and Task Shield's high FP rate is explicitly noted (Section 4.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a comprehensive taxonomy (supported by Section 3, Table 1), thorough security/usability assessment (Tables 2-3), six root causes (Section 4.2), and three adaptive attacks that 'significantly improve attack success rates' (Tables 5-6 show 2-5X improvements)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about adaptive attacks ('targeting RC 1 & 2', 'increasing ASR up to fourfold') are justified through controlled experiments: same benchmarks, same models, same frameworks with only the attack method changed. The root cause analysis in Section 4.2 provides the causal mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 explicitly bounds the threat model to template-based attacks against LLM agents. The paper tests on two models (GPT-4o, DeepSeek-V3 in Appendix F) and three benchmarks. The scope is explicitly limited to IPI against agents, not QA LLMs (Section 2.1)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides detailed root cause analysis for defense failures, it does not consider alternative explanations for its findings. For example, the LLM-judged attack success in semantic-masquerading IPI could have judge reliability issues, but this is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's measurements (ASR, task success rate, wall-clock time, token usage) directly correspond to its claims about security, utility, and overhead. The paper acknowledges that its LLM-based attack success judging (Section 5.1) is a proxy for the benchmark's state-based ground truth."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'GPT-4o (version 2024-11-20)' with specific version date. Also specifies DeepSeek-V3, Llama-3.1-8B-Instruct, and Meta SecAlign's LoRA adapter."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides full prompt templates for the payload adjust LLM, payload judge LLM, and attack success judge LLM. Table 7 (Appendix A) provides all attack template payloads with exact text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'temperature parameter set to 0' is stated. For the OpenAI function calling template, the specific template choice is documented. Temperature=0 implies greedy decoding, fully specifying the sampling strategy."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes each defense framework's architecture in detail (plan-exec decoupling, code-then-exec, dual LLM, etc.). Figure 2 illustrates technical paradigms. Figure 5 describes the adaptive attack pipeline. Framework implementations are described with code examples."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark datasets are described with sizes and attack methods (Section 4). The semantic-masquerading payload adjustment process is documented in Section 5.1 and Figure 5, including the refining loop and 949 output payloads. Framework configurations are standardized (Section 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. Section 6 (Conclusion) is brief and does not discuss limitations. Only Ethics Considerations and LLM Usage Considerations sections are present at the end."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues like the reliability of LLM-as-judge for attack success, the limitation of testing only template-based attacks, or the generalizability across model versions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly states 'This SoK focuses solely on IPI against LLM agents' (not QA LLMs). Section 2.3 bounds the threat model to template-based attacks. Section 5.3 acknowledges 'large-scale evaluation is rather impractical' for the isolation-breach attack."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation data, failure logs, or adapted payloads are made available. Only aggregated results in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the three benchmarks with dataset sizes (AgentDojo: 629 cases, ASB: 2040 cases, InjecAgent: 1054 cases), attack methods (Table 7), and framework selection rationale. The adaptive attack payload generation process is described in Section 5.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All data sources are standard public benchmarks (AgentDojo, ASB, InjecAgent)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 documents the full semantic-masquerading payload pipeline (adjust → judge → refine → test → success judge). Section 4 documents the evaluation pipeline. The cascading IPI payload structure is shown in Figure 6."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding section is present in the paper. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: HKUST, Zhejiang University of Technology, Lingnan University, Southeast University, and ZTE Corporation. ZTE is an industry affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence. ZTE Corporation (a telecommunications company) has an author but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates defense frameworks against IPI attacks, not pre-trained model capabilities on knowledge benchmarks. The benchmarks test whether defenses can block injections, not model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defense mechanisms rather than model knowledge. Train/test overlap for the model is not the relevant concern — the benchmarks measure defense effectiveness, not model capability on known tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense frameworks rather than model capabilities on benchmarks. Contamination of model knowledge is not the primary threat to validity here."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations are automated benchmark runs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethics Considerations section confirms 'This study relies on public benchmarks within isolated environments and involves no human subjects.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports wall-clock time and token usage per framework per scenario. Figure 4(b-c) visualize time and token consumption. For example, Task Shield averages 501,294 tokens and 17.23s per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (total API spend, total GPU hours, overall cost) is stated. Per-scenario overhead is reported but the aggregate cost of the full evaluation campaign is not."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The paper uses temperature=0 (deterministic decoding) but does not explicitly discuss this as a deliberate reproducibility measure or report results across different conditions."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not explicitly stated. Temperature=0 implies single deterministic runs, but this is not discussed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was conducted or reported. Frameworks were evaluated with their default configurations and temperature=0 was used uniformly."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explains the standardization approach: 'we standardize the configurations across all benchmarks and defense frameworks' using GPT-4o (2024-11-20) at temperature=0 with OpenAI function calling template. This eliminates configuration selection bias."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied. The paper compares 11 frameworks across 3 benchmarks and multiple attack templates using only raw percentages."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss author-evaluation bias. For the adaptive attacks they propose, their own implementation and judging pipeline could introduce favorable bias, but this is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 and Figure 4 present utility alongside overhead (time and token usage) per framework, enabling readers to assess the security-utility-overhead tradeoff. The paper explicitly discusses this tradeoff in Section 4.1."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly discusses how benchmark design affects results: 'ASB features attacker-intended tools designed to mimic legitimate, user-intent-like functionalities' which explains why Tool Filter and CaMeL show higher ASR on ASB. Framework performance variation across benchmarks is analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 standardizes the backend LLM (GPT-4o with same version and temperature) and tool calling template across all frameworks. Appendix F confirms consistent trends with DeepSeek-V3, addressing model-specific confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o or DeepSeek-V3 training data may have included the benchmark datasets (AgentDojo, ASB, InjecAgent), which could affect baseline ASR or defense behavior."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information that would not be available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or structural similarities between the three benchmarks' test cases, or whether results on one benchmark could be dependent on another."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied to the evaluation."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram or structured search protocol is described. The paper presents a taxonomy of 26 defense frameworks (Table 1) but does not describe the systematic search strategy used to identify them."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "The paper goes beyond simple cataloging by conducting empirical evaluation of representative frameworks on three benchmarks (Tables 2-3), measuring security, utility, and overhead. Section 4.2 provides detailed quality assessment through failure analysis."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether published defense frameworks skew toward positive results or whether unsuccessful approaches are underrepresented in the literature."
       }
     }
   },
   "claims": [
     {
       "claim": "All evaluated defense frameworks significantly reduce ASR compared to undefended baselines, with most lowering ASR below 10%.",
       "evidence": "Table 2 shows undefended GPT-4o averages 23.69% ASR while all defense frameworks are below 12%, with many below 5%. Consistent across three benchmarks (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Policy enforcement and system design defenses achieve near-0% ASR, outperforming other paradigms.",
       "evidence": "Table 2: Progent achieves 0.00% average ASR, ACE 0.10%, IsolateGPT 0.12%. These are the lowest across all evaluated frameworks (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "System design frameworks cause the most severe utility degradation among all defense paradigms.",
       "evidence": "Table 3: IsolateGPT drops utility to 37.11% and CaMeL to 59.79% compared to undefended GPT-4o's 80.41%. Runtime checking (Task Shield 83.51%) and policy enforcing (Progent 72.17%) show much less degradation (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Semantic-masquerading IPI increases ASR up to fourfold against specific defense frameworks.",
       "evidence": "Table 5: ASR increases from 0.48% to 1.91% (3.9X) on IsolateGPT, 1.05% to 3.06% (2.9X) on Tool Filter, 0.95% to 1.90% (2X) on CaMeL, and 0.84% to 3.37% (4.0X) on LlamaFirewall (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "Cascading IPI increases ASR nearly fivefold against MELON framework.",
       "evidence": "Table 6: MELON ASR increases from 0.95% (Important template baseline) to 4.53% (4.8X). Task Shield from 2.07% to 5.37% (2.6X) (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Defense frameworks maintain consistent ASR regardless of attack template, indicating bypasses stem from inherent design flaws rather than template sensitivity.",
       "evidence": "Figure 3 shows 'tightly clustered and overlapping lines of different attack templates' for defended frameworks, unlike baselines which vary significantly (e.g., GPT-4o on Important vs TODO in AgentDojo) (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "An isolation-breach vulnerability exists in IsolateGPT where untrusted Spoke data propagates to the Hub LLM, enabling plan corruption.",
       "evidence": "Section 5.3 demonstrates a two-stage attack where email content propagates from Spoke to Hub, tricking Hub into adding a malicious tool (AugustSmartLockUnlockDoor) to the plan. This is described as a zero-day vulnerability.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge for attack success",
       "detail": "In the semantic-masquerading IPI evaluation (Section 5.1), attack success is judged by an LLM rather than the benchmark's ground-truth state checks. The paper acknowledges this departure ('it is no longer viable after payload adjustment') but does not validate the LLM judge's reliability or accuracy against any ground truth."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across 11 frameworks, 3 benchmarks, and multiple attack templates rely on single-run point estimates with no confidence intervals, significance tests, or variance measures. The absolute ASR differences between frameworks are often small (e.g., 0.12% vs 0.00%) and could reflect noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations or threats to validity. Key unaddressed concerns include: LLM judge reliability, generalizability beyond template-based attacks, potential ceiling effects at low ASR values, and framework configuration choices."
     },
     {
       "flag": "Single case study for strongest attack claim",
       "detail": "The isolation-breach IPI attack (Section 5.3), which claims discovery of a 'zero-day vulnerability,' is demonstrated through only a single case study. The paper acknowledges 'large-scale evaluation is rather impractical' but still claims significance from one example."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite evaluating 11 frameworks across 3 benchmarks and designing 3 novel attacks, no code, evaluation scripts, adapted payloads, or raw results are released. This prevents independent verification of all reported results."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents",
       "authors": ["E. Debenedetti", "J. Zhang", "M. Balunovic", "L. Beurer-Kellner", "M. Fischer", "F. Tramèr"],
       "year": 2024,
       "relevance": "Primary benchmark for evaluating IPI attacks and defenses in LLM agent systems, central to this paper's evaluation."
     },
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": ["Q. Zhan", "Z. Liang", "Z. Ying", "D. Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for indirect prompt injection in tool-integrated LLM agents, one of three benchmarks used in the evaluation."
     },
     {
       "title": "Agent Security Bench (ASB): Formalizing and Benchmarking Attacks and Defenses in LLM-Based Agents",
       "authors": ["H. Zhang", "J. Huang", "K. Mei", "Y. Yao", "Z. Wang", "C. Zhan", "H. Wang", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2410.02644",
       "relevance": "Comprehensive benchmark formalizing agent security evaluation, used as one of three evaluation benchmarks."
     },
     {
       "title": "Defeating Prompt Injections by Design",
       "authors": ["E. Debenedetti", "I. Shumailov", "T. Fan", "J. Hayes", "N. Carlini", "D. Fabian", "C. Kern", "C. Shi", "A. Terzis", "F. Tramèr"],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "CaMeL framework for defeating prompt injections through dual LLM architecture and code-then-exec planning, evaluated and attacked in this paper."
     },
     {
       "title": "Progent: Programmable Privilege Control for LLM Agents",
       "authors": ["T. Shi", "J. He", "Z. Wang", "L. Wu", "H. Li", "W. Guo", "D. Song"],
       "year": 2025,
       "arxiv_id": "2504.11703",
       "relevance": "Policy enforcement defense framework achieving 0% ASR with manual policies, representing the strongest defense paradigm evaluated."
     },
     {
       "title": "SecAlign: Defending Against Prompt Injection with Preference Optimization",
       "authors": ["S. Chen", "A. Zharmagambetov", "S. Mahloujifar", "K. Chaudhuri", "D. Wagner", "C. Guo"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "Fine-tuning based defense using DPO to train LLMs to resist prompt injection, evaluated as representative of fine-tuning paradigm."
     },
     {
       "title": "MELON: Provable Defense Against Indirect Prompt Injection Attacks in AI Agents",
       "authors": ["K. Zhu", "X. Yang", "J. Wang", "W. Guo", "W. Y. Wang"],
       "year": 2025,
       "arxiv_id": "2502.05174",
       "relevance": "Runtime checking defense using parallel agent comparison to detect injection, evaluated and targeted by cascading IPI attack."
     },
     {
       "title": "LlamaFirewall: An Open Source Guardrail System for Building Secure AI Agents",
       "authors": ["S. Chennabasappa", "C. Nikolaidis", "D. Song", "D. Molnar"],
       "year": 2025,
       "arxiv_id": "2505.03574",
       "relevance": "Meta's detection-based defense framework combining guardrails, evaluated and found vulnerable to semantic-masquerading attacks."
     },
     {
       "title": "IsolateGPT: An Execution Isolation Architecture for LLM-Based Agentic Systems",
       "authors": ["Y. Wu", "F. Roesner", "T. Kohno", "N. Zhang", "U. Iqbal"],
       "year": 2024,
       "arxiv_id": "2403.04960",
       "relevance": "Plan-then-execute system design defense, found vulnerable to parameter-level attacks and isolation breaches."
     },
     {
       "title": "Adaptive Attacks Break Defenses Against Indirect Prompt Injection Attacks on LLM Agents",
       "authors": ["Q. Zhan", "R. Fang", "H. S. Panchal", "D. Kang"],
       "year": 2025,
       "arxiv_id": "2503.00061",
       "relevance": "Prior work on optimization-based adaptive attacks against IPI defenses, which this paper extends with logic-driven attacks."
     },
     {
       "title": "The Attacker Moves Second: Stronger Adaptive Attacks Bypass Defenses Against LLM Jailbreaks and Prompt Injections",
       "authors": ["M. Nasr", "N. Carlini", "C. Sitawarin"],
       "year": 2025,
       "arxiv_id": "2510.09023",
       "relevance": "Demonstrates that optimization and search-based adaptive attacks can bypass 12 defense frameworks, motivating the logic-driven attacks in this paper."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "relevance": "Foundational work defining the indirect prompt injection threat against LLM-integrated applications."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu", "N. Du", "I. Shafran", "K. Narasimhan", "Y. Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agentic framework for LLM reasoning and acting, relevant as the architecture most defense frameworks are designed to protect."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The taxonomy and identified vulnerabilities are directly useful for security practitioners building or deploying LLM agent defense systems."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges 'provably secure' claims of frameworks like CaMeL and IsolateGPT by demonstrating exploitable architectural flaws."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates novel attacks that bypass state-of-the-art defenses by up to 5X, claims a zero-day vulnerability in IsolateGPT, and shows fundamental architectural flaws in agent security."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly names and attacks specific frameworks (CaMeL, IsolateGPT, LlamaFirewall) challenging their security claims, though the tone remains academic."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released. Attack templates are in appendices but no executable implementation is provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "References Meta's LlamaFirewall and uses GPT-4o but the paper itself is from HKUST, not a major AI lab."
     }
   }
 }