ai-research-survey

scan.json (30391B)

---

 {
   "paper": {
     "title": "FATH: Authentication-based Test-time Defense against Indirect Prompt Injection Attacks",
     "authors": [
       "Jiongxiao Wang",
       "Fangzhou Wu",
       "Wendi Li",
       "Jinsheng Pan",
       "Edward Suh",
       "Z. Morley Mao",
       "Muhao Chen",
       "Chaowei Xiao"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2410.21492",
     "doi": "10.48550/arXiv.2410.21492"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FATH, an authentication-based test-time defense against indirect prompt injection, achieves near 0% attack success rate on GPT-3.5 across all tested attack methods including adaptive and optimization-based attacks. The method uses HMAC-based dynamic tags to label LLM responses and rule-based parsing to filter outputs, outperforming four existing test-time defenses. However, defense effectiveness is weaker on Llama3-8B (26-34% ASR under adaptive attacks), and the method incurs a Judge Score penalty of ~1.0-1.6 points, indicating reduced generation quality.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly states: 'Our code is released at: https://github.com/Jayfeather1024/FATH'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmark data is publicly available: Stanford Alpaca (Apache-2.0), QA from Zverev et al. 2024 (CC BY 4.0), CLF from OpenPromptInjection (CC BY 4.0), InjecAgent (MIT), and URLs generated by the Python 'fake' package (MIT). Appendix G documents all dataset details and licenses."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Meta-Llama-3-8B-Instruct with 1x NVIDIA A100 GPU' and 'gpt-3.5-turbo with OpenAI API' but provides no requirements.txt, Dockerfile, or detailed library version list for dependencies like the hmac package or Sentence Transformers."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repository is provided and prompt templates are given in appendices, the paper itself does not include step-by-step reproduction instructions (commands to run, scripts to execute, or a 'Reproducing Results' section)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All ASR values in Tables 2 and 3 are reported as point estimates (e.g., 0.08, 0.00, 0.26) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims FATH outperforms all baselines based on comparing raw ASR numbers without any statistical significance tests (no p-values, no t-tests, no bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results in Tables 2 and 3 show both baseline and defense ASR values, providing clear context for the magnitude of improvement (e.g., No Defense ASR 0.60 → FATH ASR 0.00 for GPT-3.5 URL Injection under Combined Attack)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 text examples per injection task for OpenPromptInjection+ and 510 for InjecAgent without justifying these sample sizes or performing power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run results are reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares FATH against four baseline test-time defenses: Instructional Prevention, Sandwich Prevention, Text Instruction Isolation, and ICL Defense, plus a No Defense setting."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are from Liu et al. 2023b and Yi et al. 2023, which are the most recent test-time defense methods for indirect prompt injection at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 and Table 4 present ablation studies removing Authentication Tags and Security Policy individually, showing the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both Attack Success Rate (ASR) for defense effectiveness and Judge Score for generation quality impact (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: ASR is computed programmatically and Judge Score uses GPT-3.5 as an LLM judge. No human evaluation of defense effectiveness or output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a separate dev/test split. In-context examples are selected via semantic similarity to the user instruction, which means the ICL selection process operates on the same benchmark used for evaluation, without a clear separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by three injection task categories (URL, QA, CLF), by five attack methods plus adaptive attacks, and across two models (Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports that FATH has higher ASR under adaptive attacks on Llama3 (0.26-0.34 in Table 2), it does not analyze specific failure cases or explain why the defense fails in those instances."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that FATH reduces the Judge Score from 8.31 to 6.73 on Llama3 and from 7.94 to 6.91 on GPT-3.5, acknowledging 'a small decrease in the Judge Score.' It also reports that FATH under Llama3 still allows 26-34% ASR under adaptive attacks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'near 0% ASR on GPT3.5 for various attack methods, surpassing all previous defenses,' which is confirmed by Table 2 showing 0.00-0.02 ASR across all attacks. The abstract also specifies 'under Llama3 and GPT3.5 models,' appropriately bounding the performance claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that FATH's components (Authentication Tags, Security Policy) contribute to defense effectiveness. The ablation study in Section 5.6 provides controlled single-variable manipulation evidence, showing removal of Security Policy increases ASR by 30%+ under adaptive attacks (Table 4)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "While the abstract bounds specific performance claims to 'Llama3 and GPT3.5,' the title and broader claims like 'effectively defend against indirect prompt injection attacks' and 'securing LLM-integrated applications' are not bounded to the two models tested. The Limitations section partially addresses this but the framing remains broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for the observed results. For example, it does not discuss whether the low ASR could be due to prompt complexity overwhelming the model rather than the authentication mechanism specifically, or whether the effectiveness is specific to the tested attack patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Attack Success Rate, which directly corresponds to its claims about defense against prompt injection attacks. The Judge Score metric is also clearly framed as a measure of generation quality impact. No proxy-outcome gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Meta-Llama-3-8B-Instruct' (a specific model version) but only 'gpt-3.5-turbo' without a snapshot date or API version for the OpenAI model. GPT-3.5-turbo changes behavior over time without a pinned version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 3, 4, 7, 8, and appendices D, E, with the actual text used. The security policy, in-context examples, attack templates, and adaptive attack prompts are all included with explicit text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 states 'We set all parameters to default for model generation' without specifying the actual values of temperature, top-p, or other sampling parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "FATH is a prompt-based defense method, not an agentic scaffolding system. It uses HMAC tag generation and rule-based parsing, but these are not agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes how OpenPromptInjection+ was constructed: selecting examples from Stanford Alpaca with both 'instruction' and 'input', defining three injection task categories (URL, QA, CLF), and details on dataset sources. Appendix G provides dataset licenses and sources."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing three specific limitations of FATH."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses threats specific to this study: (1) substantial manual effort for prompt design across applications, (2) reliance on instruction-following ability making FATH ineffective on weaker models like Alpaca, (3) limited benchmark coverage that does not include real tool usage scenarios."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific boundaries: FATH 'may be reduced when applied to LLMs with comparatively weaker instruction-following abilities,' and current benchmarks 'can not provide real tool usage scenarios' so they 'directly provide external text information to simulate the results of tool execution.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release raw experimental outputs, LLM responses, or per-example evaluation logs for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix G describe benchmark construction: 100 examples from Stanford Alpaca per injection task, 510 examples from InjecAgent for direct harm, with sources and licenses documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks and programmatically generated data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the pipeline from data source selection (Section 5.1), through attack injection (Appendix C), defense prompt construction (Section 4 and appendices), to evaluation metric computation (Section 5.2). Dataset construction for OpenPromptInjection+ is described step by step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UW-Madison, Huazhong University, University of Rochester, NVIDIA, Cornell University, University of Michigan, UC-Davis."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence. One author (Edward Suh) is affiliated with NVIDIA, which has a commercial interest in LLM security."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a defense method against prompt injection attacks, not model knowledge or capability on a benchmark. Contamination of training data with benchmark answers is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense effectiveness, not model knowledge. Whether the model saw similar attack patterns during training is a different concern from traditional train/test contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper tests defense robustness, not whether models memorized benchmark answers."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "FATH requires generating HMAC tags, running semantic search for ICL examples, and querying the LLM with substantially longer prompts (the defense prompt in Figure 3 is very long). No inference cost, latency, or API cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '1x NVIDIA A100 GPU' for Llama3 but does not state total GPU hours, API costs, or wall-clock time for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs with no seed sensitivity analysis reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the reported results. No mention of averaging across runs or number of trials."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "FATH involves design choices (number of ICL examples, tag generation, prompt template structure) but no hyperparameter search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents one configuration of FATH (5 tags, N+1 ICL examples) without explaining how this configuration was selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 5 defense methods, 6 attack types, 3 injection tasks, and 2 models, but no correction for multiple comparisons is applied. No statistical tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose FATH and evaluate it against their own implementations of baseline defenses without acknowledging potential self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "FATH uses significantly longer prompts with ICL examples and security policies compared to simpler baselines, but no compute cost comparison across methods is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper introduces OpenPromptInjection+ but does not discuss whether this benchmark accurately represents real-world prompt injection scenarios. The Limitations section acknowledges that 'current benchmarks can not provide real tool usage scenarios' but does not formally analyze construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "FATH is a prompt-level defense, not a scaffold. No scaffolding comparison is involved."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether attack patterns in the benchmarks could have appeared in the training data of GPT-3.5 or Llama3, which could affect how models respond to injections."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup could potentially leak information about the expected response format through the extensive ICL examples."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the ICL demonstration examples and test examples share structural similarities that could inflate performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "FATH achieves near 0% ASR on GPT-3.5 across all attack methods and injection tasks on OpenPromptInjection+.",
       "evidence": "Table 2 shows 0.00-0.02 ASR for GPT-3.5 under Threat Modeling 1 attacks and 0.00 ASR under adaptive attacks across URL, QA, and CLF injection tasks.",
       "supported": "strong"
     },
     {
       "claim": "FATH achieves 0% ASR on both GPT-3.5 and Llama3 on the InjecAgent benchmark for Combined and Adaptive attacks.",
       "evidence": "Table 3 shows 0.00 ASR for both models under both attack types, compared to 99.3% (Llama3) and 1.00 (GPT-3.5) without defense.",
       "supported": "strong"
     },
     {
       "claim": "FATH achieves 0% ASR under optimization-based attacks on Llama3, while the no-defense setting has 70% ASR.",
       "evidence": "Section 5.5 reports these results with an example of the optimized injection prompt and FATH's successful defense output.",
       "supported": "moderate"
     },
     {
       "claim": "FATH surpasses all previous test-time defense methods in defense performance.",
       "evidence": "Table 2 shows FATH has the lowest ASR across all attack methods for both models compared to Instructional Prevention, Sandwich Prevention, Isolation, and ICL Defense.",
       "supported": "strong"
     },
     {
       "claim": "Both Authentication Tags and Security Policy are necessary components for FATH's effectiveness, especially against adaptive attacks.",
       "evidence": "Table 4 ablation study on GPT-3.5 shows removing Security Policy increases adaptive attack ASR to 34-56%, and removing Authentication Tags increases it to 6-18%, compared to 0% for full FATH.",
       "supported": "strong"
     },
     {
       "claim": "FATH defense introduces only a small decrease in generation quality.",
       "evidence": "Table 2 shows Judge Score drops from 8.31 to 6.73 on Llama3 (~19% decrease) and from 7.94 to 6.91 on GPT-3.5 (~13% decrease). Characterizing a 1.0-1.6 point drop as 'small' is debatable.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results are reported as single-point estimates without confidence intervals, standard deviations, or information about how many runs produced each number. Given that LLM outputs are stochastic, this is a significant omission."
     },
     {
       "flag": "Small sample sizes without justification",
       "detail": "Only 100 examples per injection task category are used for OpenPromptInjection+. With ASR values at or near 0%, the precision of these estimates is limited (1 success out of 100 = 1% ASR, creating a floor effect)."
     },
     {
       "flag": "Inconsistent framing of Llama3 adaptive attack results",
       "detail": "The paper emphasizes 'near 0% ASR' in framing but FATH allows 26-34% ASR on Llama3 under adaptive attacks (Table 2), which is substantially higher than 'near 0%' and only briefly acknowledged."
     },
     {
       "flag": "Judge Score penalty downplayed",
       "detail": "The paper characterizes the Judge Score drop (8.31→6.73 on Llama3, 7.94→6.91 on GPT-3.5) as 'a small decrease,' but a 1.0-1.6 point decline on a 10-point scale represents meaningful degradation of response quality."
     },
     {
       "flag": "Only two models tested",
       "detail": "Evaluation is limited to Llama3-8B-Instruct and GPT-3.5-turbo. The Limitations section notes FATH depends on instruction-following ability, but no larger or more capable models (GPT-4, Claude, Llama 70B) are tested."
     },
     {
       "flag": "Simulated rather than real tool usage evaluation",
       "detail": "The InjecAgent evaluation simulates tool usage by directly providing external text information rather than testing in a real tool-calling environment, as acknowledged in the Limitations section."
     }
   ],
   "cited_papers": [
     {
       "title": "Prompt injection attacks and defenses in LLM-integrated applications",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2023,
       "arxiv_id": "2310.12815",
       "relevance": "Proposes the OpenPromptInjection benchmark and several defense methods (Instructional Prevention, Sandwich Prevention, Isolation) used as baselines in this paper."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications, demonstrating practical exploitation scenarios."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Provides the InjecAgent benchmark for evaluating prompt injection in tool-integrated LLM agents, used as a primary evaluation benchmark."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Training-time defense against prompt injection using structured queries, representing an alternative defense paradigm."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Keegan Hines", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Proposes ICL Defense baseline and special token-based training-time defense against indirect prompt injection."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati", "Yonatan Zunger", "Emre Kiciman"],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Test-time defense using data spotlighting techniques to help LLMs distinguish instructions from external data."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang", "Ning Zhang", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Proposes the optimization-based prompt injection framework used as a worst-case attack in FATH evaluation."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R Narasimhan", "Yuan Cao"],
       "year": 2022,
       "relevance": "Foundational agentic framework used in InjecAgent benchmark for tool-integrated LLM evaluation."
     },
     {
       "title": "A new era in LLM security: Exploring security concerns in real-world LLM-based systems",
       "authors": ["Fangzhou Wu", "Ning Zhang", "Somesh Jha", "Patrick McDaniel", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2402.18649",
       "relevance": "Demonstrates practical security vulnerabilities in LLM systems including exploitation of code interpreter and web access capabilities."
     },
     {
       "title": "Tensor Trust: Interpretable prompt injection attacks from an online game",
       "authors": ["Sam Toyer", "Olivia Watkins", "Ethan Adrian Mendes", "Justin Svegliato", "Luke Bailey", "Tiffany Wang", "Isaac Ong", "Karim Elmaaroufi", "Pieter Abbeel", "Trevor Darrell"],
       "year": 2023,
       "relevance": "Crowdsourced prompt injection attacks and defenses providing real-world attack pattern data."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang", "Tianwei Zhang", "Yepang Liu", "Haoyu Wang", "Yan Zheng", "Yang Liu"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Early systematic study of prompt injection attacks providing attack templates (Naive, Escape Characters, Context Ignoring, Fake Completion) used in this paper."
     },
     {
       "title": "Mind2Web: Towards a generalist agent for the web",
       "authors": ["Xiang Deng", "Yu Gu", "Boyuan Zheng", "Shijie Chen", "Sam Stevens", "Boshi Wang", "Huan Sun", "Yu Su"],
       "year": 2024,
       "relevance": "Web agent benchmark demonstrating LLM-integrated applications vulnerable to indirect prompt injection."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "FATH can be applied by developers to defend LLM-integrated applications at inference time without model fine-tuning, but requires significant per-application prompt engineering."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The authentication-based framing is a novel angle but the core idea of using tags and ICL for defense is incremental over prior work."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses prompt injection (OWASP Top 1 for LLMs) and demonstrates both attacks and defenses, raising awareness of LLM security risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; straightforward defense method comparison."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with prompt templates provided in appendices, but not packaged as a pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from UW-Madison and NVIDIA; uses GPT-3.5 and Llama3 which are well-known but not the flagship models being tested."
     }
   }
 }