ai-research-survey

scan.json (28341B)

---

 {
   "paper": {
     "title": "Can Indirect Prompt Injection Attacks Be Detected and Removed?",
     "authors": [
       "Yulin Chen",
       "Haoran Li",
       "Yuan Sui",
       "Yufei He",
       "Yue Liu",
       "Yangqiu Song",
       "Bryan Hooi"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.16580"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the introduction footnote: 'Code is publicly available at https://github.com/LukeChen-go/indirect-pia-detection.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark and training data are constructed from publicly available datasets (SQuAD validation, TriviaQA validation, Stanford-Alpaca). The injected instructions were crafted using GPT-4o. The code repository presumably includes the benchmark construction, and the underlying QA datasets are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 states: 'We conduct our experiments using PyTorch 2.1.0. The experiments are performed on a single NVIDIA H100-96G GPU.' Training hyperparameters (learning rate 1e-5, epochs 1, max length 1280 with DeepSpeed) and generation parameters (do_sample false, max_new_tokens 256, max_length 8192) are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and hyperparameters are listed, the paper does not provide step-by-step reproduction instructions or a README with commands to run. The information is spread across the paper and appendix but no explicit reproduction guide is mentioned."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-5 report only point estimates (e.g., '99.12%', '97.20%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'the segmentation method demonstrates better overall performance', 'stronger models exhibit less severe over-defense') but provides no statistical significance tests. Comparisons are made by looking at raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw percentages (e.g., true positive rates, removal rates, ASR) but does not report standardized effect sizes. While absolute performance numbers are given, there is no formal effect size reporting such as Cohen's d or odds ratios."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 900 samples per dataset (Inj-SQuAD and Inj-TriviaQA) and training data of ~18,891-19,000 samples, but there is no justification for why these sizes were chosen or whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results from multiple experimental runs are reported. All results appear to be from single runs with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple baselines: existing LLMs (Llama3-8B-Instruct, Qwen2-7B-Instruct), open-source detection models (Llama-Guard3-8B, Protect-AI-detector, Prompt-Guard), and defense baselines (Sandwich, Instructional, StruQ). Results are compared in Tables 3 and 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent models and methods: Prompt-Guard (Meta 2024), Protect-AI-detector (2024), StruQ (Chen et al. 2024a), Llama3-8B-Instruct, and Qwen2-7B-Instruct. These are contemporary and relevant to the task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper investigates multiple factors: injection position effects on detection (Figure 6, Section 5.2), injection rate trade-offs (Figure 3), in-domain vs. out-of-domain evaluation (Table 1), and model size effects (Qwen2-0.5B vs. Qwen2-1.5B). These serve as ablation-like analyses of design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: true positive rate (detection), false positive rate (over-defense), removal rate, and attack success rate (ASR) for defense evaluation, as described in Section 3.3."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here. The paper evaluates automated detection and removal systems where ground truth is known (whether injected instructions are present and whether probes appear in responses). Automated metrics are appropriate."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses separate evaluation benchmarks (Inj-SQuAD and Inj-TriviaQA from validation sets) distinct from the training data (constructed from SQuAD and TriviaQA training sets). Out-of-domain evaluation (trained on SQuAD, tested on TriviaQA) is also performed."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by attack type (Naive, Ignore, Escape, Fakecom, Combined), injection position (head, middle, tail), model type, and in-domain vs. out-of-domain scenarios. Tables 2, 3, and 5 provide detailed breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: over-defense problems (Section 5.2, Table 1), position generalization failures (Figure 6), extraction method struggles with head/middle positions (Section 5.3), and limited generalization against Fakecom attacks (Section 5.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: existing detection models fail at indirect injection detection (Table 5), extraction removal performs poorly at head/middle positions, reducing injection rate does not optimally solve over-defense (Figure 3), and both StruQ and filtering methods struggle with Fakecom attacks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (1) existing models struggle at indirect detection — supported by Table 5; (2) over-defense is position/domain dependent — supported by Table 1 and Figure 3; (3) segmentation shows better overall removal, extraction better at tail — supported by Table 2; (4) combined filtering is effective — supported by Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily from controlled experiments: 'stronger models exhibit less severe over-defense' (comparing model sizes with same training), position effects (training on specific positions), injection rate effects (Figure 3). These are based on controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'Can Indirect Prompt Injection Attacks Be Detected and Removed?' broadly, but experiments are limited to QA-based document scenarios with SQuAD/TriviaQA. The paper does not adequately bound its claims to this specific setting. The conclusions section discusses the findings broadly without qualifying that they apply only to this particular benchmark setup."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For instance, the over-defense problem on out-of-domain data could be due to domain shift rather than model capacity, but this is not explored. No threats-to-validity or alternative explanations section is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to models as 'Llama3-8B-Instruct', 'Qwen2-7B-Instruct', 'Qwen2-0.5B', 'Qwen2-1.5B', 'DeBERTa', 'Llama-Guard3-8B', 'Llama3.2-3B' without specifying exact version snapshots or model card dates. GPT-4o is used for crafting injections but no version/snapshot is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides actual prompt templates in Tables 8-14 showing the exact format for attack methods and defense baselines. The detection prompt format is also implied through the system/user/data structure shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports: learning rate 1e-5, epochs 1, max length 1280, DeepSpeed for training; do_sample false, max_new_tokens 256, max_length 8192 for generation. Training data split ratios (40% clean, 15% head, 30% middle, 15% tail) are specified in Section 3.2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The detection and removal methods are straightforward model inference pipelines without agent loops, tool use, or multi-step reasoning scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the benchmark and training data construction in detail: source datasets (SQuAD/TriviaQA), sample counts (900 per benchmark, 18,891/19,000 training), injection categories (advertisement/phishing/propaganda, 100 each), injection positions and rates, and the distinction between detection and extraction training data (Ddet and Dext)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a dedicated 'Limitations' section after the conclusion, discussing that the removal methods are 'simple and easy to implement, but their performance is not entirely satisfactory' and that direct prompt injection was not considered."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is brief and mostly forward-looking ('leaving room for future exploration'). It does not discuss specific threats to validity of the current results, such as the narrow benchmark domain (only QA tasks), the artificial nature of the injected instructions (GPT-4o generated), or the limited attack diversity in training."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations mention not considering direct prompt injection and that removal methods need improvement, but do not explicitly state what the results do NOT show. For example, the paper does not state that results may not generalize to non-QA domains, multi-turn conversations, or real-world attack scenarios with more sophisticated injections."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (SQuAD, TriviaQA) are publicly available, code is released at the GitHub repository, and the benchmark construction process is documented. The crafted injected instructions should be reproducible from the description."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1-3.2 describes how the benchmark and training data were constructed: source datasets, injection categories with counts (Table 6), the process for creating injected instructions using GPT-4o, the tuple structure (p, d, a, x, y), and the training data split ratios."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data is constructed from existing public datasets (SQuAD, TriviaQA, Stanford-Alpaca) and GPT-4o generated injections. This criterion does not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Sections 3.1-3.2: starting from QA datasets, constructing clean document/injection pairs, splitting into detection training data (Ddet with 40/15/30/15% ratios) and extraction training data (Dext with tripled samples for three positions), and the benchmark construction with 900 samples per dataset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states: 'The work described in this paper was conducted in full or in part by Dr. Haoran Li, JC STEM Early Career Research Fellow, supported by The Hong Kong Jockey Club Charities Trust.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: National University of Singapore and HKUST. No authors are affiliated with companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Hong Kong Jockey Club Charities Trust is a philanthropic organization with no financial stake in prompt injection detection outcomes. The funder is independent of the research findings."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper. The Ethical Considerations section discusses adherence to codes of ethics but does not include a declaration of financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs (Llama3-8B-Instruct, Qwen2-7B-Instruct) on detection tasks but does not state their training data cutoff dates. While the benchmark is custom-constructed, the models' ability to detect injections could be influenced by training data composition."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluated LLMs may have seen SQuAD or TriviaQA data during pre-training, which could affect their detection performance on these domains. The in-domain vs. out-of-domain analysis partially addresses this for trained models but not for the pre-trained LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SQuAD (2016) and TriviaQA (2017) were published well before the training cutoffs of the evaluated models. While the injected instructions are custom-crafted, the underlying documents may have been seen during pre-training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a benchmark evaluation of automated detection and removal methods."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The Ethical Considerations section addresses responsible conduct but not IRB approval, which is not needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes filtering pipelines that add detection and removal steps before LLM inference but does not report the latency or computational cost of these additional steps. No wall-clock time, API costs, or throughput numbers are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions using a single NVIDIA H100-96G GPU (Appendix A.1), it does not state the total training time, GPU hours, or computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing instructed LLMs and open-source detection models struggle to effectively detect indirect prompt injection attacks, while specifically trained models show satisfactory performance.",
       "evidence": "Table 5 shows Llama3-8B-Instruct achieves only 78.74% average accuracy on Inj-SQuAD, Qwen2-7B-Instruct 42.54%. Llama-Guard3-8B max 39.11%. Trained DeBERTa reaches 99.12% accounting for over-defense. (Section 5.2)",
       "supported": "strong"
     },
     {
       "claim": "The over-defense problem rarely occurs with in-domain documents but occurs with out-of-domain documents, where stronger models and more fluent documents are less prone to this issue.",
       "evidence": "Table 1 shows in-domain false positive rates at 0.0-0.44% vs. out-of-domain rates up to 27.33%. Qwen2-1.5B shows fewer over-defense problems than Qwen2-0.5B. SQuAD documents (more fluent) trigger less over-defense than TriviaQA. (Section 5.2)",
       "supported": "strong"
     },
     {
       "claim": "The segmentation removal method demonstrates better overall performance than extraction, but extraction excels at removing tail-position injections.",
       "evidence": "Table 2 shows extraction-Qwen2-1.5B achieves >=94.66% removal at tail position but only 42.33-67.77% for Fakecom/Combined at head/middle. Segment-DeBERTa achieves 79.88-100% across all positions. (Section 5.3)",
       "supported": "strong"
     },
     {
       "claim": "Combining detection and removal as filtering methods is effective for defending against indirect prompt injection attacks, generally outperforming prompt-engineering and fine-tuning baselines.",
       "evidence": "Table 3 shows Segment filtering achieves ASR of 0.11% in most configurations (vs. None baseline 4-96%, Sandwich 2.77-57.22%, Instructional 2.22-95.66%). However, both StruQ and filtering struggle against Fakecom attacks at certain positions. (Section 5.3)",
       "supported": "moderate"
     },
     {
       "claim": "Detection models trained on data with a single injection position struggle to detect attacks at other positions.",
       "evidence": "Figure 6 shows models trained on head-only injection achieve high detection for head attacks but drop significantly for middle and tail attacks (e.g., DeBERTa-OOD drops from 100% head to 20% middle). (Section 5.2)",
       "supported": "strong"
     },
     {
       "claim": "The removal methods will not eliminate key information in clean data despite over-defense of detection models.",
       "evidence": "Table 4 shows QA task accuracy remains comparable across defense methods: None (77.77-80.11%), Segment (77.77-80.00%), Extraction (77.77-80.00%), with only StruQ showing noticeable degradation (75.22-76.11%). (Section 5.3)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper demonstrates that existing LLMs and open-source detection models are largely ineffective at detecting indirect prompt injection attacks, but models specifically trained on crafted datasets can achieve high detection accuracy (>97%). The study reveals an over-defense trade-off that depends on domain match, model capacity, and injection rate. For removal, segmentation outperforms extraction overall, though extraction excels at tail-position injections. Combined detection-plus-removal filtering achieves near-zero attack success rates in most scenarios, outperforming prior prompt-engineering and fine-tuning defenses.",
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, variance across runs, or statistical significance tests. Given the stochastic nature of some components and the relatively small benchmark sizes (900 samples), this is a significant omission."
     },
     {
       "flag": "Narrow benchmark domain",
       "detail": "The evaluation is limited to QA-based scenarios using SQuAD and TriviaQA documents. Real-world indirect prompt injection occurs in diverse contexts (emails, web searches, code, etc.) but generalization to these settings is not tested or discussed."
     },
     {
       "flag": "Training only on Naive attacks",
       "detail": "Detection and extraction models are trained only on 'Naive attack' data, yet the paper claims generalization to other attack types. The acknowledged failure on Fakecom attacks suggests this training strategy is insufficient, but the implications are understated."
     },
     {
       "flag": "Missing inference cost analysis",
       "detail": "Adding a detection-then-removal pipeline has practical implications for latency and cost that are never quantified, making it difficult to assess real-world deployability."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications, directly related to the security evaluation of agentic AI systems."
     },
     {
       "title": "Struq: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Key defense baseline using adversarial fine-tuning to separate instructions from data, directly compared in this paper's experiments."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati", "Yonatan Zunger", "Emre Kiciman"],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Proposes spotlighting as a defense against indirect prompt injection, relevant to the landscape of prompt injection defenses."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Early systematic study of prompt injection attack techniques used as a baseline attack method in this paper."
     },
     {
       "title": "Evaluating the instruction-following robustness of large language models to prompt injection",
       "authors": ["Zekun Li", "Baolin Peng", "Pengcheng He", "Xifeng Yan"],
       "year": 2023,
       "relevance": "Evaluates LLM robustness to prompt injection, providing methodology and benchmarks relevant to the survey scope."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang", "Ning Zhang", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Develops automated prompt injection optimization techniques, representing the evolving attack landscape for LLM security."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Provides formal frameworks for prompt injection evaluation, relevant to benchmarking methodology for LLM security."
     },
     {
       "title": "The instruction hierarchy: Training llms to prioritize privileged instructions",
       "authors": ["Eric Wallace", "Kai Xiao", "Reimar Leike", "Lilian Weng", "Johannes Heidecke", "Alex Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Proposes instruction hierarchy for LLM defense against injection, a fundamental approach to the instruction-data separation problem."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmarks indirect prompt injection in agentic LLM settings with tool use, directly relevant to agentic AI security evaluation."
     },
     {
       "title": "Can llms separate instructions from data? and what do we even mean by that?",
       "authors": ["Egor Zverev", "Sahar Abdelnabi", "Soroush Tabesh", "Mario Fritz", "Christoph H Lampert"],
       "year": 2024,
       "arxiv_id": "2403.06833",
       "relevance": "Examines the fundamental question of instruction-data separation in LLMs, directly relevant to understanding prompt injection vulnerabilities."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen", "Zeming Wei", "Elizabeth Sun", "Basel Alomair", "David Wagner"],
       "year": 2023,
       "arxiv_id": "2312.17673",
       "relevance": "Proposes task-specific fine-tuning as a prompt injection defense, an alternative approach to the detection-based methods studied in this paper."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Keegan Hines", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Provides benchmarks and defenses for indirect prompt injection, directly related to the evaluation of LLM security mechanisms."
     },
     {
       "title": "Prompt injection attack against llm-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Studies prompt injection attacks against LLM-integrated applications, relevant to understanding security vulnerabilities in agentic AI systems."
     }
   ]
 }