ai-research-survey

scan.json (26878B)

---

 {
   "paper": {
     "title": "Attention is All You Need to Defend Against Indirect Prompt Injection Attacks in LLMs",
     "authors": ["Yinan Zhong", "Qianhao Miao", "Yanjiao Chen", "Jiangyi Deng", "Yushi Cheng", "Wenyuan Xu"],
     "year": 2025,
     "venue": "NDSS Symposium 2026",
     "arxiv_id": "2512.08417"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or repository URL is provided in the paper. The paper mentions the FIPI dataset 'will be made publicly available' (Section I) but does not provide a download link."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the FIPI dataset will be 'open-sourced to support further research' (Abstract) and 'will be made publicly available' (Section I), but no actual download link or repository URL is provided. This is a promise of future release, not actual release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch and NVIDIA A100 GPUs (Section VI-A) but does not provide a requirements.txt, Dockerfile, or detailed library version list sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction scripts, or step-by-step instructions are provided. The paper describes the experimental setup in Section VI-A but does not provide sufficient detail for an independent researcher to reproduce the results without significant guessing."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables I-XII) report only point estimates (e.g., '99.05% accuracy') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RENNERVATE 'outperforms' all baselines but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on raw numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout. For example, Section VI-B states ASR reductions: 'reducing the ASRs by 85.80% on ChatGLM' from specific baseline values (85.9% to 0.1%). Tables II-III show both undefended ASR and defended ASR, allowing readers to compute effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why the FIPI dataset contains 100,000 injected and 10,000 benign instances, or why 1,000 instances were used for sanitization evaluation. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or multi-run statistics are reported anywhere in the paper. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against 15 baselines: 4 classifier-based detection methods (Prompt-Guard, ProtectAI-v2, Attention Tracker, TaskTracker), 5 LLM-based detection methods, 3 prompt-modification prevention methods (Sandwich, Spotlighting, Instructional), StruQ (model-modification), and 2 LLM-based sanitization baselines (Section VI-A)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work: Attention Tracker (2024), TaskTracker (2025), ProtectAI-v2 (2024), StruQ (2024), SecAlign (2024). These represent the current state of the art in IPI defense."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-E presents a thorough ablation study examining three components: 2-step attentive pooling, token-level detector, and mean filter. Each is replaced or removed individually, with results in Table X across 6 datasets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 6 evaluation metrics: Accuracy, False Positive Rate, False Negative Rate (for detection), and Attack Success Rate, Win Rate, Jaccard Similarity (for sanitization), as described in Section VI-A."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual evaluation is performed in two places: (1) 1,000 randomly selected FIPI instances are manually evaluated for quality (Section VI-A, Step 5), and (2) 200 randomly sampled instances are manually evaluated to validate the DeepSeek LLM-as-a-judge approach (Section VI-C, achieving 94.50% accuracy)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A Step 5 explicitly separates training and testing data: 5,000 injected + 5,000 benign instances for testing, with the remaining 100,000 for training. 'There is no overlap between training and testing data, as they originate from different user instruction-clean data pairs.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables II and III break down sanitization performance by attack type (Naive, Escape Characters, Context Ignoring, Fake Completion, Combined). Table V breaks down detection performance by unseen dataset. Table VII breaks down performance by attack method and dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows and discusses two illustrative failure cases: a false positive (benign role-playing prompt misclassified as injection) and a false negative (naive IPI attack undetected). Section VI-C also discusses utility degradation in certain scenarios (Jfleg-RTE) with explanations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports utility degradation in certain unseen dataset scenarios (Table IV, Win Rates falling below 10% for J-R and S-M), and explicitly discusses why: 'RENNERVATE struggles to fully eliminate injected-task-specific content' (Section VI-C). The Discussion section also acknowledges limitations with injected content."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims RENNERVATE 'outperforms 15 commercial and academic IPI defense methods, achieving high precision on 5 LLMs and 6 datasets' and is 'transferable to unseen attacks and robust against adaptive adversaries.' These are supported by Tables I-IX, which show results on 5 LLMs, 6 datasets, unseen attacks, and adaptive adversaries."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily through ablation studies (Section VI-E), which use controlled single-variable manipulation to show that removing each component (2-step attentive pooling, token-level detector, mean filter) degrades performance. This is adequate causal design for component contribution claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds generalization by specifying the 5 LLMs tested (ChatGLM, Dolly, Falcon, LLaMA2, LLaMA3), the specific datasets used, and the attack types evaluated. The Discussion section explicitly acknowledges limitations including the requirement for attention weight access and issues with injected-task-specific content."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for observed patterns. For instance, utility degradation in unseen dataset scenarios is attributed to the 'dual nature of injections' containing task-specific content (Section VI-C). The Discussion section considers whether model-modification methods like StruQ are better suited in some contexts, and addresses limitations of the attention-based approach."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model families and sizes (e.g., 'ChatGLM-6B', 'Dolly-7B', 'Falcon-7B', 'LLaMA2-7B', 'LLaMA3-8B') but does not provide exact snapshot dates or version identifiers. 'GPT-3.5-Turbo' is used without a snapshot date. 'DeepSeek' is referenced without specifying v2, v2.5, or a version identifier (only Table XIII in the appendix mentions 'DeepSeek-v2.5')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The Appendix (Sections B, C, D) provides full prompt templates for all LLM-based baselines (Naive Detection, Response-Based Detection, Known-Answer Detection, LLM-Based Sanitization) and the DeepSeek determination prompt. The probe-witness pairs are described in detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A reports detailed hyperparameters: m=32 response tokens, N=2 residual blocks, hidden dim=512, Adam optimizer with lr=1e-3, annealing rate=0.3, batch size=128, mean filter kernel size k=5, threshold=5. GCG uses 500 iterations, Neural Exec uses 250 iterations. Section VI-F provides sensitivity analysis."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "RENNERVATE is not an agentic scaffolding system. It is a detection/sanitization framework that processes attention features through a neural network. No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A describes the 5-step FIPI construction pipeline in detail: (1) preparing benign instances by rewriting duplicates, (2) creating probe-witness pairs, (3) employing IPI attacks with specified distribution ratios, (4) constructing injected instances with randomized positions and token-level labels, (5) splitting into train/test with specific counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Discussion & Limitations' explicitly discusses limitations including the requirement for attention weight access, challenges with injected-task-specific content, and real-world deployment constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section identifies specific threats: (1) RENNERVATE requires internal attention weights, limiting applicability to API-only users; (2) injected-task-specific content may not be fully removed, leading to utility degradation in grammar correction tasks; (3) model-modification methods may be more effective but require LLM retraining."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states scope boundaries: the approach requires white-box access to attention features, it may not fully remove injected-task-specific content, and it was not tested on multi-modal IPI attacks (identified as future work in Section VIII). The threat model in Section IV bounds what adversary and defender capabilities are assumed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is made available for independent verification. The FIPI dataset is promised for future release but not currently accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A describes the 5-step data collection pipeline in detail: starting from the SEP dataset (9,160 pairs), using GPT-3.5-Turbo to rewrite duplicates, manually designing 100 probe-witness pairs, employing 7 attack methods with specified distribution ratios, and annotating token-level labels."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data consists of synthetically constructed prompt injection examples built from the existing SEP benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section VI-A Steps 1-5 with specific counts: 9,160 original pairs → 10,000 rewritten pairs → 100,000 injected instances + 10,000 benign instances → 5,000+5,000 test, remainder for training. The 1,000-instance quality check is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states: 'This work is supported by National Natural Science Foundation of China Grant 62271280 and Zhejiang Key Laboratory of Electrical Technology and System on Renewable Energy.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Zhejiang University. The paper evaluates open-source models (ChatGLM, Dolly, Falcon, LLaMA2, LLaMA3) with no apparent conflict between authors and the models evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (National Natural Science Foundation of China and Zhejiang Key Laboratory) are academic/governmental bodies with no apparent financial stake in the outcome of IPI defense research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capabilities on a benchmark. It trains a separate detector on attention features to classify injections. The LLMs are used as fixed feature extractors, not as models being benchmarked for their learned knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper is not benchmarking pre-trained model knowledge. It trains a custom detector and explicitly separates training/test data (Section VI-A Step 5). Contamination in the traditional sense (model saw test data during pre-training) is not applicable here."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the evaluation tests a defense system's detection/sanitization capabilities, not a pre-trained model's knowledge. Benchmark contamination of the underlying LLMs is not relevant to the claims being made."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses synthetically constructed datasets and automated evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per example are reported. Section VII mentions the system 'can run efficiently on a single NVIDIA RTX 3090 GPU' but does not quantify inference time or cost per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on 'two NVIDIA A100 GPUs' (Section VI-A) but does not state total GPU hours, training time, or API costs for the LLM-based baselines. The computational budget for GCG and Neural Exec optimization iterations is also not quantified in terms of wall-clock time."
       }
     }
   },
   "claims": [
     {
       "claim": "RENNERVATE achieves 97.88-99.58% IPI detection accuracy across 5 LLMs, outperforming all 15 baselines.",
       "evidence": "Table I shows accuracy of 99.05% (ChatGLM), 97.88% (Dolly), 99.58% (Falcon), 99.43% (LLaMA2), 99.37% (LLaMA3), compared to the best baseline TaskTracker at 95.07% on LLaMA3.",
       "supported": "strong"
     },
     {
       "claim": "RENNERVATE reduces Attack Success Rate to near 0% across all 5 LLMs while preserving text utility.",
       "evidence": "Tables II-III show total ASR reductions from 60.80-85.90% to 0.00-0.20%. Table IV shows Win Rates of 43.60-46.78% on FIPI (close to 50% parity), and Jaccard Similarity scores of 0.9-1.0 (Figures 4-5).",
       "supported": "strong"
     },
     {
       "claim": "RENNERVATE transfers effectively to 5 unseen datasets and 2 unseen attack methods.",
       "evidence": "Table V shows 80.20-100.0% detection accuracy on unseen datasets. Table VII shows 92.50-100% detection accuracy against unseen GCG and Neural Exec attacks, with ASRs reduced to 0-7%.",
       "supported": "strong"
     },
     {
       "claim": "RENNERVATE is robust against black-box (PAIR, TAP) and white-box (adaptive GCG) adaptive adversaries.",
       "evidence": "Table VIII shows PAIR ASR reduced from 94-100% to 0-19%, TAP from 95-100% to 0-9%. Table IX shows adaptive GCG ASR reduced from 93-99.5% to 0-5%.",
       "supported": "strong"
     },
     {
       "claim": "The 2-step attentive pooling mechanism improves generalization to unseen datasets.",
       "evidence": "Table X shows removing 2-step attentive pooling causes accuracy drops of up to 12.65% on unseen datasets (J-R), despite slight improvement on FIPI training distribution.",
       "supported": "strong"
     },
     {
       "claim": "RENNERVATE has a compact parameter size of 0.5-0.8M parameters.",
       "evidence": "Table XIII in the Appendix compares parameter sizes: RENNERVATE at 0.5-0.8M vs. Prompt-Guard at 86M, ProtectAI-v2 at 98M, GPT-3.5 at >=175B.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RENNERVATE is a defense framework against Indirect Prompt Injection (IPI) attacks that leverages attention features for token-level detection and sanitization. It achieves 97.88-99.58% detection accuracy across 5 LLMs, outperforming 15 commercial and academic baselines, while reducing attack success rates to near 0% with minimal utility loss. The system demonstrates transferability to unseen datasets and unseen attack methods (including gradient-based attacks like GCG and Neural Exec), and robustness against adaptive adversaries using both black-box (PAIR, TAP) and white-box strategies. A key limitation is that it requires access to the target LLM's attention weights and may not fully remove injected-task-specific content.",
   "red_flags": [
     {
       "flag": "No variance or multi-run statistics",
       "detail": "All results appear to be single-run numbers with no standard deviations, confidence intervals, or multi-seed experiments reported. For a machine learning system with stochastic training, this makes it impossible to assess result stability."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of outperformance over baselines are based purely on comparing point estimates. No statistical tests are performed to determine whether differences are significant."
     },
     {
       "flag": "No code or data released",
       "detail": "Despite claims that the FIPI dataset will be open-sourced, no repository URL or download link is provided. Code for RENNERVATE is also not released. This prevents independent verification of all reported results."
     },
     {
       "flag": "No inference cost quantification",
       "detail": "The paper does not report inference latency, throughput, or cost per example for RENNERVATE, despite requiring attention feature extraction from the target LLM for every input. This makes practical deployment assessment difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Foundational benchmark for IPI attacks and defenses, used as the basis for FIPI dataset construction."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Model-modification IPI defense baseline achieving comparable performance to RENNERVATE on LLaMA2."
     },
     {
       "title": "Aligning LLMs to be robust against prompt injection",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "Saeed Mahloujifar", "Kamalika Chaudhuri", "Chuan Guo"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "SecAlign approach using alignment techniques for IPI defense during fine-tuning."
     },
     {
       "title": "Attention Tracker: Detecting prompt injection attacks in LLMs",
       "authors": ["Kuo-Han Hung", "Ching-Yun Ko", "Ambrish Rawat", "I Chung", "Winston H Hsu", "Pin-Yu Chen"],
       "year": 2024,
       "arxiv_id": "2411.00348",
       "relevance": "Prior attention-based IPI detection method used as a baseline, demonstrating the attention feature approach for prompt injection."
     },
     {
       "title": "Get my drift? Catching LLM task drift with activation deltas",
       "authors": ["Sahar Abdelnabi", "Aideen Fay", "Giovanni Cherubin", "Ahmed Salem", "Mario Fritz", "Andrew Paverd"],
       "year": 2025,
       "relevance": "TaskTracker method using LLM activations for task drift detection, strongest classifier baseline."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack method used as unseen attack baseline for evaluating defense transferability."
     },
     {
       "title": "Neural Exec: Learning (and learning from) execution triggers for prompt injection attacks",
       "authors": ["Dario Pasquini", "Martin Strohmeier", "Carmela Troncoso"],
       "year": 2024,
       "relevance": "Gradient-based IPI attack used as unseen attack baseline for evaluating defense transferability."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban", "Hamed Hassani", "George J Pappas", "Eric Wong"],
       "year": 2023,
       "arxiv_id": "2310.08419",
       "relevance": "PAIR adaptive attack method adapted for IPI evaluation of defense robustness."
     },
     {
       "title": "Tree of attacks: Jailbreaking black-box LLMs automatically",
       "authors": ["Anay Mehrotra", "Manolis Zampetakis", "Paul Kassianik", "Blaine Nelson", "Hyrum Anderson", "Yaron Singer", "Amin Karbasi"],
       "year": 2023,
       "arxiv_id": "2312.02119",
       "relevance": "TAP adaptive attack extending PAIR with tree search, used for robustness evaluation of IPI defenses."
     },
     {
       "title": "Can LLMs separate instructions from data? And what do we even mean by that?",
       "authors": ["Egor Zverev", "Sahar Abdelnabi", "Soroush Tabesh", "Mario Fritz", "Christoph H Lampert"],
       "year": 2024,
       "arxiv_id": "2403.06833",
       "relevance": "SEP benchmark for IPI evaluation, used as the foundation dataset for constructing FIPI."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Seminal work on indirect prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "BIPIA benchmark and adversarial training defense for IPI attacks."
     },
     {
       "title": "InjecGuard: Benchmarking and mitigating over-defense in prompt injection guardrail models",
       "authors": ["Hao Li", "Xiaogeng Liu", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2410.22770",
       "relevance": "Addresses the false positive problem in IPI detection, relevant to defense evaluation methodology."
     }
   ]
 }