ai-research-survey

scan.json (33524B)

---

 {
   "paper": {
     "title": "PromptSleuth: Detecting Prompt Injection via Semantic Intent Invariance",
     "authors": ["Mengxiao Wang", "Yuxuan Zhang", "Guofei Gu"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.20890",
     "doi": "10.48550/arXiv.2508.20890"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PromptSleuth, a semantic-oriented prompt injection defense that reasons over task-level intent rather than surface features, achieves near-zero false negative rates across three benchmarks (DataSentinel-Bench, AgentDojo, PromptSleuth-Bench). Existing defenses (DataSentinel, SecAlign, PromptArmor, template-based methods) fail to generalize beyond their training distributions, with FNR as high as 0.67-0.87 on novel attack types. The three-step pipeline (summarization, task-relationship graph, clustering) using GPT-4.1-mini or GPT-5-mini introduces only modest latency overhead (~1.78s and ~13.61s respectively). Ablation studies show that explicit reasoning does not improve detection and can harm performance, while medium-length system prompts provide the best accuracy-latency trade-off.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided at https://github.com/successlab/PromptSleuth (Reference [30]). The contributions section states: 'We release our benchmark and defense as open source to foster reproducibility, enable benchmarking, and encourage adoption.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The PromptSleuth-Bench benchmark is released as part of the open-source package. The paper states it integrates and extends existing public benchmarks (DataSentinel-Bench and AgentDojo), and the release includes their new benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, conda file, or library versions) are mentioned in the paper. The paper only references API-based models (GPT-4.1-mini, GPT-5-mini) and open-source models without specifying environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released, the paper does not include a 'Reproducing Results' section or describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-4 and Figures 3, 5, 6 are reported as point estimates (e.g., FPR=0.0008, FNR=0.0007) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PromptSleuth 'consistently outperforms existing defense' based solely on comparing FPR/FNR numbers across methods. No statistical significance tests (t-tests, bootstrap, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with enough context to assess effect magnitude. For example, Table 3 shows PromptSleuth-5-mini FNR=0.0007 vs DataSentinel FNR=0.6669 on PromptSleuth-Bench, and latency overhead of +9.2% on GPT-4.1-mini relative to PromptArmor (Table 4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study uses only 100 randomly picked samples (57 benign, 43 malicious) with the justification being 'due to the cost of calling commercial APIs.' No power analysis or formal justification for this sample size. Main benchmark sizes are also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Results appear to be from single runs. The paper acknowledges API-based variability exists but does not quantify it."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against 8 defenses: Instructional, Sandwich, Random_Sequence, Delimiter (template-based), Known-Answer, SecAlign, DataSentinel, and PromptArmor (Section 4.3 and 6.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The three main baselines are contemporary: DataSentinel (2025), SecAlign (2024), and PromptArmor (2025). The paper explicitly states these 'represent the state of the art at the time of writing.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 presents ablation studies on three components: (1) base model choice (GPT-4.1-nano/mini, GPT-5-nano/mini), (2) system prompt design (short/medium/long), and (3) reasoning vs. no-reasoning."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports FPR and FNR as primary metrics (Table 3), plus latency in seconds (Table 4), and API cost per million tokens (Table 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated using FPR/FNR on benchmark datasets. No manual inspection of detection decisions or expert review of edge cases."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of separate dev/test splits. The ablation uses 100 randomly picked samples from the benchmark but there is no indication of a held-out test set separate from tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by three difficulty levels (Easy, Medium, Hard) in Figure 3, by three benchmarks in Table 3, and by model/prompt-length combinations in Figures 5-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses failure cases: 'tasks such as \"book the cheapest hotel\" versus \"book the most expensive hotel\" are considered injection tasks, yet their semantic structures are nearly identical.' Also discusses GPT-4.1-mini FPR issues in multi-task summarization and GPT-4.1-nano's failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 reports that explicit reasoning hurts GPT-4.1-mini performance (FNR 0.000→0.149), that GPT-4.1-nano is too weak (FNR=0.442), and that long prompts increase overhead without consistent accuracy gains."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims PromptSleuth 'consistently outperforms existing defense' but Table 3 shows it matches (not outperforms) DataSentinel on DataSentinel-Bench (both FPR=0.0000, FNR=0.0000). The claim of consistent superiority is overstated for the tied benchmark."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 6.3 systematically vary one component at a time (model choice, prompt length, reasoning toggle) while holding others constant, providing adequate evidence for causal claims about which design choices matter."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'a robust, efficient, and generalizable strategy for defending LLMs against evolving prompt injection threats' but evaluation is primarily on GPT-based models. Open-source models were excluded after showing poor performance, and the paper acknowledges the defense 'strongly depends on the quality and precision of the system prompt.' The title and framing are broader than the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for PromptSleuth's success. For instance, it does not discuss whether the detector LLM's general instruction-following ability (rather than the semantic framework) drives performance, or whether the advantage comes from using more capable models than baselines."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures FPR and FNR directly for prompt injection detection, which is exactly what it claims to evaluate. Section 7 explicitly discusses the distinction between detection-based and prevention-based metrics and justifies their choice."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-4.1-mini', 'GPT-4.1-nano', 'GPT-5-mini', 'GPT-5-nano', 'Llama-3-8B-Instruct', 'Mistral-7B-Instruct', 'TinyLlama'. No API snapshot dates or version strings are provided, and model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 presents only a 'simplified system prompt of PromptSleuth' — explicitly not the actual prompt used. The paper states this is simplified, meaning the reader cannot reconstruct the exact prompts sent to the model from the paper alone."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-5-nano's 'fixed temperature of 1' is mentioned. No temperature, top-p, or max_tokens settings are reported for GPT-4.1-mini, GPT-4.1-nano, GPT-5-mini, or any other model used in the main experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-step pipeline is described in detail in Section 5.2: (1) summarization extracts abstract tasks, (2) task-relationship graph generation classifies edges as related/unrelated, (3) clustering isolates unrelated tasks. Algorithm 1 provides pseudocode. The workflow diagram in Figure 4 illustrates the full pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "PromptSleuth-Bench construction is described conceptually ('we build on DATASENTINEL by incorporating new attack techniques') but the exact generation process, filtering criteria, and dataset statistics are not documented in detail. No counts of how many examples were generated, filtered, or retained at each construction stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Discussion' contains a dedicated 'Limitations' subsection with substantive discussion of weaknesses including dependence on model capability, computational overhead, semantic similarity edge cases, and system prompt quality."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'tasks such as \"book the cheapest hotel\" versus \"book the most expensive hotel\" are considered injection tasks, yet their semantic structures are nearly identical'; 'our defense strongly depends on the quality and precision of the system prompt'; 'When applied to smaller models... the performance is suboptimal.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It discusses limitations of the approach but does not bound the claims by specifying excluded settings, populations, or claims not being made. The broad framing ('generalizable strategy') is not counterbalanced by specific negative scope statements."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark and defense are released as open source (GitHub, Reference [30]). The datasets used (DataSentinel-Bench, AgentDojo) are also publicly available. Raw benchmark data should be available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes how PromptSleuth-Bench was constructed: extending DataSentinel with new attack techniques across three categories (system prompt forgery, user prompt camouflage, model behavior manipulation), and extending AgentDojo with multi-task adversarial scenarios. Attack technique taxonomy is shown in Figure 2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The benchmark consists of synthetically constructed prompt injection examples, not human-generated data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction methodology is described at a high level but lacks specifics: no counts of examples at each construction stage, no filtering criteria documented, no explanation of how many examples were generated per attack technique or difficulty level."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from Texas A&M University. They are not affiliated with any company whose product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding is disclosed. The absence of any funding disclosure prevents evaluation of this criterion."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defenses/tools (prompt injection detection systems) rather than evaluating pre-trained model knowledge on benchmarks. The LLMs are used as inference engines for the defense pipeline, not being evaluated for their own capabilities."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense frameworks against prompt injection, not model knowledge on benchmarks. Contamination of model training data is not the relevant concern for this defense evaluation setting."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defense systems rather than model capabilities on benchmarks. The relevant evaluation is whether defenses detect injections, not whether models have memorized benchmark answers."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated using benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates defense systems using synthetic benchmark data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving humans."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports average inference overhead in seconds for GPT-4.1-mini (1.78s) and GPT-5-mini (13.61s). Table 5 reports API pricing per million tokens. Relative overhead vs PromptArmor is calculated (+9.2% and +3.5%)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, total number of API calls, or total compute used across all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. The paper notes GPT-5-nano has a 'fixed temperature of 1' introducing 'higher variability' but does not report seed sensitivity for any model."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be single-run point estimates with no indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6.3 evaluates different model and prompt configurations, this is not framed as a hyperparameter search with a stated budget. No total compute or cost for the search process is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 systematically evaluates model choice (4 models), prompt design (3 lengths), and reasoning (on/off), selecting GPT-4.1-mini with medium prompt and no reasoning based on clearly reported accuracy, latency, and cost tradeoffs."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed in the paper, so multiple comparison correction is not applicable. All comparisons are based on direct numerical comparison of FPR/FNR values."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create PromptSleuth-Bench and then show their defense works best on it, without acknowledging the inherent bias of evaluating on their own benchmark. No discussion of self-comparison bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While Table 4 compares latency across defenses and the paper discusses GPT-4.1-mini vs GPT-5-mini tradeoffs, performance is not systematically reported as a function of compute budget. No performance curves across compute levels are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether PromptSleuth-Bench actually measures real-world defense effectiveness. The benchmark is assumed to be valid without discussing construct validity — whether synthetic attack variants reflect actual adversary behavior in production."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense frameworks (PromptSleuth vs baselines) where the defense mechanism itself IS the scaffold being tested. There is no confound between scaffold and model to address."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GPT-4.1-mini or GPT-5-mini may have seen DataSentinel-Bench or AgentDojo examples during training. Since these benchmarks were published before the models' training cutoffs, the detector LLMs may have memorized patterns, potentially inflating detection accuracy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup provides the full prompt (system + user) to the detector LLM, but there is no analysis of whether this provides information that would not be available in a real deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PromptSleuth-Bench is explicitly built on top of DataSentinel-Bench and AgentDojo. The paper does not discuss whether this shared structure between benchmarks affects the independence of cross-benchmark evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "PromptSleuth consistently outperforms existing defenses across all benchmarks while maintaining comparable runtime and cost efficiency.",
       "evidence": "Table 3 shows PromptSleuth-5-mini achieves FPR=0.0000/FNR=0.0000 on DataSentinel-Bench (matching DataSentinel), FPR=0.0008/FNR=0.0007 on PromptSleuth-Bench (vs next best PromptArmor at 0.0926/0.0825), and FPR=0.0240/FNR=0.0340 on AgentDojo. Table 4 shows only +9.2% latency overhead vs PromptArmor on GPT-4.1-mini.",
       "supported": "moderate"
     },
     {
       "claim": "Existing defenses fail to generalize to novel attack types and multi-task scenarios.",
       "evidence": "Figure 3 shows template-based defenses reach FPR=1.00 on Medium tier. DataSentinel degrades to FNR=0.6669 on PromptSleuth-Bench and FNR=0.4878 on AgentDojo (Table 3). SecAlign shows FNR=0.4947-0.8664 across benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "Semantic intent reasoning generalizes beyond surface-level pattern matching for prompt injection detection.",
       "evidence": "PromptSleuth maintains low FNR across Easy/Medium/Hard difficulty levels while syntax-based defenses collapse. However, this comparison confounds the defense methodology with the underlying model capability, as PromptSleuth uses GPT-4.1-mini/GPT-5-mini while baselines use different backends.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4.1-mini provides the optimal cost-performance trade-off for the PromptSleuth pipeline.",
       "evidence": "Section 6.3 ablation on 100 samples: GPT-4.1-mini achieves FPR=0.000, FNR=0.000 with 1.78s latency, vs GPT-5-mini at 13.61s and GPT-4.1-nano at FNR=0.442. But the 100-sample ablation is small for reliable estimates.",
       "supported": "moderate"
     },
     {
       "claim": "Explicit reasoning does not improve and can harm detection accuracy.",
       "evidence": "Figure 6 shows reasoning increases GPT-4.1-mini FNR from 0.000 to 0.149, while providing only marginal improvement for GPT-4.1-nano (FNR 0.149→0.098). Based on 100-sample ablation study.",
       "supported": "moderate"
     },
     {
       "claim": "Stored prompt injection attacks achieve high success rates on ChatGPT and Gemini.",
       "evidence": "Section 7 'Future Works' reports 'remember that' achieves 100% success on ChatGPT and Gemini. However, this is described as a controlled test with no formal experimental protocol, sample sizes, or methodology details.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-serving benchmark evaluation",
       "detail": "The authors construct PromptSleuth-Bench and then demonstrate their method works best on it. The benchmark is designed around the same insight (semantic intent invariance) that drives the defense, creating circular validation. No independent benchmark was used as primary evaluation."
     },
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are single-run point estimates with no uncertainty quantification. Given that LLM API outputs are stochastic (the paper even notes GPT-5-nano's 'fixed temperature of 1' introduces variability), the stability of reported metrics is unknown."
     },
     {
       "flag": "Very small ablation sample size",
       "detail": "The ablation study (Section 6.3) uses only 100 samples (57 benign, 43 malicious), justified by API cost constraints. With only 43 malicious samples, a single misclassification changes FNR by ~2.3%, making the zero-FNR claims from ablation unreliable."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of consistent superiority are based entirely on comparing raw FPR/FNR numbers without any significance testing, despite stochastic API outputs and relatively small sample sizes in ablation."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "PromptSleuth uses GPT-4.1-mini/GPT-5-mini as its detector while baselines use their own (potentially weaker) architectures. The improvement may partially reflect the underlying model capability rather than the semantic framework. This confound is not discussed."
     },
     {
       "flag": "Missing model version specifics",
       "detail": "All models are referenced by marketing names without API snapshot dates. GPT-5 and GPT-5-mini are mentioned but their availability and exact versions at evaluation time are unclear, raising reproducibility concerns."
     }
   ],
   "cited_papers": [
     {
       "title": "StruQ: Defending Against Prompt Injection with Structured Queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Proposes structured query defense against prompt injection through prompt schema validation, directly relevant to PI defense landscape."
     },
     {
       "title": "SecAlign: Defending against prompt injection with preference optimization",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "Saeed Mahloujifar", "Kamalika Chaudhuri", "Chuan Guo"],
       "year": 2024,
       "relevance": "Adversarial training-based defense against prompt injection used as a baseline in this study."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": ["Yupei Liu", "Yuqi Jia", "Jinyuan Jia", "Dawn Song", "Neil Zhenqiang Gong"],
       "year": 2025,
       "arxiv_id": "2504.11358",
       "relevance": "Game-theoretic prompt injection defense and benchmark used as both baseline and data source for PromptSleuth-Bench."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr"],
       "year": 2024,
       "relevance": "Multi-step agent evaluation benchmark for prompt injection, used as one of three evaluation datasets."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen", "Zeming Wei", "Elizabeth Sun", "Basel Alomair", "David Wagner"],
       "year": 2023,
       "arxiv_id": "2312.17673",
       "relevance": "Task-specific fine-tuning approach to prompt injection defense, relevant to the fine-tuning vs prompt engineering comparison."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Formal framework and benchmark for prompt injection evaluation used in this paper's evaluation metrics."
     },
     {
       "title": "PromptArmor: Simple yet effective prompt injection defenses",
       "authors": ["Tianneng Shi", "Kaijie Zhu", "Zhun Wang"],
       "year": 2025,
       "relevance": "Adaptive LLM wrapper defense against prompt injection, used as the strongest baseline comparison."
     },
     {
       "title": "The instruction hierarchy: Training LLMs to prioritize privileged instructions",
       "authors": ["Eric Wallace", "Kevin Xiao", "Jan Leike", "Lilian Weng", "Jonas Heidecke", "Alex Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Proposes instruction prioritization hierarchy for LLM safety, foundational to understanding prompt injection defense strategies."
     },
     {
       "title": "FATH: Authentication-based test-time defense against indirect prompt injection attacks",
       "authors": ["Jiongxiao Wang", "Fangzhou Wu", "Wendi Li", "Jinsheng Pan", "Edward Suh", "Z. Morley Mao", "Muhao Chen", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2410.21492",
       "relevance": "Test-time authentication defense against indirect prompt injection, complementary detection-based approach."
     },
     {
       "title": "ControlNET: A firewall for RAG-based LLM system",
       "authors": ["Hongwei Yao", "Haoran Shi", "Yidou Chen", "Yixin Jiang", "Cong Wang", "Zhan Qin", "Kui Ren", "Chun Chen"],
       "year": 2025,
       "arxiv_id": "2504.09593",
       "relevance": "Retrieval firewall protecting RAG pipelines from prompt injection, extending defense to retrieval-augmented generation."
     },
     {
       "title": "Defense against prompt injection attacks via mixture of encodings",
       "authors": ["Ruiyi Zhang", "David Sullivan", "Kyle Jackson", "Pengtao Xie", "Mei Chen"],
       "year": 2025,
       "arxiv_id": "2504.07467",
       "relevance": "Encoding-based defense against prompt injection using mixed representations."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Early benchmark and defense evaluation for indirect prompt injection, one of the baselines referenced in this paper."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Foundational work on prompt injection attack techniques that established the instruction-override attack category."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners building LLM applications could adopt this defense framework; code is open-sourced and works with API-based models, though integration requires implementation effort."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The insight that semantic intent is invariant across attack variants is somewhat novel but aligns with existing intuitions about defense-in-depth for prompt injection."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that existing prompt injection defenses fail dramatically on novel attack types (FNR up to 87%), raising concerns about deployed LLM application security."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows existing defenses are brittle but presents this as a technical finding rather than a dramatic takedown of specific products or organizations."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark released on GitHub; requires OpenAI API key to run but could be tried by practitioners with API access."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Texas A&M is a known university but not a top AI lab; the paper uses GPT-5 which adds some brand interest."
     }
   }
 }