ai-research-survey

scan.json (32632B)

---

 {
   "paper": {
     "title": "Formalizing and Benchmarking Prompt Injection Attacks and Defenses",
     "authors": [
       "Yupei Liu",
       "Yuqi Jia",
       "Runpeng Geng",
       "Jinyuan Jia",
       "Neil Zhenqiang Gong"
     ],
     "year": 2023,
     "venue": "USENIX Security Symposium",
     "arxiv_id": "2310.12815"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes the first formal framework for prompt injection attacks and shows that a Combined Attack (merging escape characters, context ignoring, and fake completion) achieves the highest average ASV of 0.75 on GPT-4, outperforming all individual attacks (0.62–0.70). Systematic evaluation of 10 defenses across 10 LLMs and 7 tasks finds no existing defense is sufficient: prevention-based defenses have limited effectiveness or incur large utility losses, and detection-based defenses either miss many attacks or have high false positive rates. Larger LLMs are more vulnerable to prompt injection (Pearson r=0.63 between model size and ASV).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public GitHub repository: 'we make our platform public at https://github.com/liu00222/Open-Prompt-Injection' (Section 1, abstract)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 7 datasets used are publicly available standard benchmarks: SST2, MRPC, HSOL, RTE, SMS Spam, Jfleg, and Gigaword. The paper specifies which splits are used for each purpose (Section 6.1, Appendix A)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is provided in the paper. The paper mentions using Azure OpenAI Studio API and specific model names but does not specify library versions or environment dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper releases a GitHub platform but does not include step-by-step reproduction instructions in the paper itself. No 'Reproducing Results' section, README commands, or scripts are described in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (ASV, MR, PNA-T, FPR, FNR) in Tables 4–9 and throughout the paper are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'Combined Attack outperforms other attacks' and makes numerous comparisons across attacks, defenses, and models, but no statistical significance tests (p-values, t-tests, etc.) are used. All comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Raw ASV/MR values are reported with full baseline context. For example, Table 4 shows Combined Attack (0.75) vs Naive Attack (0.62) vs Fake Completion (0.70) on GPT-4. Per-task breakdowns in Tables 5–6 and defense comparisons in Tables 7–8 provide magnitude context for all differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 examples per task for target and injected data, and randomly samples 100 pairs for ASV/MR computation (Section 6.1), but provides no justification for why 100 was chosen and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. For open-source LLMs, a fixed seed produces deterministic single-run results. For closed-source LLMs, temperature is set to 0.1 and the paper only notes 'non-determinism has a small impact' without quantifying it (Section 6.1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five attacks are compared (Naive, Escape Characters, Context Ignoring, Fake Completion, Combined Attack) and 10 defenses are benchmarked. The 'No defense' baseline is included for defense evaluation (Table 7a)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All attacks and defenses are from 2022–2023 works (references [4,8,9,11,14,23,25,30,31,34,35,40,43,50,51]), which were contemporary at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The Combined Attack is itself an ablation: individual components (Escape Characters, Context Ignoring, Fake Completion) are evaluated separately and then in combination (Table 4, Figure 2). Additional ablations study impact of in-context learning examples (Figure 4), number of tokens in injected data (Figure 7), and number of tokens in injected instruction (Figure 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five evaluation metrics are used: PNA-T, PNA-I, ASV, MR for attacks/prevention, and FPR/FNR for detection-based defenses (Section 6.1, Equations 2–6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated using accuracy, ROUGE-1, and GLEU score metrics. No human evaluation of attack success or defense quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully ensures no overlap between target data, injected data, in-context learning examples, and clean data for PPL threshold selection. Appendix A details the data splitting procedure across all datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: per-target-task, per-injected-task, and per-LLM results in Tables 5–6, Tables 12–20, and per-defense breakdowns in Tables 21–32."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where attacks fail (e.g., summarization as injected task achieves lowest MR of 0.67 in Table 6b), where defenses fail (all defenses shown insufficient in Tables 7–8), and specific failure modes of each detection method (Section 6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative findings are prominent: all existing defenses are insufficient (Section 6.3), paraphrasing sacrifices utility (PNA-T decreases by 0.14, Table 7b), naive LLM-based detection has very high FPR (up to 0.93, Table 8b), and PPL detection misses nearly all attacks (FNR up to 1.00, Table 8a)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: the framework is formalized (Section 4), the new Combined Attack is designed and shown effective (Tables 4–6), systematic evaluation covers 5 attacks, 10 defenses, 10 LLMs, 7 tasks (Section 6), and no existing defense is shown sufficient (Tables 7–8)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that combining attack strategies improves effectiveness — is justified through controlled single-variable manipulation: all attacks are tested on the same data/models with only the attack strategy varying (Table 4, Figure 2). Ablation studies (Figures 4, 7, 8) also use controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested setting: '5 prompt injection attacks and 10 defenses with 10 LLMs and 7 tasks.' The paper specifies which models, datasets, and tasks were tested and does not claim results generalize beyond these. The threat model is explicitly scoped (Section 3)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, it speculates that larger models are more vulnerable because they are 'more powerful in following instructions' (Section 6.2) but does not consider other factors. Section 8 discusses future work, not alternative explanations for observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's metrics directly measure what is claimed. ASV measures whether the LLM accomplishes the injected task (attack success), MR compares responses with/without attack, and FPR/FNR directly measure detection accuracy. No proxy gap exists between measurements and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models have specific versions (Vicuna-33b-v1.3, PaLM 2 text-bison-001, Llama-2-13b-chat), but the primary model GPT-4 is referenced without a snapshot date or API version. GPT-3.5-Turbo and Bard also lack version specificity (Table 3, Section 6.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 11 in Appendix provides the exact instruction prompt and injected instruction text for all 7 tasks. Table 1 shows example compromised data for each attack. The GPT-4 API message format is also specified (Section 6.1)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports temperature=0.1 for closed-source LLMs and fixed random seed for open-source LLMs (Section 6.1), but does not report top-p, max_tokens, or other API parameters that significantly affect output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper directly queries LLMs with prompts containing instruction + data, with no tools, retry logic, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides detailed data selection procedures: which dataset splits are used, how 100 examples are sampled, how label conflicts are handled when target and injected tasks are the same classification task, and how in-context examples are selected without overlap."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Discussion and Limitations' provides substantive discussion covering four specific limitations: lack of optimization-based attacks, unexplored fine-tuning defenses, absence of recovery mechanisms, and limited detection prompt exploration."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: 'All existing prompt injection attacks are limited to heuristics' (not optimization-based), known-answer detection was 'limited to a specific detection prompt,' and fine-tuned LLMs 'may still be vulnerable to new attacks that were not considered during fine-tuning.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 explicitly scopes the attacker's background knowledge ('we assume the attacker does not know such internal details'), Section 7 distinguishes prompt injection from jailbreaking, and Section 8 states specific things not tested: optimization-based attacks, fine-tuning defenses, recovery mechanisms, and alternative detection prompts."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The underlying benchmark datasets are publicly available, and the platform code is released, but raw experimental outputs (individual model responses, intermediate results) are not provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 and Appendix A describe in detail which datasets are used, how examples are sampled (100 per task, uniform random without replacement), how label conflicts are resolved, and how data splits are assigned."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public NLP benchmarks (SST2, MRPC, HSOL, RTE, SMS Spam, Jfleg, Gigaword)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: dataset selection → example sampling (100 per task) → label conflict resolution for same-task scenarios → separate sampling for in-context examples and PPL thresholds with no-overlap guarantees → 100 random pair sampling for ASV/MR computation (Section 6.1, Appendix A)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists NSF grants (No. 2112562, 1937786, 2131859, 2125977, 1937787), ARO grant (No. W911NF2110182), and Microsoft Azure credits."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Penn State University and Duke University. No authors are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Primary funding is from NSF and ARO, which are government agencies independent of the evaluated products. Microsoft Azure credits provided computing resources but the paper's finding that GPT-4 is vulnerable to injection is not favorable to Microsoft."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests prompt injection attack/defense effectiveness, not model knowledge or capability on benchmarks. The core metric (ASV) measures whether the LLM follows injected instructions, not whether it knows correct benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates attacks and defenses rather than model knowledge. Whether models have seen the NLP benchmark data does not undermine the core attack effectiveness measurements."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Contamination is structurally less relevant here: the paper measures instruction-following behavior under adversarial conditions, not model capability on benchmark tasks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve automated querying of LLMs on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time are reported despite querying 10 LLMs across 49 task combinations, 5 attacks, and 10 defenses."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported. The paper acknowledges using Azure credits but does not quantify the compute used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "For open-source LLMs, a single fixed seed is used. For closed-source LLMs, temperature=0.1 is used. No sensitivity analysis across multiple seeds is performed (Section 6.1)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state the number of experimental runs. Fixed seed implies single deterministic runs for open-source models, but this is never explicitly stated as 'one run' or 'averaged over K runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Attack parameters (escape characters, task-ignoring text, fake response) use fixed templates. The PPL detection threshold is set via a principled procedure, but no search budget is reported for other configuration choices."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All five attacks are evaluated transparently without cherry-picking. Defense configurations follow original papers. The PPL detection threshold is selected via a principled FPR-based procedure on separate clean data (Section 5.2). No config selection bias is evident."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes hundreds of comparisons across 10 LLMs, 49 task combinations, 5 attacks, and 10 defenses, but no statistical significance tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own versions of all attacks and defenses and propose the Combined Attack that outperforms all others. They do not acknowledge the bias of evaluating their own system against their own implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models ranging from 7B to 1.5T parameters and finds larger models are more vulnerable, but does not control for or report compute budget differences. No performance-vs-compute curves are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASV/MR on standard NLP tasks actually captures real-world prompt injection risk. The gap between benchmarked attack success on controlled tasks and real-world LLM application vulnerability is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All experiments involve direct prompting of LLMs via API or local inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests GPT-4 and other models on datasets like SST2 (2013), MRPC (2005), and SMS Spam (2011), which are almost certainly in training data. This is not discussed. While the core attack metric (ASV) is less affected, PNA baseline performance could be artificially inflated."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup provides the LLM with full task instruction and data, which matches the intended use case, but potential information leakage through benchmark memorization is not considered."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Target data and injected data are sampled from the same datasets (though without overlap), and independence from training data is not verified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference tests, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "The Combined Attack (merging escape characters, context ignoring, and fake completion) outperforms all individual prompt injection attacks, achieving an average ASV of 0.75 on GPT-4 versus 0.62–0.70 for other attacks.",
       "evidence": "Table 4 shows average ASVs: Naive 0.62, Escape Characters 0.66, Context Ignoring 0.65, Fake Completion 0.70, Combined 0.75. Figure 2 shows per-task breakdowns confirming Combined Attack leads in nearly all task combinations.",
       "supported": "strong"
     },
     {
       "claim": "No existing prevention-based defense is sufficient: they either have limited attack prevention or incur large utility losses.",
       "evidence": "Table 7a shows ASV/MR remain high under all defenses (e.g., sandwich prevention: avg ASV still 0.26–0.70 depending on task). Table 7b shows paraphrasing decreases PNA-T by 0.14 on average; delimiters decrease it by 0.08. Grammar correction PNA-T drops to 0.00–0.01 under paraphrasing and delimiters.",
       "supported": "strong"
     },
     {
       "claim": "No existing detection-based defense is sufficient: they either miss a large fraction of attacks or have high false positive rates.",
       "evidence": "Table 8a shows PPL detection has FNR up to 1.00; windowed PPL detection up to 0.99. Naive LLM-based detection achieves FNR=0.00 but FPR up to 0.93 (Table 8b). Known-answer detection is best but still has FNR up to 0.12 for grammar correction (Table 8a) and misses many attacks in specific scenarios (Table 9).",
       "supported": "strong"
     },
     {
       "claim": "Larger LLMs are more vulnerable to prompt injection attacks, with Pearson correlation of 0.63 between model size and average ASV.",
       "evidence": "Figure 3 shows ASV and MR for each LLM ordered by model size. GPT-4 (1.5T) achieves highest ASV, while smaller models (7B) achieve lower ASV. Pearson correlation between ASV and model size is 0.63, MR and model size is 0.64 (Section 6.2).",
       "supported": "moderate"
     },
     {
       "claim": "Adding in-context learning examples for the target task has a small impact on Combined Attack effectiveness.",
       "evidence": "Figure 4 shows ASV remains similar across 0–5 in-context learning examples for all target/injected task combinations on GPT-4 (Section 6.2).",
       "supported": "moderate"
     },
     {
       "claim": "Combined Attack is consistently effective across different LLMs, target tasks, and injected tasks, with average ASV=0.62 and MR=0.78 across all 10 LLMs and 49 task combinations.",
       "evidence": "Tables 5, 12–20 show detailed results per LLM. Table 6a shows similar ASV across target tasks (0.59–0.64). Table 6b shows variation by injected task (0.34–0.89). Section 6.2 reports the overall averages.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across all tables and figures are single-run point estimates. For open-source LLMs, a single fixed seed is used; for closed-source, temperature=0.1 with no quantification of the 'small impact' of non-determinism. With 100 randomly sampled pairs per evaluation, sampling variance is not characterized."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Hundreds of comparisons are made across attacks, defenses, models, and tasks, but no significance tests are performed. Claims like 'Combined Attack outperforms other attacks' rely solely on comparing raw numbers."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "The authors propose the Combined Attack and implement all baseline attacks and defenses themselves. The Combined Attack, which is their contribution, outperforms all others. They do not acknowledge the potential bias of evaluating their own system against their own implementations of prior work."
     },
     {
       "flag": "Benchmark construct validity not discussed",
       "detail": "ASV measured on controlled NLP tasks (sentiment analysis, spam detection, etc.) may not reflect real-world prompt injection risk in deployed LLM applications with more complex prompts, multi-turn interactions, and diverse data types."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational paper on indirect prompt injection attacks against real-world LLM-integrated applications, directly related to the threat model formalized in this work."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Early work on context ignoring attacks against language models, one of the key attack strategies formalized and benchmarked in this paper."
     },
     {
       "title": "Baseline defenses for adversarial attacks against aligned language models",
       "authors": ["Neel Jain", "Avi Schwarzschild", "Yuxin Wen", "Gowthami Somepalli", "John Kirchenbauer"],
       "year": 2023,
       "relevance": "Proposes paraphrasing and retokenization defenses originally for jailbreaking, extended to prompt injection defense in this benchmark."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen", "Zeming Wei", "Elizabeth Sun", "Basel Alomair", "David Wagner"],
       "year": 2024,
       "relevance": "Proposes fine-tuning a non-instruction-tuned LLM as defense against prompt injection, a concurrent defense approach noted in the related work."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Keegan Hines", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "relevance": "Concurrent work on benchmarking indirect prompt injection attacks and defenses, complementary to this paper's formalization and evaluation framework."
     },
     {
       "title": "Jailbroken: How does llm safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analyzes how LLM safety training fails against jailbreaking attacks, distinct from but related to prompt injection attacks studied here."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "relevance": "Proposes optimization-based adversarial attacks on LLMs that transfer across models, related to the future work on optimization-based prompt injection discussed in this paper."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "relevance": "Proposes structured query defense against prompt injection, a concurrent defense approach noted in the related work."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang", "Tianwei Zhang"],
       "year": 2023,
       "relevance": "Studies prompt injection attacks from the perspective of malicious users against LLM-integrated applications, complementary attack scenario to the one formalized here."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramèr", "Eric Wallace"],
       "year": 2021,
       "relevance": "Demonstrates privacy attacks on LLMs via training data extraction, part of the broader LLM security landscape contextualizing prompt injection."
     },
     {
       "title": "PromptBench: Towards evaluating the robustness of large language models on adversarial prompts",
       "authors": ["Kaijie Zhu", "Jindong Wang", "Jiaheng Zhou", "Zichen Wang"],
       "year": 2023,
       "relevance": "Benchmarks LLM robustness against adversarial prompts (distinct from prompt injection), complementary evaluation framework for LLM security."
     },
     {
       "title": "Evaluating the susceptibility of pre-trained language models via handcrafted adversarial examples",
       "authors": ["Hezekiah J. Branch", "Jonathan Rodriguez Cefalu", "Jeremy McHugh"],
       "year": 2022,
       "relevance": "Early evaluation of language model susceptibility to adversarial inputs, one of the attack approaches formalized in this benchmark."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Releases an open-source benchmark platform (Open-Prompt-Injection) that practitioners can directly use to test their LLM applications against prompt injection attacks and evaluate defenses."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the widely-held belief that prompt injection is a serious threat and existing defenses are insufficient, rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Systematically demonstrates that LLM-integrated applications are vulnerable to prompt injection (OWASP #1 threat) and no existing defense is sufficient, with higher vulnerability in larger/more capable models."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Straightforward security research without controversy or claims that challenge specific companies or products."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository with the benchmark platform is publicly available for cloning and running, though not a pip-installable package or live demo."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Published at USENIX Security (top-tier security venue) and evaluates GPT-4, PaLM 2, Bard, and other well-known models. Authors from Penn State and Duke."
     }
   }
 }