ai-research-survey

scan-v5.json (17071B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "An Early Categorization of Prompt Injection Attacks on Large Language Models",
     "authors": [
       "Sippo Rossi",
       "Alisia Marianne Michel",
       "Raghava Rao Mukkamala",
       "Jason Bennett Thatcher"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.00898",
     "doi": "10.48550/arXiv.2402.00898"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to provide an overview and categorization of prompt injections and discuss implications — all three are delivered in the paper's body with Tables 2/3 and Sections 5.1–5.3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a descriptive taxonomy with no causal claims of the form 'X causes Y'; no study design question arises.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Authors explicitly state the categorization is 'not exhaustive,' that most attacks were demonstrated on only one or two LLM interfaces (mainly ChatGPT/GPT-3), and that generalizing to other interfaces requires 'moderate to significant altering.'",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a taxonomy with no hypothesis testing; there are no empirical findings for which alternative explanations would be relevant.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the existence and structure of attack types; what is measured (documented examples and tests) matches exactly what is claimed (a taxonomy of prompt injection classes).",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 is explicitly titled 'Limitations' and spans a full paragraph with multiple distinct concerns.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Authors identify specific threats: the categorization is incomplete because new attack types emerge continuously; most examples were demonstrated on ChatGPT/GPT-3 only and may not transfer to other interfaces; some injections could not be verified due to rapid patching.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it does not catalog which injections have been patched, that indirect injections were not empirically tested due to ethical concerns, and that the categorization excludes injections backed by only one source.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no funding acknowledgement or disclosure anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated on the first page: Copenhagen Business School and Temple University — both academic institutions with no apparent LLM product connection.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so this criterion is not applicable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines 'prompt injection' by analogy to SQL injection in the introduction, and explicitly defines 'direct' vs. 'indirect' prompt injections, as well as each of the 10 subclasses in Tables 2 and 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is stated explicitly in Section 1: '(1) describe, document, and provide a comprehensive list of known types of prompt injections; (2) provide a checklist for developers and end users.'",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper's literature review (Section 2.2) directly engages with Perez & Ribeiro (2022), Greshake et al. (2023), Kang et al. (2023), Zou et al. (2023), and Shen et al. (2023), showing how this work extends the prior direct/indirect dichotomy into a finer-grained taxonomy.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The paper describes searching Google Scholar, Google, arXiv, GitHub, Medium, and Twitter/X with the keywords 'prompt injection' and 'jailbreak' over two rounds (May–June 2023 and September 2023), and names specific community sites (jailbreakchat.com, Reddit channels).",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "For academic papers: published May 2022–September 2023, in English, discussing prompt injections as adversarial/security threat. For non-academic: must be documented by multiple sources or independently verified by the authors' own tests.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No mention of PRISMA or any other structured review protocol; no flow diagram of paper screening stages is provided.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly states it used 'prompt injection' and 'jailbreak' as search keywords across all databases.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Six databases/platforms are named: Google Scholar, Google, arXiv, GitHub, Medium, and Twitter (X), plus jailbreakchat.com and two Reddit communities.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "The paper reports finding 123 papers in the academic search and then focusing on those discussing prompt injections 'specifically,' but provides no count-by-stage screening table or flow of how 123 papers was reduced to the cited subset.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The authors justify the temporal scope (from May 2022 onward) by noting that is when prompt injections were first discovered and reported; the topic's novelty and reliance on preprints and non-academic sources is also explained.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper presents different attack types as complementary categories and never discusses cases where reviewed sources contradicted each other or offered conflicting empirical results.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality rubric or risk-of-bias assessment is applied to the reviewed papers; the only quality filter for non-academic sources is 'multiple corroborating sources and credible screenshots.'",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is never mentioned; the paper notes most sources are preprints but does not discuss how that skews the evidence base.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "The synthesis is purely narrative and taxonomic; no meta-analysis, vote counting, or effect-size aggregation is attempted.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Developer and end-user recommendations in Sections 5.1–5.2 are directly tied to the identified attack classes (e.g., 'avoid sensitive data in system prompts' follows directly from the documented instruction-manipulation class).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Prompt injections can be divided into two broad branches: direct (6 classes) and indirect (4 classes), with 17 distinct variations identified.",
       "evidence": "Tables 2, 3, and 4 enumerate and define all 17 variations with sources for each.",
       "supported": "moderate"
     },
     {
       "claim": "Direct prompt injections primarily aim to bypass content-filtering safeguards, while indirect injections have broader and more varied cyber-attack-like objectives.",
       "evidence": "Objectives column in Tables 2 and 3 consistently shows 'bypass security measures' for direct and data exfiltration/manipulation goals for indirect.",
       "supported": "strong"
     },
     {
       "claim": "Most documented prompt injection attacks have been demonstrated on ChatGPT, GPT-3, or GPT-4, limiting generalizability to other LLM interfaces.",
       "evidence": "Target column in Table 4 shows ChatGPT or GPT-3/4 for 14 of 17 examples; acknowledged as limitation in Section 5.5.",
       "supported": "strong"
     },
     {
       "claim": "Developing a fully safe LLM interface against prompt injection is 'difficult if not impossible.'",
       "evidence": "Cited as the view of premier AI labs and supported by reference to computational suffix attacks (Zou et al., 2023), but no systematic evidence is presented.",
       "supported": "weak"
     },
     {
       "claim": "Virtual prompt injection can misalign a large share of outputs with a very small number of poisoned training examples.",
       "evidence": "Attributed entirely to Yan et al. (2023) without independent verification in this paper.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "case-study"
   ],
   "key_findings": "The paper proposes an early taxonomy of 17 prompt injection attack types organized into two branches: 6 classes of direct injections (double character, virtualization, obfuscation, payload splitting, adversarial suffix, instruction manipulation) and 4 classes of indirect injections (active, passive, user-driven, virtual). Direct injections primarily bypass content filters while indirect injections enable data exfiltration, misinformation, social engineering, and training-data poisoning. The review is based on a mixed-method literature survey combining 123 academic papers with non-academic sources (Reddit, jailbreakchat.com), with partial empirical verification on ChatGPT and GPT-3. The authors conclude that fully preventing prompt injection is currently infeasible and recommend defensive design principles analogous to SQL-injection-safe database practices.",
   "red_flags": [
     {
       "flag": "No PRISMA or structured protocol",
       "detail": "The survey methodology is described narratively but follows no recognized systematic review protocol, making it difficult to assess selection bias or reproducibility rigorously."
     },
     {
       "flag": "Heavy reliance on non-peer-reviewed sources",
       "detail": "A significant portion of the evidence base is Reddit posts, blog entries, and jailbreakchat.com, with no quality rubric applied to distinguish reliable from unreliable demonstrations."
     },
     {
       "flag": "No screening flow or stage counts",
       "detail": "123 academic papers are found but no documentation is provided on how many were excluded at each stage, making the final included set opaque."
     },
     {
       "flag": "Single-platform generalizability",
       "detail": "14 of 17 catalogued attack examples target ChatGPT/GPT-3/4 only; the claim that categories apply generally to other LLM interfaces is asserted, not demonstrated."
     },
     {
       "flag": "No funding disclosure",
       "detail": "No acknowledgement or funding statement appears anywhere in the paper."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "No risk-of-bias or quality rating is applied to reviewed papers, treating a preprint and a peer-reviewed venue paper as equivalent evidence."
     }
   ],
   "cited_papers": [
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "relevance": "Primary academic source for indirect prompt injection taxonomy; directly foundational to the paper's categorization"
     },
     {
       "title": "Ignore Previous Prompt: Attack Techniques for Language Models",
       "relevance": "First systematic academic treatment of goal hijacking and prompt leaking; foundational reference"
     },
     {
       "title": "Exploiting Programmatic Behavior of LLMs: Dual-Use Through Standard Security Attacks",
       "relevance": "Introduces payload splitting and obfuscation attacks that form two of the six direct injection classes"
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "relevance": "Source for adversarial suffix attacks — one of the six direct injection classes and a key automated attack vector"
     },
     {
       "title": "Do Anything Now: Characterizing and Evaluating In-the-Wild Jailbreak Prompts on Large Language Models",
       "relevance": "Provides community-level categorization of jailbreak prompts used as prior art for the taxonomy"
     },
     {
       "title": "Virtual Prompt Injection for Instruction-Tuned Large Language Models",
       "relevance": "Sole source for the virtual prompt injection class; empirical evidence that few poisoned examples cause large output shifts"
     },
     {
       "title": "Multi-Step Jailbreaking Privacy Attacks on ChatGPT",
       "relevance": "Prior academic work on jailbreaking used to situate the direct injection categories"
     },
     {
       "title": "Evaluating the Instruction-Following Robustness of Large Language Models to Prompt Injection",
       "relevance": "Early benchmark for measuring LLM robustness to prompt injection — cited in future-work discussion on standardized tests"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a developer checklist and end-user guidelines directly derived from the taxonomy, though guidance is high-level."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms and organizes known threats rather than presenting surprising or counter-intuitive findings."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Directly concerns AI safety risks: malware generation, data exfiltration, training-data poisoning — all with real demonstrated examples."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The 'cat-and-mouse' framing and concrete examples (grandma jailbreak, zero-day malware via ChatGPT) carry inherent drama."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Many direct injection examples can be attempted (though patching means success varies); Appendix A lists sources with live prompt examples."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Copenhagen Business School and Temple University — no famous AI lab affiliation, though ChatGPT/GPT-4/Bing AI are named products throughout."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }