ai-research-survey

scan-v4.json (16247B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "An Early Categorization of Prompt Injection Attacks on Large Language Models",
     "authors": [
       "Sippo Rossi",
       "Alisia Marianne Michel",
       "Raghava Rao Mukkamala",
       "Jason Bennett Thatcher"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.00898",
     "doi": "10.48550/arXiv.2402.00898"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) overview of prompt injection threat — supported by Sections 2 and 4; (2) categorization — supported by Tables 2-4 and Section 4; (3) implications discussion — supported by Section 5. All abstract claims are substantiated in the paper.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It is a descriptive categorization/taxonomy paper that catalogs and organizes existing prompt injection types.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broad applicability to 'Large Language Models' but Section 5.5 acknowledges 'most prompt injections have been demonstrated on only one or two LLM interfaces, with ChatGPT and GPT-3 or GPT-4 being by far the most common targets.' The authors assume injections 'could either directly or with moderate to significant altering be applied to other chatbots' without evidence.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative categorization frameworks in depth, alternative explanations for why certain injection classes work, or whether the observed patterns could be artifacts of the sources reviewed (e.g., ChatGPT dominance in sources biasing the taxonomy).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its observations. It claims to categorize known prompt injection types and directly presents that categorization — no proxy gap exists between what was observed and what is claimed.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 is titled 'Limitations' and provides substantive discussion of specific limitations across multiple paragraphs.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 identifies threats specific to this study: (1) categorization is not exhaustive due to rapidly evolving landscape, (2) some prompt injections were omitted due to inability to verify, (3) most injections demonstrated on only ChatGPT/GPT with uncertain generalizability to other LLMs.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: indirect prompt injections were not tested for ethical reasons (Section 5.4), temporal scope is May-September 2023, only English-language sources were reviewed (Section 2.2), and the categorization may become obsolete as defenses improve (Section 5.6).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or statement appears in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Copenhagen Business School (Rossi, Michel, Mukkamala) and Temple University (Thatcher). No evaluated product is affiliated with these institutions.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed; appears to be unfunded academic research from two universities.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Prompt injection is defined by analogy to SQL injection in the introduction; direct vs. indirect injections are defined; LLMs are distinguished from AI chatbots in Section 2.1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is stated explicitly as two-fold: a comprehensive documented list of known prompt injection types, and a categorization checklist for developers and end users.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 reviews prior academic work including Greshake et al. (2023), Perez and Ribeiro (2022), Shen et al. (2023), and Zou et al. (2023), building the taxonomy directly on these foundations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "The methodology mentions searching Google, Google Scholar, arXiv, Github, Medium, and Twitter but provides no Boolean queries, result counts, or exact search strings sufficient for replication.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": false,
           "justification": "Academic inclusion criteria are stated (English, post-May 2022, adversarial framing) but not applied in a documented fashion; non-academic inclusion relies on ad hoc multi-source corroboration without formal criteria.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA flowchart or other structured review protocol is used; the methodology is described narratively without formal screening stages or counts.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly states that searches used 'prompt injection' and 'jailbreak' as keywords across all listed databases.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Six sources are explicitly named: Google, Google Scholar, arXiv, Github, Medium, and Twitter (now X).",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 123 academic papers were found but provides no breakdown of how many were excluded at each stage, no counts for non-academic sources screened, and no screening flowchart.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The temporal scope (May 2022 onward) is justified by the emergence of prompt injections at that time; inclusion of non-academic sources is justified by the pre-publication and rapidly evolving nature of the field.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "No conflicting findings across reviewed sources are discussed; the paper presents a single unified taxonomy without noting where sources disagree on categorization or definitions.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No formal quality rubric is applied; non-academic sources are included if backed by 'multiple sources with credible screenshots', which is an informal and inconsistently documented criterion.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is not discussed; the authors note that most literature is preprints rather than peer-reviewed but do not address how the direction or completeness of findings might be skewed.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "No quantitative synthesis is performed; the paper produces a descriptive taxonomy of 17 attack types with no counts of prevalence, attack success rates, or effect aggregation.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Recommendations in Sections 5.1-5.2 are directly tied to specific documented attack classes — e.g., recommending caution with copied prompts is grounded in the documented user-driven injection class.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Prompt injection attacks can be organized into two branches (direct/indirect) with 10 classes covering 17 documented variants.",
       "evidence": "Tables 2-3 and Appendix A document each class and variant with sources; the taxonomy is the core deliverable.",
       "supported": "moderate"
     },
     {
       "claim": "Developing a fully safe LLM interface is difficult if not impossible.",
       "evidence": "Asserted in Section 5.1 based on premier AI labs failing to prevent attacks; no systematic evidence or success rate data is provided.",
       "supported": "weak"
     },
     {
       "claim": "Indirect prompt injections can exfiltrate private user data via browser plugins and email clients.",
       "evidence": "Cited to Burgess (2023) and demonstrated via Greshake et al. (2023) examples; evidence is secondary and journalistic rather than peer-reviewed.",
       "supported": "moderate"
     },
     {
       "claim": "Virtual prompt injection can misalign a large share of LLM outputs with very few poisoned training examples.",
       "evidence": "Attributed to Yan et al. (2023) who demonstrated the attack empirically; the claim is borrowed from cited work.",
       "supported": "moderate"
     },
     {
       "claim": "Simple prompt injections can bypass LLM safety measures to produce malicious content including malware.",
       "evidence": "Illustrated by the grandmother example in Table 1 and Mulgrew (2023) blog post; verification limited to ChatGPT and GPT-3.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "case-study"
   ],
   "key_findings": "The paper taxonomizes prompt injection attacks into two branches — direct (6 classes: double character, virtualization, obfuscation, payload splitting, adversarial suffix, instruction manipulation) and indirect (4 classes: active, passive, user-driven, virtual prompt injection) — documenting 17 specific verified examples. The review draws heavily on non-peer-reviewed sources due to the topic's novelty at time of writing. The authors conclude that LLM interfaces cannot currently be made fully secure and call for standardized vulnerability test suites and developer best practices analogous to SQL injection defense patterns.",
   "red_flags": [
     {
       "flag": "Non-peer-reviewed sources dominate",
       "detail": "The survey heavily relies on blogs, tweets, Reddit posts, and jailbreakchat.com with informal multi-source corroboration as the quality check, with no formal rubric applied."
     },
     {
       "flag": "No PRISMA or structured protocol",
       "detail": "No formal screening protocol is used; 123 academic papers are mentioned but no exclusion counts or screening stages are documented."
     },
     {
       "flag": "No competing taxonomy comparison",
       "detail": "Shen et al. (2023) already proposed a jailbreak categorization; this paper notes it but does not compare or validate its own scheme against the existing one."
     },
     {
       "flag": "No funding or competing interests disclosure",
       "detail": "Neither a funding acknowledgment nor a competing interests statement appears anywhere in the paper."
     },
     {
       "flag": "No quantitative synthesis",
       "detail": "Despite surveying 123 papers, no quantitative aggregation of attack prevalence, success rates, or defense effectiveness is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "relevance": "Core reference establishing the direct/indirect injection taxonomy that this paper extends"
     },
     {
       "title": "Ignore Previous Prompt: Attack Techniques for Language Models",
       "relevance": "Foundational paper identifying goal hijacking and prompt leaking attack classes"
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "relevance": "Key reference for the adversarial suffix attack class (computational bypass of alignment)"
     },
     {
       "title": "Virtual Prompt Injection for Instruction-Tuned Large Language Models",
       "relevance": "Core reference for the virtual prompt injection class via training data poisoning"
     },
     {
       "title": "Exploiting Programmatic Behavior of LLMs: Dual-use through standard security attacks",
       "relevance": "Reference for payload splitting and obfuscation attack classes"
     },
     {
       "title": "Do Anything Now: Characterizing and Evaluating In-the-Wild Jailbreak Prompts on Large Language Models",
       "relevance": "Prior competing taxonomy for jailbreak communities that this paper builds on"
     },
     {
       "title": "Evaluating the Instruction-Following Robustness of Large Language Models to Prompt Injection",
       "relevance": "Benchmark approach for evaluating LLM robustness to prompt injection attacks"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a categorization that developers can use as a checklist for LLM interface security, though no tools or code are released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms widely known concerns about prompt injection vulnerabilities rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Systematically documents AI security vulnerabilities including data exfiltration and training data poisoning scenarios."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Highlights the cat-and-mouse dynamic between attackers and LLM developers but presents no major controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. Actual attack prompts are deliberately withheld for ethical reasons."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Discusses ChatGPT, GPT-4, and Bing AI prominently, but the paper itself is from CBS/Temple University, not a major AI lab."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }