ai-research-survey

scan.json (25857B)

---

 {
   "paper": {
     "title": "An Early Categorization of Prompt Injection Attacks on Large Language Models",
     "authors": [
       "Sippo Rossi",
       "Alisia Marianne Michel",
       "Raghava Rao Mukkamala",
       "Jason Bennett Thatcher"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2402.00898",
     "doi": "10.48550/arXiv.2402.00898"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["qualitative", "meta-analysis"],
   "key_findings": "The paper identifies 17 distinct prompt injection attack variations organized into two main branches: 6 classes of direct injections (double character, virtualization, obfuscation, payload splitting, adversarial suffix, instruction manipulation) and 4 classes of indirect injections (active, passive, user-driven, virtual prompt injection). Direct injections primarily aim to bypass content safety filters, while indirect injections have more varied goals resembling traditional cyberattacks. The authors note that even premier AI labs have difficulty fully defending against prompt injections, and that computational approaches to generating new attacks are becoming more mature.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are released. The paper contains no links to any code artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of the collected prompt injections is released. The categorization exists only in the paper's tables and appendix, with external links to sources rather than a downloadable dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper mentions testing on ChatGPT and GPT-3 interfaces but provides no technical setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The methodology section (Section 3) describes the general approach but does not provide step-by-step instructions for reproducing the categorization or the verification tests."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative survey/categorization paper with no quantitative experiments or statistical results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative survey/categorization paper with no comparative quantitative claims."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative survey/categorization paper with no quantitative experiments."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative survey/categorization paper with no quantitative experiments."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative survey/categorization paper with no quantitative experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that Shen et al. (2023) proposed a categorization for jailbreak communities and that Greshake et al. (2023) identified the direct/indirect distinction, but does not systematically compare their categorization against these prior frameworks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No systematic comparison with any prior categorization or survey is provided."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey/categorization paper — no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey/categorization paper — no quantitative evaluation performed."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey/categorization paper — no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey/categorization paper — no predictive model or test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, and 4 provide detailed per-class breakdowns of all 17 prompt injection types across 6 direct and 4 indirect classes, with descriptions, objectives, targets, and sources for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses prompt injections that could not be verified ('we preferred omitting questionable or contested prompt injections') and Section 3 notes that 'due to the rapid patching of these vulnerabilities, they do not always work without additional prompt engineering or in some cases they do not work at all anymore.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that some prompt injections no longer work due to patching, that indirect prompt injections could not be ethically tested, and that some found injections were omitted due to insufficient documentation (Section 3, Section 5.4, Section 5.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) overview of prompt injection threat — supported by Sections 2 and 4; (2) categorization — supported by Tables 2-4 and Section 4; (3) implications discussion — supported by Section 5. All abstract claims are substantiated in the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It is a descriptive categorization/taxonomy paper that catalogs and organizes existing prompt injection types."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broad applicability to 'Large Language Models' but Section 5.5 acknowledges 'most prompt injections have been demonstrated on only one or two LLM interfaces, with ChatGPT and GPT-3 or GPT-4 being by far the most common targets.' The authors assume injections 'could either directly or with moderate to significant altering be applied to other chatbots' without evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative categorization frameworks in depth, alternative explanations for why certain injection classes work, or whether the observed patterns could be artifacts of the sources reviewed (e.g., ChatGPT dominance in sources biasing the taxonomy)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its observations. It claims to categorize known prompt injection types and directly presents that categorization — no proxy gap exists between what was observed and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT', 'GPT-3', 'GPT-4', and 'Bing AI' without specifying exact model versions, snapshot dates, or API versions. No version identifiers like 'gpt-3.5-turbo-0613' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 explicitly states: 'besides the example in Table 1, we refrained from providing the prompt texts for the various prompt injection classes directly.' The actual prompts used for testing are not included in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, etc.) are reported for the ChatGPT or GPT-3 testing. The paper does not describe any API settings used during verification."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper involves manual testing of prompts through chatbot interfaces."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the search and filtering process: keyword searches on Google Scholar, arXiv, GitHub, Medium, Twitter for 'prompt injection' and 'jailbreak'; reviewing jailbreakchat.com and Reddit; academic review from May-September 2023; verification through multiple sources and own testing. Section 2.2 states 123 papers were found via Google Scholar. Filtering criteria are described (must discuss prompt injections specifically, must have multiple sources or be verified by tests)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 is titled 'Limitations' and provides substantive discussion of specific limitations across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 identifies threats specific to this study: (1) categorization is not exhaustive due to rapidly evolving landscape, (2) some prompt injections were omitted due to inability to verify, (3) most injections demonstrated on only ChatGPT/GPT with uncertain generalizability to other LLMs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: indirect prompt injections were not tested for ethical reasons (Section 5.4), temporal scope is May-September 2023, only English-language sources were reviewed (Section 2.2), and the categorization may become obsolete as defenses improve (Section 5.6)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. The collected prompt injection examples, sources reviewed, and verification results are not available as downloadable data. Only summary tables appear in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: keyword searches across multiple platforms (Google, Google Scholar, arXiv, GitHub, Medium, Twitter), specific websites reviewed (jailbreakchat.com, Reddit channels ChatGPTJailbreak and ChatGPT), time period (May-June 2023 initial, September 2023 supplementary), and a verification process requiring multiple sources or own testing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study reviews publicly available sources (academic papers, websites, forums) — no recruitment involved."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Figure 1 shows the categorization process and Section 3 describes the general pipeline, key counts are missing. The paper states 123 academic papers were found and 17 prompt injection types were identified, but does not document how many total prompt injection examples were initially collected, how many were filtered at each stage, or how many were rejected during verification."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or statement appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Copenhagen Business School (Rossi, Michel, Mukkamala) and Temple University (Thatcher). No evaluated product is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed; appears to be unfunded academic research from two universities."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey/categorization paper that does not evaluate a pre-trained model's capability on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey/categorization paper that does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey/categorization paper that does not evaluate a pre-trained model on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a literature and web review of prompt injection attacks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The ethics section (5.4) addresses ethical concerns of publishing prompt injection details, not human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey/categorization paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey/categorization paper with no significant computational work."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a search strategy (Section 3) with keywords and platforms but does not follow PRISMA or any formal review protocol. No flow diagram with counts at each stage, no protocol registration, and the search strategy is not fully reproducible (no exact queries documented)."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the quality of its source papers or non-academic sources using any rubric. Sources are verified through multiple corroboration and own testing, but no quality scoring is applied. All verified sources are treated equally regardless of their methodological rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether its sources (heavily biased toward ChatGPT/GPT, toward successful attacks, and toward attacks that generate attention online) might skew the resulting categorization."
       }
     }
   },
   "claims": [
     {
       "claim": "Seventeen distinct prompt injection attack variations were identified, organized into 6 direct and 4 indirect classes.",
       "evidence": "Section 4 and Tables 2-4 present the full categorization with descriptions, objectives, and sources for each of the 17 examples across 10 classes.",
       "supported": "moderate"
     },
     {
       "claim": "Direct prompt injections primarily aim to bypass security measures in LLM interfaces to produce malicious outputs.",
       "evidence": "Table 2 shows all 6 direct injection classes share the objective of bypassing security measures; Section 4.1 provides descriptions and demonstrations.",
       "supported": "moderate"
     },
     {
       "claim": "Indirect prompt injections have more varied goals resembling traditional cyberattacks, including data exfiltration and misinformation.",
       "evidence": "Table 3 describes 4 indirect injection classes with distinct objectives: stealing data, tricking LLMs into malicious actions, tricking users, and manipulating training data.",
       "supported": "moderate"
     },
     {
       "claim": "Developing a fully safe LLM interface is difficult if not impossible.",
       "evidence": "Section 5.1 argues this based on the observation that even premier AI labs have difficulty blocking prompt injections and that computational approaches (Zou et al., 2023) are maturing. No direct empirical evidence is provided for the impossibility claim.",
       "supported": "weak"
     },
     {
       "claim": "Security measures can be bypassed with relatively simple prompt injections.",
       "evidence": "Section 4.1 states this and Table 1 provides one example of a simple persona-based bypass. However, most prompt texts are not provided and systematic success rates are not reported.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Heavy reliance on non-academic sources",
       "detail": "The categorization draws heavily from Reddit, jailbreakchat.com, blogs, and social media posts — sources without peer review. The paper acknowledges this but the resulting taxonomy may reflect what is popular online rather than what is technically important."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey treats all verified prompt injection sources equally without assessing their methodological quality, reliability, or significance. This launders the signal-to-noise ratio of the underlying sources."
     },
     {
       "flag": "No quantitative analysis of attack effectiveness",
       "detail": "No success rates, no systematic comparison of attack effectiveness across models, no metrics on how many attempts were needed. The categorization is purely qualitative with no quantitative validation."
     },
     {
       "flag": "Actual prompts withheld",
       "detail": "For ethical reasons (Section 5.4), the paper does not provide the actual prompt texts used for testing. While ethically motivated, this makes independent verification of the categorization impossible."
     },
     {
       "flag": "ChatGPT/GPT testing bias",
       "detail": "Most prompt injections were tested only on ChatGPT and GPT-3/4. The categorization may reflect vulnerabilities specific to OpenAI's products rather than universal LLM weaknesses, despite the broad title."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications, including data exfiltration and social engineering scenarios."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["F. Perez", "I. Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Early systematic study of prompt injection attack techniques including goal hijacking and prompt leaking against language models."
     },
     {
       "title": "Exploiting programmatic behavior of LLMs: Dual-use through standard security attacks",
       "authors": ["D. Kang", "X. Li", "I. Stoica", "C. Guestrin", "M. Zaharia", "T. Hashimoto"],
       "year": 2023,
       "arxiv_id": "2302.05733",
       "relevance": "Demonstrates payload splitting and obfuscation attacks that exploit the programmatic behavior of LLMs for dual-use purposes."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "J.Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduces computationally generated adversarial suffixes that transfer across multiple LLMs including ChatGPT, Bard, and Claude."
     },
     {
       "title": "Do anything now: Characterizing and evaluating in-the-wild jailbreak prompts on large language models",
       "authors": ["X. Shen", "Z. Chen", "M. Backes", "Y. Shen", "Y. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03825",
       "relevance": "Large-scale characterization and categorization of real-world jailbreak prompts collected from online communities."
     },
     {
       "title": "Virtual prompt injection for instruction-tuned large language models",
       "authors": ["J. Yan", "V. Yadav", "S. Li", "L. Chen", "Z. Tang", "H. Wang", "V. Srinivasan", "X. Ren", "H. Jin"],
       "year": 2023,
       "arxiv_id": "2307.16888",
       "relevance": "Demonstrates training-time prompt injection where poisoned instruction-tuning data causes misaligned model behavior without runtime prompt manipulation."
     },
     {
       "title": "Multi-step jailbreaking privacy attacks on ChatGPT",
       "authors": ["H. Li", "D. Guo", "W. Fan", "M. Xu", "Y. Song"],
       "year": 2023,
       "arxiv_id": "2304.05197",
       "relevance": "Demonstrates multi-step jailbreaking techniques for extracting private information from ChatGPT."
     },
     {
       "title": "Evaluating the instruction-following robustness of large language models to prompt injection",
       "authors": ["Z. Li", "B. Peng", "P. He", "X. Yan"],
       "year": 2023,
       "arxiv_id": "2308.10819",
       "relevance": "Proposes benchmarks for evaluating LLM robustness against prompt injection attacks, relevant to standardized safety testing."
     },
     {
       "title": "Latent jailbreak: A benchmark for evaluating text safety and output robustness of large language models",
       "authors": ["H. Qiu", "S. Zhang", "A. Li", "H. He", "Z. Lan"],
       "year": 2023,
       "arxiv_id": "2307.08487",
       "relevance": "Proposes a benchmark specifically for evaluating text safety and output robustness of LLMs against jailbreak-style attacks."
     },
     {
       "title": "Use of LLMs for illicit purposes: Threats, prevention measures, and vulnerabilities",
       "authors": ["M. Mozes", "X. He", "B. Kleinberg", "L.D. Griffin"],
       "year": 2023,
       "arxiv_id": "2308.12833",
       "relevance": "Comprehensive review of LLM misuse threats including prompt injection, relevant to understanding the security landscape."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["R. Bommasani", "D.A. Hudson", "E. Adeli"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Foundational position paper on risks and opportunities of large foundation models, providing context for security concerns."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a categorization that developers can use as a checklist for LLM interface security, though no tools or code are released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms widely known concerns about prompt injection vulnerabilities rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Systematically documents AI security vulnerabilities including data exfiltration and training data poisoning scenarios."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Highlights the cat-and-mouse dynamic between attackers and LLM developers but presents no major controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. Actual attack prompts are deliberately withheld for ethical reasons."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Discusses ChatGPT, GPT-4, and Bing AI prominently, but the paper itself is from CBS/Temple University, not a major AI lab."
     }
   }
 }