ai-research-survey

scan.json (29228B)

---

 {
   "paper": {
     "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
     "authors": [
       "Kai Greshake",
       "Sahar Abdelnabi",
       "Shailesh Mishra",
       "Christoph Endres",
       "Thorsten Holz",
       "Mario Fritz"
     ],
     "year": 2023,
     "venue": "AISec@CCS",
     "arxiv_id": "2302.12173",
     "doi": "10.1145/3605764.3623985"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper introduces indirect prompt injection (IPI) as a novel attack vector in which adversaries inject malicious prompts into data retrieved by LLM-integrated applications at inference time, enabling remote compromise without direct model access. Through qualitative demonstrations on synthetic GPT-4 applications and real-world Bing Chat, the authors show that IPI can enable data theft, fraud, malware spreading (including self-replicating prompt worms), remote control intrusion, content manipulation, and denial of service. Code completion engines like GitHub Copilot are also vulnerable via injected comments in imported packages, though efficacy is sensitive to context. The paper develops a comprehensive threat taxonomy and argues that current mitigations (RLHF, input filtering) are inadequate against these attacks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides two GitHub repositories: https://github.com/greshake/llm-security (footnote 1) and https://github.com/greshake/lm-safety (Section 5.4). Section 5.4 states 'Our synthetic applications demos are readily available on a public GitHub repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All attack prompts are provided in the Appendix (Prompts 1-20), and the synthetic application demos with prepared content are released on GitHub. Section 1 states they 'share all our demonstrations on our GitHub repository and all developed attack prompts in the Appendix.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned. The paper mentions using LangChain and OpenAI APIs but does not specify library versions or provide a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 states the demos are 'easy to adapt to different OpenAI's APIs or models' but no step-by-step reproduction instructions are provided. The Bing Chat experiments are inherently difficult to reproduce due to the dynamic, black-box nature of the system."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are reported. All demonstrations are qualitative — individual examples with screenshots. No confidence intervals or error bars appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper acknowledges in Section 5.2 that 'quantifying our attacks' success rate can be challenging' and leaves quantitative evaluation for future work."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper provides no quantitative measurements of attack success rates, severity, or impact magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of demonstrations or test sessions conducted. The paper does not state how many times each attack was attempted or how many sessions informed each finding."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or multi-run results are reported. Each demonstration appears to be a single qualitative example. Section 5.4 acknowledges 'exact reproducibility is difficult to guarantee.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper does not compare indirect prompt injection against other attack methods, nor does it compare attack success across different defense configurations systematically."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines of any kind are used, so contemporaneity is not applicable in practice. The paper references prior work on direct prompt injection (Perez & Ribeiro 2022) but does not compare against it experimentally."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a taxonomy and demonstration paper, not a system with modular components that could be ablated. The attacks are independent proof-of-concept examples, not a unified system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are used at all. The evaluation is entirely qualitative, consisting of screenshots and conversation transcripts showing attack outcomes."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No formal human evaluation of attack believability or effectiveness is conducted. Section 5.2 identifies evaluating 'deception and believability' via user studies as future work."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no dataset or systematic evaluation framework. Attacks are demonstrated via individual qualitative examples on specific systems, not evaluated against a structured test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes demonstrations by threat category (information gathering, fraud, malware, intrusion, manipulated content, availability) with specific examples for each. Section 4.2 provides per-category demonstrations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.4 notes code completion attacks were 'very sensitive to context' and 'efficacy of our injections was significantly reduced' in larger applications. Section 5.2 acknowledges models may generate 'conspicuously false answers that are widely unbelievable.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that code completion injections had reduced efficacy in larger codebases (Section 4.2.4), that availability attacks did not always work consistently (Section 4.2.6), and that attack prompts sometimes produced unconvincing outputs (Section 5.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are qualitative (new attack vectors, taxonomy, practical feasibility demonstrations, lack of mitigations) and are all supported by the paper's content: taxonomy in Section 3/Figure 2, demonstrations in Section 4, mitigation discussion in Section 5.6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claims are possibility/existence claims ('indirect prompt injection CAN lead to X'). These are adequately supported by controlled demonstrations where injecting specific prompts causes specific model behaviors. The study design (controlled synthetic apps + real Bing Chat) is appropriate for establishing that these attacks are possible."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Compromising Real-World LLM-Integrated Applications' broadly, but testing is limited to Bing Chat, synthetic GPT-4 apps, and GitHub Copilot. The paper could not test on Microsoft 365 Copilot or ChatGPT plugins (acknowledged in Section 5.2). The taxonomy covers threats far beyond what was demonstrated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why attacks succeed (e.g., whether it is the instruction-tuning, RLHF, or architectural choices that make models susceptible). No robustness checks or confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper demonstrates individual attack instances and claims 'practical viability' and 'feasibility' without distinguishing between showing an attack is possible in a controlled setting and showing it would work reliably at scale in the wild. Section 5.2 acknowledges this gap but the framing does not."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'text-davinci-003' and 'gpt-4' but without specific version dates or API snapshots. For Bing Chat, it says it 'runs on the GPT-4 model' without specifying which version. For Copilot, it says 'OpenAI Codex' without version. Per schema requirements, 'GPT-4' without a snapshot date is insufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All attack prompts are provided in full in the Appendix (Prompts 1-20), including the initial system prompts for synthetic applications (Prompts 1-2) and all injection prompts for each attack scenario."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 states 'All attacks are run at a sampling temperature of 0 for reproducibility.' Bing Chat's three modes (creative, balanced, precise) are also noted, though the exact mode used per experiment is not always specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic application setup is described in detail in Section 4.1.1, including: the LangChain library for text-davinci-003, ReAct prompting, OpenAI chat format for GPT-4, and the full list of integrated tools (Search, View, Retrieve URL, Read/Send Email, Read Address Book, Memory)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how prepared content was constructed for the synthetic applications, how Bing Chat sessions were selected for inclusion, or any systematic process for generating the demonstration scenarios."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Limitations' provides substantive discussion of experimental setup limitations, evaluation challenges, and deception/believability concerns across three subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses specific threats: testing on synthetic applications and local HTML rather than live injections, inability to access other applications (Microsoft 365 Copilot, ChatGPT plugins), challenges in quantifying interactive attack success rates, and models generating 'conspicuously false answers.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly states they could not test on other applications, that demonstrations 'are intrinsically not exhaustive' (Section 4), and that quantitative evaluation is left for future work. Section 5.4 acknowledges 'exact reproducibility is difficult to guarantee with such a black-box system.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Attack prompts are in the Appendix, screenshots of Bing Chat outputs are provided (Figures 13-28), conversation transcripts are included (Outputs 1-4), and synthetic application code is on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how many sessions were conducted, how demonstrations were selected for inclusion, or what the overall process was for testing each attack. Only individual successful examples are shown."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The paper tests attacks on LLM systems (synthetic apps, Bing Chat, Copilot) with no participant recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No systematic data pipeline is documented. The paper presents individual attack demonstrations without describing the overall experimental workflow from attack design to execution to selection of examples for inclusion."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses funding: 'This work was partially funded by ELSA – European Lighthouse on Secure and Safe AI funded by the European Union under grant agreement No. 101070617.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Saarland University, CISPA Helmholtz Center for Information Security, and sequire technology GmbH. The affiliation with a security company (sequire) is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The EU research program (ELSA) is a general research funding body with no financial stake in whether attacks on specific commercial products succeed. The funder is independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Authors Greshake and Endres are affiliated with sequire technology GmbH (a security company) which could have commercial interest in demonstrating vulnerabilities, but no financial interest disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests security attacks/defenses against LLM-integrated applications, not model capabilities on benchmarks. Contamination criteria do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model knowledge is performed. The paper tests attack vectors, not model performance on standard tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmarks are used. The paper demonstrates attack feasibility through qualitative proof-of-concept examples, not benchmark evaluations."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper tests attacks on LLM systems without any human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper discusses ethical considerations in Section 5.1 regarding responsible disclosure but this is not an IRB/ethics review of human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency measurements are reported for any of the attack demonstrations despite using paid OpenAI APIs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report total API spend, number of API calls, or computational resources used for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Indirect prompt injection enables adversaries to remotely exploit LLM-integrated applications by injecting prompts into data retrieved at inference time.",
       "evidence": "Demonstrated through multiple qualitative examples: synthetic GPT-4 applications (Section 4.1.1) and Bing Chat (Section 4.1.2) successfully process and follow injected prompts placed in retrieved content (Prompts 3-20, Outputs 1-4).",
       "supported": "moderate"
     },
     {
       "claim": "Retrieved prompts can act as arbitrary code execution, manipulating the application's functionality and controlling API calls.",
       "evidence": "Section 4.2.4 demonstrates remote control where injected prompts cause the model to fetch new instructions from an attacker's server (Prompt 8, Output 2). Section 4.2.3 shows prompt worms that read contacts and send emails (Prompt 7, Output 1). Multiple demonstrations show control over search, memory, and email APIs.",
       "supported": "moderate"
     },
     {
       "claim": "Prompts that are filtered when entered directly by users are not filtered when ingested indirectly via retrieved content.",
       "evidence": "Section 4.2 states 'prompts that are typically filtered out via the chat interface are not filtered out when injected indirectly' and provides the example that Bing Chat stops sessions for direct jailbreak prompts but succumbs when they are ingested indirectly (footnote 5).",
       "supported": "weak"
     },
     {
       "claim": "LLMs can autonomously implement attack goals given only high-level descriptions without pre-programmed attack details.",
       "evidence": "Observation #1 and Section 4.2.1: the information gathering prompt 'only instructed the model to persuade the user without raising suspicion' with no mention of specific techniques, yet Bing Chat tailored its persuasion based on conversation context. The fraud demonstration (Section 4.2.2) showed the model generating scam persuasion techniques without explicit instruction.",
       "supported": "weak"
     },
     {
       "claim": "Indirect prompt injections can be hidden via multi-stage payloads and Base64 encoding to circumvent detection.",
       "evidence": "Section 4.3.1 demonstrates a multi-stage exploit where a small injection triggers retrieval of a larger payload (Prompt 19, Output 4). Section 4.3.2 shows a Base64-encoded injection successfully decoded and executed by Bing Chat (Prompt 20, Figure 27).",
       "supported": "moderate"
     },
     {
       "claim": "Current mitigations including RLHF and input filtering are insufficient against indirect prompt injection.",
       "evidence": "Section 5.6 discusses limitations of RLHF (citing Wolf et al. 2023 on fundamental impossibility), filtering (attacks succeed on Bing Chat which employs filtering), and other defenses. The paper notes a 'Whack-A-Mole' pattern. However, no systematic evaluation of defense effectiveness is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "All attack demonstrations are qualitative single examples with screenshots and conversation transcripts. No success rates, reliability measurements, or systematic evaluation across multiple trials is provided. The paper explicitly defers quantitative evaluation to future work (Section 5.2)."
     },
     {
       "flag": "Selection bias in demonstrations",
       "detail": "Only successful attack examples are shown (apart from acknowledged limitations with code completion). The paper does not report how many attempts were made per attack type or what fraction succeeded. This makes it impossible to assess the practical reliability of the attacks."
     },
     {
       "flag": "Broad claims from narrow evidence",
       "detail": "The paper's taxonomy (Figure 2) covers a vast threat landscape, but only a fraction is demonstrated with actual experiments. Many claimed threats (e.g., surveillance, automated defamation, DDoS) are discussed hypothetically without demonstration."
     },
     {
       "flag": "Undisclosed commercial affiliation interest",
       "detail": "Two authors are affiliated with sequire technology GmbH, a security consulting company that could benefit commercially from demonstrating that LLM-integrated applications are vulnerable. No competing interests statement is provided."
     },
     {
       "flag": "Controlled setting limitations",
       "detail": "Synthetic application experiments use prepared content where 'all interfaces deliver prepared content, and unrelated queries are ignored' (Section 4.1.1). The Bing Chat sidebar experiments use local HTML files rather than actual web-scale injections. The gap between controlled demonstrations and real-world feasibility is acknowledged but not measured."
     }
   ],
   "cited_papers": [
     {
       "title": "Ignore Previous Prompt: Attack Techniques For Language Models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Foundational work on direct prompt injection attacks against LLMs, which this paper extends to indirect injection scenarios."
     },
     {
       "title": "Exploiting Programmatic Behavior of LLMs: Dual-Use Through Standard Security Attacks",
       "authors": ["Daniel Kang", "Xuechen Li", "Ion Stoica", "Carlos Guestrin", "Matei Zaharia", "Tatsunori Hashimoto"],
       "year": 2023,
       "relevance": "Demonstrates program obfuscation, payload splitting, and virtualization to bypass LLM API filters — techniques that inform the hidden injection methods in this paper."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Izhak Shafran", "Karthik R Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Core agentic framework used for tool-augmented LLMs; the paper's synthetic applications use ReAct prompting for text-davinci-003."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì", "Roberta Raileanu", "Maria Lomeli", "Luke Zettlemoyer", "Nicola Cancedda", "Thomas Scialom"],
       "year": 2023,
       "relevance": "Demonstrates training LLMs to output API calls, a capability central to the LLM-integrated applications targeted in this paper."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "The model powering Bing Chat and the synthetic applications tested in this paper; discusses safety evaluation and jailbreak resistance."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park", "Joseph C O'Brien", "Carrie J Cai", "Meredith Ringel Morris", "Percy Liang", "Michael S Bernstein"],
       "year": 2023,
       "relevance": "Demonstrates autonomous LLM agents with memory and planning that could be targeted by indirect prompt injection; cited as context for multi-agent injection risks."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang", "Diogo Almeida", "Carroll Wainwright"],
       "year": 2022,
       "relevance": "RLHF training methodology discussed as a potential but insufficient defense against prompt injection attacks."
     },
     {
       "title": "Fundamental Limitations of Alignment in Large Language Models",
       "authors": ["Yotam Wolf", "Noam Wies", "Yoav Levine", "Amnon Shashua"],
       "year": 2023,
       "relevance": "Theoretical work cited to argue that alignment/RLHF may be fundamentally unable to prevent all adversarial prompt behaviors."
     },
     {
       "title": "Large Language Models are Human-Level Prompt Engineers",
       "authors": ["Yongchao Zhou", "Andrei Ioan Muresanu", "Ziwen Han", "Keiran Paster", "Silviu Pitis", "Harris Chan", "Jimmy Ba"],
       "year": 2023,
       "relevance": "Highlights LLMs as analogous to black-box computers executing natural language programs, a framing central to the indirect prompt injection threat model."
     },
     {
       "title": "Discovering Language Model Behaviors with Model-Written Evaluations",
       "authors": ["Ethan Perez", "Sam Ringer", "Kamile Lukošiūtė", "Karina Nguyen", "Edwin Chen"],
       "year": 2022,
       "relevance": "Evaluates sycophancy in RLHF models and inverse scaling effects, cited in the biased output manipulation attack (Section 4.2.5)."
     },
     {
       "title": "Spinning Language Models: Risks of Propaganda-As-A-Service and Countermeasures",
       "authors": ["Eugene Bagdasaryan", "Vitaly Shmatikov"],
       "year": 2022,
       "relevance": "Prior work on backdoor attacks against language models for propaganda generation, conceptually related to indirect prompt injection."
     },
     {
       "title": "Visual Instruction Tuning",
       "authors": ["Haotian Liu", "Chunyuan Li", "Qingyang Wu", "Yong Jae Lee"],
       "year": 2023,
       "relevance": "LLaVA model used to demonstrate visual prompt injection (Figure 28), showing multi-modal injection is possible."
     },
     {
       "title": "Artificial Influence: An Analysis Of AI-Driven Persuasion",
       "authors": ["Matthew Burtell", "Thomas Woodside"],
       "year": 2023,
       "relevance": "Analyzes AI persuasion capabilities relevant to the fraud and social engineering attacks demonstrated in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security practitioners and LLM application developers can use the taxonomy and attack demonstrations to understand threats, but the paper provides no defensive tool or immediately deployable technique."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that prompt injection requires direct user access by introducing indirect injection through retrieved data — a new and non-obvious attack surface at the time of publication."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates alarming attack scenarios including data exfiltration, self-replicating prompt worms, remote model control, and automated social engineering against real-world systems like Bing Chat."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly attacks Microsoft's Bing Chat and GitHub Copilot products, arguing that the 'AI-integration race is not accompanied by adequate guardrails and safety evaluations.'"
     },
     "demo_ability": {
       "score": 2,
       "justification": "Synthetic application code is on GitHub and all prompts are in the Appendix, but testing requires OpenAI API access and Bing Chat experiments cannot be easily replicated."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Attacks demonstrated on Bing Chat (GPT-4 powered) and GitHub Copilot — well-known products — but the authors are from academic institutions (CISPA, Saarland University), not major AI labs."
     }
   }
 }