ai-research-survey

scan.json (27744B)

---

 {
   "paper": {
     "title": "When AI Meets the Web: Prompt Injection Risks in Third-Party AI Chatbot Plugins",
     "authors": [
       "Yigitcan Kaya",
       "Anton Landerer",
       "Stijn Pletinckx",
       "Michelle Zimmermann",
       "Christopher Kruegel",
       "Giovanni Vigna"
     ],
     "year": 2025,
     "venue": "IEEE Symposium on Security and Privacy 2026",
     "arxiv_id": "2511.05797"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["observational", "benchmark-eval"],
   "key_findings": "Large-scale study of 17 third-party chatbot plugins across 10,000+ websites reveals that 8 plugins (used by 8,000 websites) fail to enforce integrity of conversation history in HTTP requests, enabling adversaries to forge system/assistant messages and boost attack success 3-8x. 15 plugins indiscriminately scrape third-party content (e.g., customer reviews) for RAG without distinguishing trusted from untrusted sources, and 13% of sampled e-commerce sites already expose chatbots to this indirect injection risk. Hardened system prompts reduce task hijacking but are ineffective against tool hijacking attacks, which succeed at high rates even with user-role-only injection.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released source code is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset release is mentioned. The collected website lists, extracted system prompts, and experimental results are not made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, dependency list, or setup details are provided beyond mentioning a local WordPress instance and OpenAI API."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The experimental setup is described narratively but lacks step-by-step replication guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3 and 4 report raw counts of successful attacks out of 10 trials. No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'injections via the user role are less effective') based on comparing raw counts without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides contextual effect sizes: '3–8×' improvement from history forging (abstract), '25–100% of cases, compared to just 0–25%' (§1), and percentage breakdowns throughout §7 results."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 repetitions per configuration, 300 websites for system prompt extraction, or 100 e-commerce sites for real-world exposure assessment were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as counts out of 10 trials. No standard deviation, variance, or spread measure is reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "User-role injection serves as the baseline (representing secure plugin behavior) against which system and assistant role injections are compared in Tables 3 and 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Experiments use 11 contemporary commercial LLMs including GPT-4o, GPT-4.1, o4-mini, Claude Sonnet 4, and Gemini 2.5 Pro — all current models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies injection role (system/assistant/user), system prompt design (insecure/hardened/hardened-specific), RAG insertion mode (5 modes), and content wrapping, isolating each factor's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three distinct attack tasks are evaluated: system prompt extraction (SPE), task hijacking (TaH), and tool hijacking (ToH) for direct injection; context hijacking for indirect injection."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "System prompt extraction success was manually labeled by checking for key features (imperative instructions, role assignments, behavioral guidelines). Spot-checks of 200 websites and manual validation of 100 e-commerce sites were performed."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a security evaluation with controlled experiments, not a ML task with train/test splits. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 provide detailed breakdowns per model, per injection role, per system prompt type, and per content insertion mode. Averages by provider are also reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses chatbot deployment failures (§4.2: quota limits, deprecated models), attack failures (refusal to comply), and cases where defenses work (user-role injections blocked by hardened prompts)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative results include: hardened system prompts do not protect against tool hijacking (§7.1), content wrapping has model-dependent effectiveness (§7.2), and ignore+instruct prompts were least effective (Appendix B)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 8 plugins lacking integrity checks (Table 2), 3-8x boost in attack success (Table 3), 13% e-commerce exposure (§5.3), 15 plugins with scraping tools (Table 2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('this oversight amplifies the impact') are justified through controlled experiments that systematically vary single factors (injection role, prompt type) while holding others constant (§7)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds findings to the 17 studied plugins and 11 tested LLMs. It acknowledges the dataset 'may underestimate the full extent' (§3) and notes custom tools 'cannot be evaluated without invasive probing' (§6.2)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§8.3 discusses distribution shift as an alternative explanation for why defenses appear stronger in controlled settings. The paper also considers that model capability vs. robustness may be a trade-off (§7.2 SA mode analysis)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures attack success (whether the LLM produced the adversary-intended response) and claims vulnerability. The measurement matches the claimed outcome with no proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as '4o-mini', '4o', '4.1-mini', '4.1', 'o4-mini', 'haiku-3.5', 'sonnet-3.5', 'sonnet-4', '2.0-flash', '2.5-flash', '2.5-pro' — marketing names without snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Listings 1-11) provides the full text of all injected prompts, trigger messages, system prompt templates, and tool instruction formats used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.5 is stated for all experiments (§7.1, §7.2). Chunking parameters for RAG (600-token chunks, 300-token overlaps, text-embedding-3-large) are specified in §7."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiments test direct LLM API interactions through plugin interfaces."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§3 documents plugin selection criteria (5 exclusion rules), website detection via HTML markers, Common Crawl scanning methodology, and §4.2 describes spot-check validation methodology with counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Limitations are scattered across §5.3 ('our findings likely underestimate'), §3, and the meta-review's §D.4."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: dataset may underestimate ecosystem (§3), 62.5% functionality rate means many chatbots are non-functional (§4.2), 5-query limit per chatbot may miss exposures (§5.3), and the meta-review notes defense robustness against adaptive attacks is unclear (§D.4)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly excludes enterprise solutions (Zendesk, Tidio, Intercom) in §3, states 'we do not aim to develop new prompt injection methods' (§7), and excludes attacks leveraging classical web vulnerabilities (§5 threat model)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (website lists, extracted system prompts, experimental logs) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3 details the data collection: Common Crawl August 2024 snapshot scanning for HTML markers, WHOIS lookups for domain registration dates, Cloudflare API for content categorization, and manual spot-checks of 200 websites."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Website sampling is described: Common Crawl scanning with specific plugin HTML markers, keyword search of WP marketplace yielding 116 results filtered by 5 exclusion criteria to 7 WP plugins, plus search engine/blog post discovery for 10 generic plugins (§3)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: CC snapshot → HTML marker matching → 10,417 unique sites → spot-check validation (200 sites) → automated validation (P1/P2/P4) → functionality rates reported. System prompt extraction: 300 sites → injection → manual labeling → 219 extractions (73% success) (§6.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "§10 discloses funding from NSF (grant 2229876), Department of Homeland Security, IBM, and U.S. Intelligence Community Postdoctoral Fellowship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with UC Santa Barbara. No author affiliations with the evaluated plugins or LLM providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF, DHS, IBM, and IC postdoctoral fellowship are government/academic funders with no financial stake in whether specific chatbot plugins or LLMs are found to be vulnerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests prompt injection attack success rates against LLMs, not model knowledge on benchmarks. Contamination of training data is not relevant to whether an LLM follows injected instructions."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests security vulnerabilities (attack success), not model knowledge that could be contaminated by training data overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark contamination concern exists — the study evaluates attack robustness, not learned knowledge on test problems."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes websites and LLM outputs, not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper does discuss ethical considerations for web scanning (§6.1) but this is not a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs experiments with 11 commercial LLMs across hundreds of configurations but does not report API costs, tokens consumed, or total experimental cost. Only UGCBuster cost (~$0.05/page) is mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for the controlled experiments or the Common Crawl scanning pipeline."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as counts out of 10 runs at temperature 0.5. No seed sensitivity analysis or results across different random seeds are provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'we set the sampling temperature to 0.5 and repeat the experiment 10 times' (§7.1) and 'Each configuration is run 10 times' (§7.2)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "'We generate five variants per goal and select the most effective based on trials with a subset of models' (§7) — but the number of models in the subset and total compute for selection are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states prompt variants were selected based on 'trials with a subset of models' but does not specify which models, how many trials, or show results for non-selected variants."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 11 models, 3 prompt types, 3 roles, and 3 attack tasks without any statistical tests or multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors evaluate third-party commercial LLMs and plugins, not their own system. There is no self-comparison bias to address for the main experiments."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The experiments test attack success rates, not model performance as a function of compute. Compute differences between models are not a confound for security evaluation."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper grounds experiments in real-world observations: attack tasks (SPE, TaH, ToH) are motivated by real incidents (§7.1), system prompts are derived from actual deployments (§6.1), and tool configurations mirror real plugin defaults (§6.2)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. LLMs are called directly through plugin APIs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The study tests attack robustness, not model knowledge. Temporal leakage (model having seen test solutions during training) is not relevant to prompt injection success rates."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not evaluate model capability on a knowledge benchmark where feature leakage would be meaningful."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no train/test split with potential non-independence issues."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark knowledge evaluation where leakage detection would be relevant."
       }
     }
   },
   "claims": [
     {
       "claim": "8 of 17 plugins (used by 8,000 websites) fail to enforce integrity of conversation history in HTTP requests, enabling adversaries to forge messages in elevated roles.",
       "evidence": "Table 2 documents which plugins allow injection into system and assistant roles. §5.2 details the vulnerability mechanism with examples in Figure 3.",
       "supported": "strong"
     },
     {
       "claim": "Direct prompt injection into non-user roles boosts attack success by 3-8x compared to user-role-only injection.",
       "evidence": "Table 3 shows system/assistant role injections achieve ~60%/~30% SPE success vs ~3% for user role; TaH achieves ~98%/~59% vs ~10% for user role under insecure prompts.",
       "supported": "strong"
     },
     {
       "claim": "~13% of e-commerce websites have already exposed their chatbots to third-party content via RAG ingestion of customer reviews.",
       "evidence": "§5.3: Manual analysis of 100 randomly selected e-commerce sites using P1; 41 allow reviews, 36 display them, 13 chatbots incorporated review content in responses.",
       "supported": "moderate"
     },
     {
       "claim": "Hardened system prompts effectively block task hijacking but offer limited protection against tool hijacking attacks.",
       "evidence": "Table 3: TaH drops from ~98% to ~24% (system role) with hardened-specific prompts, but ToH increases in some configurations. Tool hijacking succeeds at 20-100% even via user role for Gemini models.",
       "supported": "strong"
     },
     {
       "claim": "The chatbot plugin ecosystem grew nearly 50% in 2025 alone, from 10,417 sites (Aug 2024) to 17,474 sites (Apr 2025).",
       "evidence": "Table 1 and Figure 2 show longitudinal growth across Common Crawl snapshots from January 2023 to April 2025.",
       "supported": "strong"
     },
     {
       "claim": "Gemini models are the least robust against prompt injection attacks across all tested scenarios.",
       "evidence": "Tables 3 and 4: Gemini models show highest attack success rates, with user-role injections succeeding consistently and indirect attacks via tool role reaching up to 100% success.",
       "supported": "strong"
     },
     {
       "claim": "Tool instruction hardening reduces tool hijacking attack success rates by 40-75%.",
       "evidence": "§8.2 states this result from repeating §7 experiments with LLM-hardened tool instructions, but detailed results are not shown in the paper.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No artifacts released",
       "detail": "Despite analyzing 10,000+ websites and running extensive controlled experiments, no code, data, website lists, or experimental logs are released. This limits reproducibility and independent verification."
     },
     {
       "flag": "No statistical tests",
       "detail": "All comparisons are based on raw counts out of 10 trials without confidence intervals, significance tests, or variance measures. With only 10 repetitions per configuration, observed differences could be due to sampling noise."
     },
     {
       "flag": "Prompt selection on test models",
       "detail": "Attack prompts were selected based on effectiveness 'with a subset of models' (§7), introducing potential selection bias. The subset of models and non-selected prompt variants are not reported."
     },
     {
       "flag": "Plugin anonymization limits verification",
       "detail": "Plugins are anonymized as P1-P17, preventing independent verification of the claimed vulnerabilities. While reasonable for responsible disclosure, it reduces transparency."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Foundational work on indirect prompt injection in LLM-integrated applications, directly relevant to the threat model studied."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Y. Liu", "Y. Jia", "R. Geng", "J. Jia", "N. Z. Gong"],
       "year": 2024,
       "relevance": "Formalizes prompt injection attacks and evaluates defenses, providing the benchmark framework this paper builds upon."
     },
     {
       "title": "The instruction hierarchy: Training LLMs to prioritize privileged instructions",
       "authors": ["E. Wallace", "K. Xiao", "R. Leike", "L. Weng", "J. Heidecke", "A. Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Describes the instruction hierarchy defense that plugins undermine by allowing injection into privileged roles."
     },
     {
       "title": "Struq: Defending against prompt injection with structured queries",
       "authors": ["S. Chen", "J. Piet", "C. Sitawarin", "D. Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Proposes structured query defense against prompt injection; content wrapping evaluated in this paper is related."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": ["Y. Liu", "Y. Jia", "J. Jia", "D. Song", "N. Z. Gong"],
       "year": 2025,
       "relevance": "Game-theoretic prompt injection detection tested as alternative prompt type in this paper's experiments (Appendix B)."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["E. Debenedetti", "J. Zhang", "M. Balunovic", "L. Beurer-Kellner", "M. Fischer", "F. Tramèr"],
       "year": 2024,
       "relevance": "Dynamic benchmark for evaluating prompt injection in agentic settings; complementary to this paper's real-world plugin focus."
     },
     {
       "title": "PoisonedRAG: Knowledge corruption attacks to retrieval-augmented generation of large language models",
       "authors": ["W. Zou", "R. Geng", "B. Wang", "J. Jia"],
       "year": 2024,
       "arxiv_id": "2402.07867",
       "relevance": "Demonstrates RAG poisoning attacks that this paper shows are practically viable through plugin scraping vulnerabilities."
     },
     {
       "title": "A new era in LLM security: Exploring security concerns in real-world LLM-based systems",
       "authors": ["F. Wu", "N. Zhang", "S. Jha", "P. McDaniel", "C. Xiao"],
       "year": 2024,
       "arxiv_id": "2402.18649",
       "relevance": "Surveys security concerns in real-world LLM systems, providing context for the plugin vulnerability ecosystem."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["J. Piet", "M. Alrashed", "C. Sitawarin", "S. Chen", "Z. Wei", "E. Sun", "B. Alomair", "D. Wagner"],
       "year": 2024,
       "relevance": "Proposes task-specific finetuning as prompt injection defense; represents the class of advanced defenses the paper argues plugins won't adopt."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper; RAG is the primary mechanism through which plugins ingest external content, creating indirect injection risk."
     },
     {
       "title": "Ignore this title and HackAPrompt: Exposing systemic vulnerabilities of LLMs through a global prompt hacking competition",
       "authors": ["S. V. Schulhoff"],
       "year": 2023,
       "relevance": "Global competition exposing LLM prompt vulnerabilities; informs the attack strategy design in this paper."
     },
     {
       "title": "A closer look at system prompt robustness",
       "authors": ["N. Mu", "J. Lu", "M. Lavery", "D. Wagner"],
       "year": 2025,
       "arxiv_id": "2502.12197",
       "relevance": "Analyzes system prompt robustness with realistic prompt datasets; complements this paper's real-world system prompt characterization."
     }
   ]
 }