ai-research-survey

scan.json (22689B)

---

 {
   "paper": {
     "title": "Prompt Injection attack against LLM-integrated Applications",
     "authors": [
       "Yi Liu",
       "Gelei Deng",
       "Yuekang Li",
       "Kailong Wang",
       "Zihao Wang",
       "Xiaofeng Wang",
       "Tianwei Zhang",
       "Yepang Liu",
       "Haoyu Wang",
       "Yan Zheng",
       "Leo Yu Zhang",
       "Yang Liu"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2306.05499"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "HOUYI, a black-box prompt injection attack technique inspired by SQL injection and XSS, achieves an 86.1% success rate across 36 real-world LLM-integrated applications, identifying 31 as vulnerable. The attack uses three components (framework, separator, disruptor) with iterative refinement. Ten vendors including Notion confirmed the vulnerabilities. Existing defenses (instruction defense, sandwich defense, XML tagging, etc.) are shown to be insufficient against HOUYI's generated payloads.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions implementing HOUYI in Python (2,150 lines) but provides no repository URL or code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of prompts, attack payloads, or application responses is released. Application names are anonymized (Table 5)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements.txt, or dependency details are provided beyond mentioning Python and GPT-3.5-turbo."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes methodology at a high level but does not provide steps to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as success counts out of 5 attempts (e.g., '5/5', '3/5') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of effectiveness are based on raw success/failure counts across 5 attempts per scenario."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the overall success rate (86.1%, 31/36 applications) and per-application success rates across exploit scenarios (Table 4), providing baseline context (prior attacks succeed on 2/10)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 36 applications were chosen, why 5 attempts per scenario, or whether this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation across runs is reported. Each attack is run 5 times but only raw counts are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 (pilot study) tests three existing attack strategies (direct injection, escape characters, context ignoring) as baselines on 10 applications, achieving only partial success on 2/10."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary prompt injection techniques from Perez & Ribeiro (2022), Apruzzese et al. (2023), and Greshake et al. (2023), which were state-of-the-art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 presents an ablation study on the separator component with three variants (HOUYI-SYNTAX-ONLY, HOUYI-LANGUAGE-ONLY, HOUYI-SEMANTIC-ONLY), shown in Figure 5."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five exploit scenarios are tested (prompt leaking, code generation, content manipulation, spam generation, information gathering), each evaluated separately in Table 4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We manually verify each result to ensure its accuracy' (Section 6.1) and 'each prompt injection attack is manually scrutinized to ascertain its success' (Section 6.1)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a security testing study on real applications, not a ML model evaluation with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-application, per-exploit-scenario breakdown of results. Table 1 provides per-category breakdown for the pilot study."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses 5 applications that resisted attacks and provides detailed reasons (domain-specific LLMs, internal processing, multimodal models). Section 3.2.2 provides a detailed failure case study."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The pilot study (Section 3) reports that existing attacks mostly fail (Table 1 shows overwhelmingly negative results). Section 6.2 reports 5/36 applications could not be exploited."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 31/36 vulnerable applications (86.1%), 10 vendor confirmations, and specific attack capabilities (prompt theft, LLM abuse) are all supported by Section 6 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Section 6.3) provides controlled single-variable manipulation for the causal claim that each separator strategy contributes to effectiveness. The main claim is descriptive (X applications are vulnerable)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Prompt Injection attack against LLM-integrated Applications' broadly, but testing was limited to 36 applications from SUPERTOOLS. The paper does not bound its generalization to this specific sample or application types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for why HOUYI succeeds. For instance, the success could be due to the specific LLM versions used at the time of testing, which would change as models update."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures success as whether specific keywords appear in output (e.g., correct answer to Q1/Q2) or output deviates from intended functionality. No discussion of whether these proxy measures capture the full security risk claimed (e.g., 'millions of users potentially affected')."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT3.5-turbo' for the feedback/generation LLM but does not specify a snapshot date or version. Target applications' underlying LLM versions are unknown (black-box)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides concrete examples of framework, separator, and disruptor components. The DECISIONAI example (Section 5.1) shows a complete injected prompt. Table 3 shows disruptor prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states 'This model functions under the default parameters, with both the temperature and top_p set as 1' for GPT-3.5-turbo."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-phase workflow (context inference, payload generation, feedback refinement) is described in detail in Sections 4-5, with Algorithm 1 providing pseudocode and Figure 4 showing the architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes the evaluation setup: application selection criteria (availability + LLM integration), manual API extraction, documentation collection, and success criteria definitions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) includes subsections on defenses (7.1), separator component limitations (7.2), and reproducibility concerns (7.3)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses specific reproducibility threats: 'certain detected vulnerabilities may become non-reproducible over time' due to patching and LLM evolution. Section 7.2 acknowledges the separator strategies 'likely only scratch the surface.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what settings or application types are excluded from its claims. The threat model (Section 2.3) excludes some attack vectors but does not bound the generalization of results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw attack logs, prompts, or response data are made available. Applications are anonymized, preventing verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes how applications were selected from SUPERTOOLS, selection criteria (availability, LLM integration), and how attacks were executed (RESTful API extraction, 5 repetitions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study tests software applications, not people."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from application selection through attack execution to result classification is described at a high level, but specifics like how many total attack attempts were made, how many internal errors occurred, and filtering criteria are not fully documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is visible in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Griffith University, NTU, UNSW, HUST, Indiana University, SUSTech, Tianjin University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests prompt injection attacks on real applications — it does not evaluate a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model knowledge. The study tests security vulnerabilities of deployed applications."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model knowledge. The study tests security vulnerabilities of deployed applications."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study tests software applications."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No cost of running HOUYI is reported. The paper mentions PAREA's cost ($259.2/day for prompt abuse) but not the cost of executing the attack toolkit itself."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget for the attack experiments is stated. The number of API calls, tokens consumed, or time spent is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results vary across runs (acknowledged via 5 repetitions) but no systematic seed analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states 'we execute each exploit prompt five times' and the pilot study also uses 5 repetitions."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is stated. The iterative refinement process (Algorithm 1) does not report how many iterations were needed per application."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 describes iterative component generation but does not report how many configurations were tried per application or justify the selection of reported results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own HOUYI system against their own implementations of baseline attacks without acknowledging potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of how compute budget (number of iterations, API calls) relates to attack success rate."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 5 exploit scenarios and success criteria adequately capture real-world prompt injection risk. The success criteria (keyword matching) may not reflect actual security impact."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "HOUYI is itself the scaffold being evaluated, not comparing models across different scaffolds."
       }
     }
   },
   "claims": [
     {
       "claim": "HOUYI achieves an 86.1% success rate in prompt injection attacks across 36 real-world LLM-integrated applications (31/36 vulnerable).",
       "evidence": "Table 4 in Section 6.2 shows per-application results across 5 exploit scenarios with 5 attempts each. 31 applications had at least one successful exploit.",
       "supported": "strong"
     },
     {
       "claim": "Existing prompt injection techniques are largely ineffective against real-world LLM-integrated applications.",
       "evidence": "Table 1 in Section 3.2 shows existing attacks succeed meaningfully on only 2/10 pilot applications (chatbots), with 0/10 success on Q3 (prompt leaking).",
       "supported": "moderate"
     },
     {
       "claim": "10 vendors have validated the discovered vulnerabilities, including Notion (20M+ users).",
       "evidence": "Section 6.4 states 10 vendors confirmed findings. Notion, Parea, and Writesonic are named. Direct quotes from Parea developers confirming the vulnerability are provided.",
       "supported": "strong"
     },
     {
       "claim": "Prompt abuse could cause financial losses of $259.2/day per vulnerable application like PAREA.",
       "evidence": "Section 6.4.2 derives this from 90k tokens/minute at $0.002/1k tokens over 1440 minutes. Based on GPT-3.5-turbo pricing.",
       "supported": "weak"
     },
     {
       "claim": "Existing defenses (instruction defense, post-prompting, sandwich defense, XML tagging, etc.) are insufficient against HOUYI.",
       "evidence": "Section 7.1 states defenses were implemented and evaluated on open-source projects, and HOUYI can 'effectively circumvent these security measures,' but no detailed results are provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "Despite implementing a 2,150-line Python toolkit, no code, attack logs, or data are released, making the results impossible to independently verify."
     },
     {
       "flag": "Very small sample per application",
       "detail": "Only 5 attempts per exploit scenario per application. LLM outputs are stochastic, making 5 attempts insufficient to reliably estimate success rates."
     },
     {
       "flag": "Defense evaluation lacks detail",
       "detail": "The claim that HOUYI bypasses all existing defenses (Section 7.1) is stated without detailed experimental results — only 'manual inspection' is mentioned."
     },
     {
       "flag": "Financial loss extrapolation",
       "detail": "The $259.2/day loss claim assumes 90k tokens/minute are entirely from malicious prompt abuse, which is unrealistic. The extrapolation methodology is simplistic."
     },
     {
       "flag": "Anonymization prevents verification",
       "detail": "Most applications are anonymized (31 of 36), making it impossible to verify which applications were tested or reproduce the results."
     }
   ],
   "cited_papers": [
     {
       "title": "Ignore Previous Prompt: Attack Techniques For Language Models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Foundational prompt injection attack paper targeting GPT-3, serves as baseline in this study."
     },
     {
       "title": "Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Introduces indirect prompt injection via poisoned external resources, complementary attack vector to HOUYI's direct approach."
     },
     {
       "title": "Jailbreaking ChatGPT via Prompt Engineering: An Empirical Study",
       "authors": ["Yi Liu", "Gelei Deng", "Zhengzi Xu", "Yuekang Li"],
       "year": 2023,
       "relevance": "Related jailbreak attack study by overlapping authors, explores prompt manipulation techniques for LLM safety bypass."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Demonstrates LLMs as tool-using agents, relevant to understanding the attack surface of LLM-integrated applications."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "Foundational agentic LLM framework combining reasoning and actions, relevant to LLM-integrated application security."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph C O'Brien", "Carrie J Cai"],
       "year": 2023,
       "relevance": "LLM-backed autonomous agents that could be vulnerable to prompt injection attacks."
     },
     {
       "title": "\"Real Attackers Don't Compute Gradients\": Bridging the Gap between Adversarial ML Research and Practice",
       "authors": ["Giovanni Apruzzese", "Hyrum S. Anderson"],
       "year": 2023,
       "relevance": "Discusses gap between adversarial ML research and real-world attacks, provides context for practical prompt injection."
     },
     {
       "title": "OWASP Top 10 List for Large Language Models",
       "year": 2023,
       "relevance": "Lists prompt injection as top LLM security risk, providing industry context for this research."
     }
   ]
 }