ai-research-survey

scan.json (24285B)

---

 {
   "paper": {
     "title": "Automatic and Universal Prompt Injection Attacks against Large Language Models",
     "authors": [
       "Xiaogeng Liu",
       "Zhiyuan Yu",
       "Yizhe Zhang",
       "Ning Zhang",
       "Chaowei Xiao"
     ],
     "year": 2024,
     "arxiv_id": "2403.04957"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at https://github.com/SheltonLiu-N/Universal-Prompt-Injection', providing a working repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available standard datasets (MRPC, Jfleg, HSOL, RTE, SST2, SMS Spam, Gigaword) which are all publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. The paper mentions using Llama2-7b-chat but does not specify library versions or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and hyperparameters are listed in Section 3.1, the paper does not provide step-by-step reproduction instructions or a README-style guide for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates of attack success rates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims their method outperforms baselines (e.g., 'an average 50% attack success rate' vs 0% for baselines) but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete attack success rates with baseline context, e.g., baselines at 0% ASR vs their method at ~50% average, and '21% improvement' of M-GCG over GCG in Table 2. These provide sufficient context for effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on 200 samples per dataset (1400 total) but provides no justification for why this sample size is sufficient. The training set of 5 samples is noted but not justified via power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any measure of spread across runs is reported. It is unclear whether results are from single or multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included: naive, combined (Liu et al., 2023c), and repeated (Toyer et al., 2023), described in Section 3.1 and Appendix B."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are from 2022-2023 (Toyer et al. 2023, Liu et al. 2023c), which are contemporary to this 2024 paper and represent state-of-the-art handcrafted prompt injection methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 presents an ablation comparing GCG (without momentum) vs M-GCG (with momentum), with loss curves in Figure 4 and quantitative results in Table 2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: keyword-evaluation ASR (KEY-E) and LLM-evaluation ASR (LM-E), as defined in Section 3.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. Attack success is measured only by keyword matching and GPT-4 automated evaluation. Human evaluation of attack outputs would be relevant to validate whether attacks actually deceive users."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper trains on only 5 samples and tests on 200 samples per dataset (1400 total). Additionally, two datasets (Spam Detection and Summarization, marked with * in Table 1) were not used during training, serving as held-out evaluation tasks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset breakdowns across all 7 datasets and all 3 attack objectives, rather than just aggregate averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses the limitation that their method is weak against PPL detection defense (Alon & Kamfonas, 2023). Section 3.2 notes interesting variation patterns, e.g., summarization is hardest for static but easiest for semi-dynamic."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports low success rates for certain objectives/datasets (e.g., 0.10 for semi-dynamic on Natural Language Inference, 0.10 for dynamic on Spam Detection). The PPL defense limitation is also reported as a negative result in Section 5."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'superior performance compared with baselines' with 'only five training samples (0.3% relative to the test data)'. Table 1 supports this: baselines achieve 0% ASR while the proposed method achieves substantial rates. The abstract also claims effectiveness 'even in the face of defensive measures', supported by Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Section 3.3 compares GCG vs M-GCG with controlled single-variable manipulation (adding momentum), providing adequate evidence for the causal claim that momentum enhances convergence and performance (Table 2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Universal Prompt Injection Attacks against Large Language Models' (plural), but experiments are conducted only on Llama2-7b-chat. No other LLMs are tested. The claim of universality across models is not supported."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the baselines' 0% success rate might be due to the evaluation protocol rather than inherent ineffectiveness, or whether the results are specific to Llama2's safety training."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The victim model is specified as 'Llama2-7b-chat' (Section 3.1), which is a specific model version. The LLM evaluator is specified as 'GPT-4-0613' in Appendix C."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides the full text of baseline attack prompts for all three objectives. Appendix C provides the full LLM evaluator prompt. The optimization target format is described in Section 2.3 with examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports: top-k=128, batch size=256, iterations=1000, momentum weight=1.0, injection content token length=150."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use agentic scaffolding. The attack is a gradient-based optimization method, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists the datasets used but does not describe how data was preprocessed, what subset of each dataset was selected, or how the 200 test samples per dataset were chosen."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Conclusions, Limitation, and Future Work' and includes a dedicated paragraph on the weakness against PPL detection defense."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitation discussion mentions only the PPL detection weakness. It does not discuss broader threats to validity such as single-model evaluation, potential overfitting to Llama2, or whether the evaluation protocol itself might bias results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Despite testing only on Llama2-7b-chat, the title and claims suggest broad applicability to 'Large Language Models' without bounding the scope to the tested model."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (model responses, per-example attack results) are released. Only aggregate ASR numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the datasets used and Appendix A describes how adversarial goals were generated using ChatGPT with a danger scale rating process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard NLP benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document the full pipeline from dataset selection to final evaluation results. How the 200 test samples per dataset were selected and how training samples were chosen is not explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: superscripts 1, 2, 3 are shown. Based on arXiv listing, the authors are from University of Wisconsin-Madison, Washington University in St. Louis, and Apple."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. One author appears to be affiliated with Apple, which develops LLM products, but this is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama2-7b-chat as the victim model but does not state its training data cutoff date, which is relevant for understanding whether the model may have seen the evaluation datasets during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the NLP benchmark datasets (MRPC, SST2, RTE, etc.) appeared in Llama2's training data. Since these are well-known benchmarks, contamination is plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets (MRPC, SST2, RTE, etc.) were all published well before Llama2's training. The paper does not discuss whether the model's familiarity with these datasets affects the attack evaluation results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API costs are reported despite the method requiring 1000 optimization iterations with gradient computation on an LLM."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget is stated, despite the method requiring gradient-based optimization on a 7B parameter model."
       }
     }
   },
   "claims": [
     {
       "claim": "With only five training samples (0.3% of test data), the proposed attack achieves superior performance compared with baselines.",
       "evidence": "Table 1 shows baselines (naive, combined, repeated) achieve 0% ASR across all datasets and objectives, while the proposed method achieves an average 81% KEY-E for static, 37%/35% KEY-E/LM-E for semi-dynamic, and 39%/34% KEY-E/LM-E for dynamic objectives.",
       "supported": "strong"
     },
     {
       "claim": "The momentum-enhanced optimization (M-GCG) improves convergence speed and solution quality compared to standard GCG.",
       "evidence": "Table 2 shows M-GCG achieves 0.81/0.37/0.39 vs GCG's 0.79/0.21/0.34 across static/semi-dynamic/dynamic objectives. Figure 4 shows loss curves with consistently faster convergence for M-GCG.",
       "supported": "moderate"
     },
     {
       "claim": "The attack remains effective even against defense mechanisms.",
       "evidence": "Figure 5 shows that without adaptive strategy, defenses cause a 32% performance drop but the attack remains effective. With EOT adaptive scheme, performance recovers to 85% of the no-defense baseline (Section 3.4).",
       "supported": "moderate"
     },
     {
       "claim": "Existing defense mechanisms that were reported effective cannot mitigate the threat of prompt injection attacks.",
       "evidence": "Figure 5 demonstrates that all five defenses (paraphrasing, retokenization, data prompt isolation, instructional prevention, sandwich prevention) are bypassed to varying degrees. However, the PPL detection defense is acknowledged as effective in Section 5.",
       "supported": "moderate"
     },
     {
       "claim": "The attack demonstrates universality across various user interactions and datasets.",
       "evidence": "Table 1 shows effectiveness on 7 datasets including 2 unseen during training (marked with *). However, only one model (Llama2-7b-chat) is tested, so universality across models is not demonstrated.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces a momentum-enhanced gradient-based optimization method (M-GCG) for automatically generating universal prompt injection attacks against LLMs. Using only 5 training samples, the attack achieves an average 50% attack success rate across 7 NLP datasets and 3 attack objectives (static, semi-dynamic, dynamic), while all handcrafted baseline attacks achieve 0% under the same evaluation protocol. The attack also demonstrates resilience against 5 existing defense mechanisms, with adaptive strategies recovering 85% of original performance. A key limitation is that all experiments are conducted on a single model (Llama2-7b-chat), limiting generalizability claims.",
   "red_flags": [
     {
       "flag": "Single model evaluation with broad generalization claims",
       "detail": "The title claims 'Attacks against Large Language Models' (plural) but all experiments use only Llama2-7b-chat. No closed-source models (GPT-4, Claude) or other open models are tested. The universality claim is only validated across datasets and tasks, not across different LLMs."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates with no error bars, confidence intervals, or indication of variance across runs. For a stochastic optimization process with random token sampling, this is a significant omission."
     },
     {
       "flag": "Baselines may be unfairly weak under the evaluation protocol",
       "detail": "All baselines achieve exactly 0% ASR across every dataset and objective. The paper's evaluation protocol focuses on 'malicious goals' rather than 'benign task-switching', which may structurally disadvantage handcrafted methods designed for a different threat model. The paper acknowledges this shift but does not test baselines with adaptations to the new protocol."
     },
     {
       "flag": "Missing compute costs",
       "detail": "Running 1000 gradient optimization iterations on a 7B parameter model is computationally expensive, but no hardware, runtime, or cost information is provided. This makes practical applicability assessment impossible."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The evaluation uses well-known NLP benchmarks (MRPC, SST2, RTE, etc.) that may be in Llama2's training data. Whether the model's familiarity with these tasks affects attack success is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": ["Andy Zou", "Zifan Wang", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational work on GCG jailbreak attacks that this paper builds upon; core method for adversarial prompt optimization."
     },
     {
       "title": "Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Seminal work on indirect prompt injection attacks in LLM-integrated applications, defining the threat model this paper formalizes."
     },
     {
       "title": "Ignore Previous Prompt: Attack Techniques For Language Models",
       "authors": ["F\u00e1bio Perez", "Ian Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Pioneering academic study of prompt injection, introducing goal hijacking and prompt leaking taxonomies."
     },
     {
       "title": "Tensor Trust: Interpretable Prompt Injection Attacks from an Online Game",
       "authors": ["Sam Toyer", "Olivia Watkins", "Ethan Adrian Mendes", "Justin Svegliato", "Luke Bailey", "Tiffany Wang", "Isaac Ong", "Karim Elmaaroufi", "Pieter Abbeel", "Trevor Darrell", "Alan Ritter", "Stuart Russell"],
       "year": 2023,
       "arxiv_id": "2311.01011",
       "relevance": "Large-scale crowdsourced prompt injection study that provides one of the baselines evaluated in this paper."
     },
     {
       "title": "Prompt Injection Attacks and Defenses in LLM-Integrated Applications",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2023,
       "arxiv_id": "2310.12815",
       "relevance": "Proposes the combined prompt injection attack and benchmark used as the primary baseline in this paper."
     },
     {
       "title": "Prompt Injection attack against LLM-integrated Applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang", "Tianwei Zhang", "Yepang Liu", "Haoyu Wang", "Yan Zheng", "Yang Liu"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Framework for prompt injection attacks applied to study 36 LLM-integrated applications, identifying 31 as vulnerable."
     },
     {
       "title": "Jatmo: Prompt Injection Defense by Task-Specific Finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen", "Zeming Wei", "Elizabeth Sun", "Basel Alomair", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2312.17673",
       "relevance": "Defense mechanism against prompt injection through task-specific finetuning; relevant to the defense evaluation in this paper."
     },
     {
       "title": "Benchmarking and Defending Against Indirect Prompt Injection Attacks on Large Language Models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Keegan Hines", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Benchmarks indirect prompt injection attacks and proposes defenses; closely related evaluation of LLM vulnerability to injection."
     },
     {
       "title": "Baseline defenses for adversarial attacks against aligned language models",
       "authors": ["Neel Jain", "Avi Schwarzschild", "Yuxin Wen", "Gowthami Somepalli", "John Kirchenbauer", "Ping-yeh Chiang", "Micah Goldblum", "Aniruddha Saha", "Jonas Geiping", "Tom Goldstein"],
       "year": 2023,
       "relevance": "Proposes paraphrasing and retokenization defenses evaluated in this paper's Section 3.4."
     },
     {
       "title": "AutoDAN: Generating Stealthy Jailbreak Prompts on Aligned Large Language Models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2023,
       "arxiv_id": "2310.04451",
       "relevance": "Related automated jailbreak generation method from some of the same authors; relevant to automated adversarial attack generation."
     },
     {
       "title": "TrustLLM: Trustworthiness in Large Language Models",
       "authors": ["Lichao Sun", "Yue Huang", "Haoran Wang", "Siyuan Wu", "Qihui Zhang", "Chujie Gao", "Yixin Huang", "Wenhan Lyu", "Yixuan Zhang", "Xiner Li"],
       "year": 2024,
       "arxiv_id": "2401.05561",
       "relevance": "Comprehensive trustworthiness benchmark for LLMs covering safety, robustness, and other dimensions relevant to this survey."
     },
     {
       "title": "Assessing Prompt Injection Risks in 200+ Custom GPTs",
       "authors": ["Jiahao Yu", "Yuhang Wu", "Dong Shu", "Mingyu Jin", "Xinyu Xing"],
       "year": 2023,
       "arxiv_id": "2311.11538",
       "relevance": "Large-scale empirical assessment of prompt injection vulnerability in deployed GPT applications."
     }
   ]
 }