ai-research-survey

scan.json (27617B)

---

 {
   "paper": {
     "title": "Bypassing LLM Guardrails: An Empirical Analysis of Evasion Attacks against Prompt Injection and Jailbreak Detection Systems",
     "authors": [
       "William Hackett",
       "Lewis Birch",
       "Stefan Trawicki",
       "Neeraj Suri",
       "Peter Garraghan"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.11168"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided in the paper. The authors mention a HuggingFace dataset of evaded samples (footnote 2) but no code for reproducing the attacks or experimental pipeline."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors released evaded prompt injection and jailbreak samples on HuggingFace (https://huggingface.co/datasets/Mindgard/evaded-prompt-injection-and-jailbreak-samples, footnote 2 in Section 5). The datasets used (safe-guard-prompt-injection, hackGPT) are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions using TextAttack (Morris et al., 2020) but does not specify library versions or Python version."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The experimental setup section (Section 4) describes the methodology at a high level but not enough to reproduce without guessing implementation details."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates of Attack Success Rate (ASR) with no confidence intervals or error bars. Tables and figures show single percentage values only."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about which attacks and defenses are more effective (e.g., 'Vijil Prompt Injection exhibited the highest susceptibility') but provides no statistical significance tests. Comparisons are based solely on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides ASR percentages with baseline context. Table 3 reports baseline ASR, new ASR, and delta (percentage change). Section 5 gives specific ASR values for each technique against each guardrail, providing enough context to assess magnitude (e.g., 'Emoji Smuggling achieved 100% ASR', 'Protect AI v2 reduced prompt injection ASR to 20.26%')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The jailbreak dataset contains only 78 prompts, which is quite small for drawing broad conclusions. No justification for this sample size is provided, nor is there any acknowledgment that 78 prompts may be too few for reliable ASR estimates."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All ASR results appear to be from single experimental runs with no indication of multiple runs or seed variation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table A.1 reports detection baselines for each guardrail system on the unperturbed adversarial samples. The evasion results are compared against these baseline detection rates, showing how much each attack degrades detection."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The six guardrail systems tested are contemporary: Azure Prompt Shield (2024), ProtectAI v2 (April 2024), Meta Prompt Guard (2024), Vijil (2025), NeMo Guard (2024). These represent current state-of-the-art protection systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests multiple attack techniques independently but does not perform ablation studies to isolate which components of the attacks contribute most to evasion success. For example, the word importance transferability experiment (Section 5.3) is a separate experiment, not a systematic ablation of the AML pipeline."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only a single metric: Attack Success Rate (ASR), defined as the rate at which modified prompts are misclassified as benign. No other metrics such as semantic similarity preservation, query efficiency, or perceptibility scores are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the perturbed prompts is included. The paper claims perturbations 'maintain adversarial utility' and are 'imperceptible' but provides no human judgment to verify these claims. Section 8 (Limitations) acknowledges that 'more rigorous quantitative analyses are needed to determine how perturbations affect the success rate and intended behavior of modified prompt injections or jailbreaks.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The prompt injection dataset uses a test set split: 'From its test set (2,060 examples), we selected only adversarial samples (650 examples), finally filtering out jailbreak samples totaling 476 prompt injection prompts' (Section 4). The jailbreak dataset is a separate repository of 78 prompts."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per attack technique and per guardrail system, shown in Figures 2-5 and Tables A.5-A.8. This allows readers to see which specific attacks are effective against which specific guardrails."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where attacks were less effective: 'Deletion Characters' was the least effective (26.82% for prompt injections, 7.95% for jailbreaks), and Pruthi showed decreased ASR with transferability (-18.18% and -1.01%). The Limitations section acknowledges adversarial prompt efficacy concerns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that some attack techniques had low success rates (e.g., Pruthi, Alzantot on jailbreaks), and that transferability decreased ASR for Pruthi (-18.18% for jailbreaks, -1.01% for prompt injections in Table 3). It also notes that AML evasion had lower overall success than character injection."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'both methods can be used to evade detection while maintaining adversarial utility achieving in some instances up to 100% evasion success' — this is supported by the results (Emoji Smuggling achieves 100% ASR). The claim about word importance ranking transferability is supported by Table 3. The qualifier 'in some instances' appropriately hedges the claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are about attacks causing evasion, which is directly demonstrated through controlled experiments: applying specific perturbations (the treatment) and measuring whether classification changes (the outcome). This is an adequate experimental design for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Bypassing LLM Guardrails' generally, but the study tests only 6 specific guardrail systems. The abstract says 'vulnerabilities within current LLM protection mechanisms' without bounding this to the tested systems. Section 8 partially addresses this by noting 'it limits the generalizability of our findings' regarding the black-box scope, but the title and conclusion still overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section (Section 6) considers alternative explanations: differences in training data exposure (Section 6.1), differences in tokenizer handling (Section 6.1), and design differences in input size and token support (Section 6.3). Section 6.1 explains why success varies across models."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The guardrail models are named but specific version hashes or dates are not given for most. ProtectAI v1 and v2 have release dates ('25th November 2023' and '21st April 2024'). Meta Prompt Guard is described as fine-tuned from mDeBERTa-v3-base (86M parameters). GPT-4o-mini is mentioned without a snapshot date. Azure Prompt Shield has no version identifier."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense — it tests classification-based guardrail systems, not prompt-based LLM evaluations. The 'prompts' are the adversarial inputs themselves, which are described via the datasets used and example perturbations shown in the Appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the AML evasion techniques (e.g., perturbation budget, similarity thresholds, number of iterations). TextAttack is mentioned as the framework but no configuration details are provided. The character injection techniques have no tunable parameters, but the AML techniques do and these are not documented."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper tests classification-based guardrail systems with automated attack techniques, not agentic pipelines."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data pipeline: 'From its test set (2,060 examples), we selected only adversarial samples (650 examples), finally filtering out jailbreak samples totaling 476 prompt injection prompts.' The jailbreak dataset source and size (78 prompts) are specified. The filtering steps and counts are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Limitations' and provides substantive discussion of three specific limitations: black-box target scope, transferability mechanisms, and adversarial prompt efficacy."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'Our study focused solely on Azure Prompt Shield as the representative black-box target' (Section 8), and 'the underlying mechanisms driving this transferability, particularly regarding word importance, remain unclear' — these are specific to the experimental design, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states scope boundaries: the black-box evaluation is limited to Azure Prompt Shield only, transferability mechanisms are not fully understood, and the efficacy of perturbed prompts against actual LLMs has not been rigorously tested. These are specific things the paper did NOT test."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The evaded prompt samples are released on HuggingFace (footnote 2), the prompt injection dataset (safe-guard-prompt-injection) is public, and the jailbreak dataset (hackGPT) is publicly available on GitHub. This allows independent verification of the attack results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes how the evaluation data was collected: the prompt injection dataset (Erdogan et al., 2024) with 10,296 examples, filtering to 476 prompt injections from the test set, and the jailbreak dataset (NoDataFound, 2024) with 78 prompts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses publicly available datasets and automated attacks against guardrail systems."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the pipeline: dataset selection, filtering criteria (test set only, adversarial samples only, jailbreak samples removed), final sample counts (476 prompt injections, 78 jailbreaks), attack application via automated system/TextAttack, and evaluation against guardrails. The data flow is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors, only a brief acknowledgment of guardrail vendors for the disclosure process (Section 10)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: '1Mindgard, 2Lancaster University'. All authors have dual affiliations with Mindgard (an AI security company) and Lancaster University. The Mindgard affiliation is clearly listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. However, all authors are affiliated with Mindgard, an AI security company. Mindgard has a commercial interest in demonstrating that existing guardrails are vulnerable (their business is AI security testing). This potential conflict is not discussed. The funder independence cannot be assessed because no funding is disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. The authors are affiliated with Mindgard, an AI security company that would commercially benefit from demonstrating guardrail vulnerabilities, but this financial interest is not declared or discussed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests evasion attacks against guardrail classification models, not pre-trained LLM capabilities on benchmarks. The study evaluates whether guardrails can detect perturbed malicious prompts, not whether a model has memorized test data."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper does not evaluate a pre-trained model's knowledge on a benchmark. It tests whether classification-based guardrails can detect adversarially perturbed inputs."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper is a security evaluation of guardrail classifiers, not a benchmark evaluation of LLM capabilities. Data contamination in the traditional sense is not a relevant concern here."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or wall-clock time is reported for the evasion attacks or the guardrail evaluations. This is relevant because AML attacks require many queries to the target model, and the paper does not report query counts or costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report GPU hours, API costs, hardware used, or total time to run experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Character injection techniques and AML evasion techniques can bypass LLM guardrail detection, achieving up to 100% evasion success in some instances.",
       "evidence": "Figures 2-3 and Tables A.5-A.6 show Emoji Smuggling achieves 100% ASR across all tested guardrails for both prompt injections and jailbreaks. Multiple other character injection techniques achieve high ASRs (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Vijil Prompt Injection exhibited the highest susceptibility to character injection attacks, with average ASRs of 87.95% for prompt injections and 91.67% for jailbreaks.",
       "evidence": "Section 5.1 reports these figures, with per-technique breakdowns in Tables A.5 and A.6 supporting the claim.",
       "supported": "strong"
     },
     {
       "claim": "AML evasion techniques exhibit lower overall success rates compared to character injection.",
       "evidence": "Section 5.2 reports AML ASRs ranging from under 18% to 57.57%, compared to character injection techniques reaching up to 100%. This pattern is consistent across the results in Figures 4-5.",
       "supported": "strong"
     },
     {
       "claim": "White-box word importance ranking transferability can enhance ASR against black-box targets.",
       "evidence": "Table 3 shows 6 of 8 techniques improved jailbreak ASR and 7 of 8 improved prompt injection ASR when using Protect AI v2 to compute word importance rankings for attacking Azure Prompt Shield. However, improvements are modest (e.g., Bert-Attack prompt injection ASR improved from 65.34% to 73.11%).",
       "supported": "moderate"
     },
     {
       "claim": "Perturbed prompts maintain adversarial utility while evading detection.",
       "evidence": "The paper provides qualitative examples in Appendix Tables A.2-A.4 but acknowledges in Section 8 (Limitations) that 'more rigorous quantitative analyses are needed to determine how perturbations affect the success rate and intended behavior.' No systematic evaluation of prompt efficacy post-perturbation is conducted.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Character injection techniques (especially emoji smuggling, Unicode tags, and number substitution) can achieve near-complete evasion of LLM guardrail systems, with emoji smuggling reaching 100% ASR across all tested guardrails. Adversarial ML evasion techniques are less effective overall but still achieve significant bypass rates, particularly against white-box targets. Word importance ranking computed from white-box models can transfer to improve attack success against black-box guardrails, though improvements are modest. The study reveals that the gap between what guardrail classifiers can detect and what LLMs can interpret creates an exploitable vulnerability.",
   "red_flags": [
     {
       "flag": "Company evaluating competitor products",
       "detail": "All authors are affiliated with Mindgard, an AI security company. The paper demonstrates that competitor guardrail products (Azure, Meta, ProtectAI, Vijil, NeMo) are vulnerable to attacks, which directly benefits Mindgard's commercial interests. No conflict of interest statement is included."
     },
     {
       "flag": "Small jailbreak dataset",
       "detail": "The jailbreak evaluation uses only 78 prompts (from the hackGPT repository), which is small for drawing generalizable conclusions about attack effectiveness. ASR percentages on 78 samples have wide implicit confidence intervals (e.g., 11.54% = 9/78 samples)."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, or variance measures. Given the small jailbreak dataset size, the reported ASR percentages may not be stable estimates."
     },
     {
       "flag": "Adversarial utility not rigorously verified",
       "detail": "The paper claims perturbed prompts 'maintain adversarial utility' but provides no systematic evaluation of whether perturbed prompts actually achieve their malicious intent when passed to an LLM. The Limitations section acknowledges this gap."
     },
     {
       "flag": "Missing attack cost analysis",
       "detail": "AML evasion techniques require multiple queries to the target model for word importance ranking and perturbation refinement, but the number of queries needed per successful evasion is never reported, making practical threat assessment difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "Breaking down the defenses: A comparative survey of attacks on large language models",
       "authors": ["Arijit Ghosh Chowdhury", "Md Mofijul Islam", "Vaibhav Kumar", "Faysal Hossain Shezan", "Vaibhav Kumar", "Vinija Jain", "Aman Chadha"],
       "year": 2024,
       "arxiv_id": "2403.04786",
       "relevance": "Comprehensive survey of attacks on LLMs, directly relevant to understanding the threat landscape for LLM guardrails."
     },
     {
       "title": "Safeguarding large language models: A survey",
       "authors": ["Yi Dong", "Ronghui Mu", "Yanghao Zhang", "Siqi Sun", "Tianle Zhang", "Changshun Wu", "Gaojie Jin", "Yi Qi", "Jinwei Hu", "Jie Meng", "Saddek Bensalem", "Xiaowei Huang"],
       "year": 2024,
       "arxiv_id": "2406.02622",
       "relevance": "Survey on LLM safeguarding techniques including guardrails, directly relevant to the defense mechanisms evaluated in this paper."
     },
     {
       "title": "Bad characters: Imperceptible NLP attacks",
       "authors": ["Nicholas Boucher", "Ilia Shumailov", "Ross Anderson", "Nicolas Papernot"],
       "year": 2021,
       "arxiv_id": "2106.09898",
       "relevance": "Foundational work on character-level adversarial attacks against NLP models, a core technique category evaluated in this paper."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang", "Ning Zhang", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Automatic prompt injection attack methods against LLMs, directly relevant to the prompt injection threat model."
     },
     {
       "title": "AutoDAN: Generating stealthy jailbreak prompts on aligned large language models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2310.04451",
       "relevance": "Automated jailbreak generation method relevant to understanding the jailbreak attack landscape."
     },
     {
       "title": "Fundamental limitations of alignment in large language models",
       "authors": ["Yotam Wolf", "Noam Wies", "Oshri Avnery", "Yoav Levine", "Amnon Shashua"],
       "year": 2024,
       "relevance": "Theoretical analysis of alignment limitations in LLMs, providing foundational context for why guardrails may be fundamentally bypassable."
     },
     {
       "title": "Emoji attack: Enhancing jailbreak attacks against judge LLM detection",
       "authors": ["Zhipeng Wei", "Yuqi Liu", "N. Benjamin Erichson"],
       "year": 2025,
       "arxiv_id": "2411.01077",
       "relevance": "Directly related work on emoji-based attacks against LLM detection systems, one of the most effective techniques found in this paper."
     },
     {
       "title": "TextAttack: A framework for adversarial attacks, data augmentation, and adversarial training in NLP",
       "authors": ["John Morris", "Eli Lifland", "Jin Yong Yoo", "Jake Grigsby", "Di Jin", "Yanjun Qi"],
       "year": 2020,
       "relevance": "The framework used to implement the AML evasion techniques in this paper, relevant to understanding reproducibility of adversarial NLP attacks."
     },
     {
       "title": "Improved large language model jailbreak detection via pretrained embeddings",
       "authors": ["Erick Galinkin", "Martin Sablotny"],
       "year": 2024,
       "arxiv_id": "2412.01547",
       "relevance": "Describes the NeMo Guard jailbreak detection system, one of the six guardrails evaluated in this paper."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey et al."],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Describes Meta's Prompt Guard classifier used in guardrail systems, one of the six guardrails evaluated in this paper."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2024,
       "relevance": "LLM evaluation methodology paper relevant to understanding how LLM-based content safety systems are assessed."
     },
     {
       "title": "'Prompter Says': A linguistic approach to understanding and detecting jailbreak attacks against large-language models",
       "authors": ["Dylan Lee", "Shaoyuan Xie", "Shagoto Rahman", "Kenneth Pat", "David Lee", "Qi Alfred Chen"],
       "year": 2024,
       "relevance": "Linguistic approach to jailbreak detection, relevant to understanding alternative detection methods for LLM guardrails."
     }
   ]
 }