ai-research-survey

scan.json (20187B)

---

 {
   "paper": {
     "title": "A Critical Evaluation of Defenses against Prompt Injection Attacks",
     "authors": ["Yuqi Jia", "Zedian Shao", "Yupei Liu", "Jinyuan Jia", "Dawn Song", "Neil Zhenqiang Gong"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.18333"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/PIEval123/PIEval, stated in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use publicly available benchmarks (OpenPromptInjection under MIT license, MMLU) and construct MMLU-PI from MMLU. Code/data repository is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'one single NVIDIA RTX A5000 GPU with 24GB memory' but provides no requirements.txt, library versions, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-style instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (ASV, FPR, FNR, utility scores) are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims defenses are 'not as effective as previously reported' based on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are contextualized with baselines. E.g., utility drops from 0.65 to 0.54 for StruQ on OpenPromptInjection (Table 1b), ASV increases from 0.04 to 0.80 under GCG (Table 2). Absolute values with baseline context provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. For GCG attacks they select 50 tuples from OpenPromptInjection and 25 from MMLU-PI 'for computational efficiency' but do not justify whether this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported across any experiments. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Undefended LLMs serve as baselines (Llama-3-8B-Instruct, Llama-3-8B-undefended) and results are compared against original defense papers' claims."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Defenses evaluated are recent (StruQ 2025, SecAlign 2024, Instruction Hierarchy 2024, PromptGuard 2024, Attention Tracker 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "For adaptive attacks on Attention Tracker, they ablate what components are optimized: separator only, separator+instruction, separator+instruction+data (Table 3b)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: ASV for effectiveness, absolute utility, relative utility (win rate), FPR, FNR, and AUC for detection-based defenses."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a security evaluation paper measuring attack success rates and defense effectiveness via automated metrics. Human evaluation of outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "They use separate benchmarks (OpenPromptInjection, MMLU-PI) from those used by the original defense papers, providing independent evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per defense (StruQ, SecAlign, Instruction Hierarchy, PromptGuard, Attention Tracker), per benchmark (OpenPromptInjection, MMLU-PI), and per attack type (existing vs adaptive, Combined vs GCG)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about demonstrating failure cases of existing defenses. They discuss where adaptive Combined Attack does not improve over existing (Section 5.1) due to long-context issues."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "They report that adaptive Combined Attack does not always outperform existing attacks (Section 5.1), and that Llama-3-8B base models show smaller utility drops than Instruct models (Appendix A)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that 'existing defenses are not as successful as previously reported' are supported by Tables 1-5 showing utility degradation and high ASVs under diverse evaluation."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited and well-supported. E.g., adaptive attacks using embedding-similar tokens cause higher ASV (controlled manipulation). The ablation of optimizable components in Table 3b supports causal reasoning about attack strategies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates only Llama-3-8B variants and GPT-4o-mini but makes broad claims about 'existing defenses' in general. The title and abstract suggest comprehensive coverage but only 4 defenses are tested on limited model families."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, utility drops could be partially due to evaluation methodology differences rather than inherent defense weakness. The paper does not consider that the original evaluations may have had valid reasons for their benchmark choices."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named (Llama-3-8B-Instruct, GPT-4o-mini, Qwen2-1.5B-Instruct) but no specific version snapshots, API dates, or checkpoint identifiers are provided. GPT-4o-mini and GPT-4-Turbo are used without version/snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt structures formally (concatenation of instruction, data, separator) but does not provide actual prompt text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only α=0.01 for the adaptive attack loss balance is reported. GCG optimization hyperparameters, temperature settings for LLM inference, and other critical parameters are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a direct attack/defense evaluation study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes how benchmarks are constructed: OpenPromptInjection has 700 prompt-response pairs (100 per task), T contains 4,900 tuples. MMLU-PI construction from MMLU test split is described with exact counts (200 pairs, 1,000 tuples)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not clarify what the results do NOT show or what settings were not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code and data are available at the GitHub repository. The benchmarks used (OpenPromptInjection, MMLU) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes how benchmarks were constructed, including sampling procedures and dataset sizes for both OpenPromptInjection and MMLU-PI."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use automated benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark datasets to evaluation tuples (pt, rt, pe, re) is formally specified with equations and exact counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Duke University, Penn State, UC Berkeley."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests defense effectiveness against attacks, not model knowledge on benchmarks. Contamination of training data with benchmark content is not relevant to the claims."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates attack/defense dynamics, not model capability on knowledge benchmarks. Train/test overlap is not a concern for the claims made."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates defenses against prompt injection, not model knowledge. Benchmark contamination is irrelevant."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times reported despite running extensive experiments with GCG optimization and multiple LLMs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only the GPU type is mentioned (NVIDIA RTX A5000 24GB). No total compute budget, GPU hours, or training/inference time reported."
       }
     }
   },
   "claims": [
     {
       "claim": "StruQ and SecAlign lead to a loss of both relative and absolute utility, contrary to original claims",
       "evidence": "Table 1: StruQ achieves only 21.60% win rate vs undefended model; absolute utility drops 0.11 and 0.10 on two benchmarks. SecAlign drops 0.17 and 0.11.",
       "supported": "strong"
     },
     {
       "claim": "StruQ and SecAlign are not as effective against existing attacks as previously reported when evaluated on diverse prompts",
       "evidence": "Table 2: GCG achieves ASVs exceeding 0.80 against StruQ vs original reported 0.04; SecAlign shows 0.72 ASV on MMLU-PI vs original reported 0.",
       "supported": "strong"
     },
     {
       "claim": "Instruction Hierarchy (deployed on GPT-4o-mini) is not effective against diverse injected prompts",
       "evidence": "Section 5.2: Combined Attack achieves ASVs of 0.68 and 0.75 on OpenPromptInjection and MMLU-PI respectively.",
       "supported": "moderate"
     },
     {
       "claim": "AUC alone is insufficient for assessing detection-based defenses",
       "evidence": "Table 3a: PromptGuard has AUC 0.92 but FPR 0.89; Attention Tracker has AUC 1.00 but FNR 0.69 on MMLU-PI.",
       "supported": "strong"
     },
     {
       "claim": "Adaptive attacks significantly undermine detection-based defenses",
       "evidence": "Table 3b: Adaptive attacks raise Attention Tracker FNR from 0.00 to 0.66-1.00 depending on strategy and benchmark.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Existing prompt injection defenses (StruQ, SecAlign, Instruction Hierarchy, PromptGuard, Attention Tracker) are not as effective as originally claimed when evaluated with diverse target/injected prompts and adaptive attacks. Prevention-based defenses show significant utility degradation when measured with absolute utility metrics rather than only relative win rates. Detection-based defenses that appear strong by AUC metrics show high false positive or false negative rates in practice. Adaptive attacks, particularly optimization-based ones like GCG, can dramatically increase attack success values against all tested defenses.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single point estimates with no error bars, confidence intervals, or variance across runs. GCG optimization is stochastic, so single-run results may not be representative."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion despite making strong claims about the inadequacy of prior defense evaluations."
     },
     {
       "flag": "Small GCG evaluation subset",
       "detail": "GCG attacks are evaluated on only 50 tuples (OpenPromptInjection) and 25 tuples (MMLU-PI) 'for computational efficiency' without justification that this is representative."
     },
     {
       "flag": "Limited model coverage for broad claims",
       "detail": "Only Llama-3-8B variants and GPT-4o-mini are tested, but claims extend to defenses generally. Different model architectures or sizes could yield different results."
     }
   ],
   "cited_papers": [
     {
       "title": "Struq: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2025,
       "relevance": "Prevention-based defense against prompt injection evaluated in this paper."
     },
     {
       "title": "Aligning llms to be robust against prompt injection",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "Saeed Mahloujifar", "Kamalika Chaudhuri", "Chuan Guo"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "SecAlign defense using DPO fine-tuning for prompt injection resistance, evaluated in this paper."
     },
     {
       "title": "The instruction hierarchy: Training llms to prioritize privileged instructions",
       "authors": ["Eric Wallace", "Kai Xiao", "Reimar Leike", "Lilian Weng", "Johannes Heidecke", "Alex Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Instruction hierarchy defense deployed on GPT-4o-mini, shown to be ineffective against diverse attacks."
     },
     {
       "title": "Attention tracker: Detecting prompt injection attacks in llms",
       "authors": ["Kuo-Han Hung", "Ching-Yun Ko", "Ambrish Rawat", "I-Hsin Chung", "Winston H. Hsu", "Pin-Yu Chen"],
       "year": 2024,
       "arxiv_id": "2411.00348",
       "relevance": "Detection-based defense shown vulnerable to adaptive attacks in this evaluation."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "OpenPromptInjection benchmark used as primary evaluation framework in this paper."
     },
     {
       "title": "Obfuscated gradients give a false sense of security: Circumventing defenses to adversarial examples",
       "authors": ["Anish Athalye", "Nicholas Carlini", "David Wagner"],
       "year": 2018,
       "relevance": "Foundational work on adaptive attacks against adversarial example defenses, motivating the adaptive attack methodology used here."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack method used as the primary optimization-based attack in this evaluation."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang", "Ning Zhang", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Optimization-based prompt injection attack method relevant to the attack taxonomy discussed."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": ["Yupei Liu", "Yuqi Jia", "Jinyuan Jia", "Dawn Song", "Neil Zhenqiang Gong"],
       "year": 2025,
       "relevance": "Game-theoretic approach to prompt injection detection from the same research group."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Benchmark and defense study for indirect prompt injection attacks."
     }
   ]
 }