ai-research-survey

scan.json (24518B)

---

 {
   "paper": {
     "title": "CAPTURE: Context-Aware Prompt Injection Testing and Robustness Enhancement",
     "authors": ["Gauri Kholkar", "Ratinder Ahuja"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.12368",
     "doi": "10.48550/arXiv.2505.12368"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The dataset generation pipeline code will be shared upon publication' (footnote 1, Section 1). A promise of future release counts as NO per the schema."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No link to the generated CAPTURE datasets is provided in the paper. The datasets (MALICIOUS-GEN, SAFE-GEN) are described but not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions using DeBERTaV3-base and GPT-4o but does not specify library versions or a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the data generation pipeline is described at a high level (Sections 2.1-2.3), there are no runnable commands, scripts, or a README with reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, and 6 are reported as point estimates (e.g., '0.15%', '79.04%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., CaptureGuard outperforms others) but uses no statistical significance tests. Differences are assessed purely by comparing raw FNR/FPR numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw FNR and FPR percentages but does not report formal effect sizes (e.g., Cohen's d, odds ratios). While percentage differences are shown, there is no baseline-contextualized effect size measure."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset sizes (1274 training, 641 test/validation for MALICIOUS-GEN; 339 training, 171 test/validation for SAFE-GEN) are stated but not justified. No power analysis or rationale for these specific sizes is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. CaptureGuard models appear to be trained once per domain with single-run results reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates five specialized guardrail models (ProtectAIv2, InjecGuard, PromptGuard, Deepset, Fmops) and two LLMs (GPT-4o, Llama3.2-1B-Instruct) as baselines (Section 3, Tables 2 and 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary models: InjecGuard (2024), PromptGuard (Meta, 2024), ProtectAIv2 (2024), GPT-4o (2024), and Llama3.2-1B-Instruct (Meta, 2024). These represent current state-of-the-art prompt injection detectors."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper does not analyze the individual contributions of the Framework, Separator, or Disruptor components, nor does it ablate the training data composition for CaptureGuard."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses FNR (False Negative Rate) and FPR (False Positive Rate) as two complementary metrics (Tables 2, 3), plus accuracy on external benchmarks (Table 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human validation is mentioned briefly ('human validation showed approximately 90% agreement with its malicious/benign classifications' in Section 3), but no detailed human evaluation of the system's outputs or the benchmark quality is presented. The details of this validation (who, how many, protocol) are absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes explicit train/test/validation splits: '30 train, 15 test, 15 validation examples' for base sets (Section 2.1), with separate test sets used for evaluation in Tables 2, 3, and external benchmarks in Table 6."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (Stock, Movies, Python, Travel, Covid, Shopping) across all six domains in Tables 2 and 3, and by benchmark (NotInject, WildGuard, BIPIA) in Table 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis is provided. The paper reports aggregate FNR/FPR numbers but does not examine specific examples where CaptureGuard fails, nor analyze patterns in its errors."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CaptureGuard underperforms InjecGuard on NotInject benchmark (79.04% vs 87.31%) and on BIPIA Injection (54.77% vs 68.34%), as shown in Table 6. The paper acknowledges 'slightly lower than InjecGuard's 87.32%, indicating a marginal trade-off in benign prompt detection.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that existing models 'suffer from high false negatives in adversarial cases and excessive false positives in benign scenarios' — supported by Tables 2-3 (e.g., Fmops 100% FNR, PromptGuard 100% FPR). The claim that CaptureGuard 'drastically reduces both false negative and false positive rates' is supported by its near-zero FNR and low FPR in Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CaptureGuard's performance improvement results from 'context-aware training data' (Section 3), implying a causal link. However, no ablation isolates this factor — CaptureGuard also uses InjecGuard's 26 existing datasets alongside CAPTURE data, making it unclear which data source drives the improvement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion make broad claims about 'more robust and practical prompt injection defenses' and 'a clear methodology...to advance the field,' but CaptureGuard was only trained and tested on three domains (Python, Movies, Stocks) out of six. The paper's title implies general context-awareness but results are limited to specific domain configurations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 5) discusses the key alternative explanation: GPT-4o was used for both data generation and as an evaluation baseline, introducing potential bias where 'generated data may inadvertently reflect the stylistic and logical patterns of the generator model, potentially giving GPT-4o an advantage in detection.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o' and 'Llama3.2-1B-Instruct' without specifying API versions or snapshot dates. GPT-4o behavior changes across versions, so 'GPT-4o' alone is insufficient. No version date or API endpoint is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in the Appendix (Figures 2-5) for all key generation and evaluation tasks: SAFE-GEN generation (Figure 2), S/D identification (Figure 3), subtle separator generation (Figure 4), and LLM evaluation (Figure 5)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports temperature settings for each GPT-4o task (0, 0.5, 0.7). Table 5 provides CaptureGuard hyperparameters: batch size 32, learning rate 2e-5, max sequence length 64, optimizer Adam, 1 epoch, classification threshold 0.5."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a data generation pipeline plus standard classification model training, not an agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data generation pipeline is documented in detail across Sections 2.1-2.3: sourcing from public QA datasets, expansion via GPT-4o (100 examples per domain per split), decomposition of attacks into S and D components, refinement of separators, and final assembly. Dataset sizes at each stage are provided in Table 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'LIMITATIONS' is a dedicated limitations section with substantive discussion of scope, attack diversity constraints, and generator model bias."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) focus limited to 'direct, single-turn prompt injections' excluding 'indirect and multi-turn attacks,' (2) 'reliance on a single powerful model, GPT-4o, for both data generation and as an evaluation baseline' creating potential bias, (3) 'attack diversity is constrained by the source datasets used.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what is not tested: indirect attacks, multi-turn attacks, and non-conversational LLM applications. It states the scope is limited to 'direct, single-turn prompt injections' and acknowledges the benchmark is 'not limited to conversational LLM applications' as a future goal."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated datasets (MALICIOUS-GEN, SAFE-GEN) are not released. Only example prompts are shown in Table 1. The raw data is not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection process is described in detail: source QA datasets are named (Shopping, Covid, Movies, Stock, Travel, Python Code with citations), the expansion process via GPT-4o is documented (Section 2.1), and attack source datasets are cited (Erdogan et al., 2024; Schulhoff et al., 2023; Yugen.ai, 2023)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data is entirely synthetic, generated from public datasets and GPT-4o."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from source data to final datasets is documented: (1) source QA datasets → (2) GPT-4o expansion to 100 per domain/split → (3) attack decomposition into S/D → (4) separator refinement → (5) assembly into context-aware prompts. Dataset sizes are given at each stage (Section 2.1-2.3, Table 4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper. Both authors are affiliated with Pure Storage."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: both Gauri Kholkar and Ratinder Ahuja are affiliated with Pure Storage, with institutional email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The work comes from Pure Storage, a cloud storage company. While Pure Storage does not appear to be a direct competitor to the evaluated guardrail models, the lack of any funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4o and Llama3.2-1B-Instruct as prompt injection detectors, and uses GPT-4o for data generation, but does not state the training data cutoff for any model. This is relevant because GPT-4o may have seen the source attack datasets in its training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs (GPT-4o, Llama3.2) may have seen the source attack datasets (HackAPrompt, Safe-Guard, etc.) during pre-training. The paper also does not address whether GPT-4o's data generation creates a favorable evaluation setting for itself beyond the brief mention in limitations."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The external benchmarks used for evaluation (NotInject, WildGuard, BIPIA) were published before GPT-4o's likely training cutoff, but this contamination risk is not discussed. The paper does not address whether GPT-4o may have seen these benchmarks during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The data is entirely synthetic."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed is reported. The paper uses GPT-4o for data generation and evaluation but does not quantify the associated API costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report GPU hours for CaptureGuard training, total API spend for GPT-4o usage, or hardware specifications."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing prompt guardrail models suffer from high false negatives in context-aware adversarial settings (e.g., Fmops achieves 100% FNR across all domains).",
       "evidence": "Tables 2 and 3 show Fmops with 100% FNR across all six domains, InjecGuard with 99.84-100% FNR in most domains, and Llama3.2-1B-Instruct with 58-76% FNR.",
       "supported": "strong"
     },
     {
       "claim": "Existing guardrail models exhibit excessive false positive rates on benign context-aware prompts (e.g., PromptGuard achieves 100% FPR in Stock and Movies).",
       "evidence": "Tables 2 and 3 show PromptGuard with 100% FPR in Stock, Movies, Travel, Covid, and Shopping domains, and InjecGuard with 98-100% FPR in most domains.",
       "supported": "strong"
     },
     {
       "claim": "CaptureGuard achieves near-zero FNR (0.00-0.15%) and low FPR (0.00-2.05%) on the CAPTURE benchmark.",
       "evidence": "Table 2 shows CaptureGuard FNR of 0.15%, 0.00%, 0.00% and FPR of 0.00%, 2.05%, 2.05% for Stock, Movies, and Python domains respectively.",
       "supported": "moderate"
     },
     {
       "claim": "CaptureGuard generalizes effectively to external benchmarks, achieving competitive performance.",
       "evidence": "Table 6 shows CaptureGuard achieving 79.04% on NotInject, 75.00% on WildGuard, and 54.77% on BIPIA Injection. However, it underperforms InjecGuard on all three benchmarks (87.31%, 76.11%, 68.34% respectively).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CAPTURE reveals that existing prompt injection guardrail models are highly vulnerable to context-aware attacks, with models like Fmops showing 100% false negative rates and PromptGuard showing 100% false positive rates. The proposed CaptureGuard model, trained on CAPTURE's context-aware generated data, achieves near-zero FNR and FPR on the in-domain benchmark but shows weaker generalization to external benchmarks (NotInject, WildGuard, BIPIA), where it underperforms InjecGuard. The work highlights a fundamental tradeoff between attack detection sensitivity and over-defense in existing guardrail models.",
   "red_flags": [
     {
       "flag": "Generator-evaluator conflation",
       "detail": "GPT-4o is used both to generate the benchmark data and as an evaluation baseline. The paper acknowledges this but does not adequately control for it — GPT-4o's strong performance on detection (low FNR, low FPR) may partly reflect familiarity with its own generated patterns rather than genuine detection capability."
     },
     {
       "flag": "In-domain evaluation advantage",
       "detail": "CaptureGuard's impressive near-zero FNR/FPR is measured on the same CAPTURE benchmark it was trained on (same domains, same generation pipeline). On external benchmarks (Table 6), CaptureGuard underperforms InjecGuard on all three tests, suggesting limited generalization despite claims of 'generalizing effectively.'"
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, or variance across multiple training seeds. Given the relatively small test set sizes (e.g., 171 safe samples), individual classification decisions can substantially shift FPR."
     },
     {
       "flag": "Missing ablation",
       "detail": "CaptureGuard is trained on both CAPTURE-generated data AND InjecGuard's 26 existing datasets. No ablation isolates the contribution of the context-aware CAPTURE data versus the existing datasets, making the causal claim about context-aware training data unsubstantiated."
     },
     {
       "flag": "Code and data not released",
       "detail": "Despite claims of providing 'a clear methodology and a powerful baseline to advance the field,' neither the code nor the datasets are released. The promise of future release upon publication does not allow independent verification."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications, directly relevant to prompt injection security research."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Systematic formalization and benchmarking of prompt injection attacks and defenses, published at USENIX Security."
     },
     {
       "title": "InjecGuard: Benchmarking and mitigating over-defense in prompt injection guardrail models",
       "authors": ["Hao Li", "Xiaogeng Liu"],
       "year": 2024,
       "arxiv_id": "2410.22770",
       "relevance": "Key baseline for prompt injection detection that identifies the over-defense problem; provides the training framework adopted by CaptureGuard."
     },
     {
       "title": "Prompt injection attack against llm-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Introduces the Framework-Separator-Disruptor attack structure for context-aware prompt injection that CAPTURE builds upon."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin"],
       "year": 2024,
       "arxiv_id": "2312.17673",
       "relevance": "Proposes task-specific fine-tuning as a defense against prompt injection, relevant to evaluating defense strategies."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Benchmarking indirect prompt injection attacks and defenses on LLMs, directly relevant to the survey's coverage of LLM security evaluation."
     },
     {
       "title": "PromptShield: Deployable detection for prompt injection attacks",
       "authors": ["Dennis Jacob", "Hend Alzahrani", "Zhanhao Hu", "Basel Alomair", "David Wagner"],
       "year": 2025,
       "arxiv_id": "2501.15145",
       "relevance": "Recent prompt injection detection system relevant to evaluating the state of deployable guardrail defenses."
     },
     {
       "title": "Ignore this title and hackaprompt: Exposing systemic vulnerabilities of llms through a global prompt hacking competition",
       "authors": ["Sander Schulhoff", "Jeremy Pinto", "Anaum Khan"],
       "year": 2023,
       "relevance": "Large-scale prompt hacking competition providing attack techniques and datasets used as source data for CAPTURE's MALICIOUS-GEN pipeline."
     },
     {
       "title": "WildGuard: Open one-stop moderation tools for safety risks, jailbreaks, and refusals of llms",
       "authors": ["Seungju Han", "Kavel Rao", "Allyson Ettinger"],
       "year": 2024,
       "arxiv_id": "2406.18495",
       "relevance": "Open-source LLM moderation toolkit used as an external benchmark for evaluating guardrail model generalization."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Early systematic study of prompt injection attack techniques against language models."
     }
   ]
 }