ai-research-survey

scan.json (24679B)

---

 {
   "paper": {
     "title": "Prompt Injection Attacks in Large Language Models via a Comprehensive Analysis of Attack Vectors, Defense Mechanisms, and Future Directions",
     "authors": ["Ilkin Javadov"],
     "year": 2025,
     "venue": "Proceedings of IAM, V.14, N.2, 2025",
     "doi": "10.30546/2225-0530.14.2.2025.2013"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Empirical evaluation on GPT-4, Claude 3 Opus, and Gemini 1.5 Pro using 1,000 attack scenarios shows baseline vendor safeguards provide ~58% protection (42% ASR). A proposed six-layer defense-in-depth framework reduces ASR to 3.2% ± 1.1%. Ablation identifies architectural separation and multi-model verification as the key contributors. Analysis of five real-world incidents (2023–2025) documents financial losses up to $2.3M and detection delays up to 45 days.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The curated dataset of 1,000 attack scenarios is not released. No download link or data availability statement is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. A researcher could not replicate the experiments from the information given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports '58% ± 3.2%' for baseline protection and '3.2% ± 1.1%' for the proposed framework, described as '95% CI' in the methods section."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'p < 0.001 vs. baseline' and states 'paired t-tests' were used in the statistical methods."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports '92.4% relative reduction' in ASR, with baseline context (from 42% ASR to 3.2% ASR), providing magnitude in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,000 attack scenarios but provides no justification for this number. No power analysis or rationale for why 1,000 is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states '3 runs/scenario for variance' and reports ± values (e.g., '3.2% ± 1.1%') across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against baseline vendor safeguards (~58% protection) and state-of-the-art single-layer defenses (~82%), establishing comparison points for the proposed framework."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include current vendor safeguards and reference recent techniques: spotlighting [15], multi-model verification [16], Azure AI Content Safety prompt shields [18], and Anthropic's alignment methods [17]."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "An ablation is claimed ('Ablation confirms architectural separation and multi-model verification as key contributors') but no ablation data, tables, or per-layer contribution figures are shown anywhere in the paper."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports attack success rate (ASR), false-positive rate (~5.3%), and latency overhead (+18%), providing multiple evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Attack success was determined via 'regex + manual review, Kappa=0.92' — human reviewers assessed whether model outputs achieved adversarial goals."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No train/test split or held-out evaluation is described. All 1,000 scenarios appear to be used for evaluation without separation from any tuning of defense parameters."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The dataset composition is described (40% direct, 30% indirect, etc.) but results are only reported as aggregates. No breakdown of ASR by attack category, model, or defense layer."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The residual 3.2% failure rate is acknowledged but no specific failure cases are analyzed. No examples of attacks that bypassed the defense framework are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. No discussion of approaches that were tried and failed or configurations that didn't work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (58% baseline, 82% single-layer, 3.2% framework ASR) are restated in the results and conclusion sections with matching numbers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that the defense framework 'reduces' ASR is supported by controlled comparison: same 1,000 scenarios tested with and without defense layers. The ablation design (removing components) is an adequate causal identification strategy, though the ablation data itself is not shown."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only GPT-4 (0613), Claude 3 Opus, and Gemini 1.5 Pro, but the title and conclusion make broad claims about 'Large Language Models' and 'modern AI systems' without bounding to the three tested models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider confounds such as whether the defense framework's effectiveness is specific to the attack dataset used, or whether the 1,000 scenarios are representative of real-world attacks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures ASR (attack success rate) and frames results in terms of ASR and protection rate. The measured proxies match the claims — no inflated framing beyond what was measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'GPT-4 (0613, OpenAI API)', 'Claude 3 Opus (v1.2)', 'Gemini 1.5 Pro (2025-11)'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual attack prompts or system prompts are provided. The attack scenarios are described categorically (direct, indirect, etc.) but no example prompts are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are stated: 'Temp=0.7, max_tokens=512, safety on.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The six-layer defense architecture is a filter pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Dataset construction is described only as '1,000 scenarios manually crafted + validated ... Sources: augmented from GLUE/Alpaca + taxonomy-based attacks.' How GLUE/Alpaca (NLP benchmarks) were converted to attack scenarios is unexplained. Filtering and augmentation steps are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion mentions 'residual 3.2% vulnerability highlights fundamental limitations in current LLM designs' but this discusses general LLM limitations, not methodological limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. No mention of dataset representativeness, potential biases in manual annotation, or external validity of results from three models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show or what settings/models/attack types are excluded from the conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 1,000 attack scenarios and raw model outputs are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Data collection is described only superficially: 'manually crafted + validated' and 'augmented from GLUE/Alpaca + taxonomy-based attacks.' The actual crafting methodology, augmentation process, and validation criteria are not explained."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Three annotators validated the dataset (Kappa=0.92) but their qualifications, expertise, and selection are not described. The provenance and representativeness of the 1,000 scenarios is undocumented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The paper does not explain how raw GLUE/Alpaca data was transformed into attack scenarios, what filtering was applied, or how the 40%/30% category distribution was determined."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is listed: Azerbaijan Technical University, Baku, Azerbaijan. The author is not affiliated with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding from outcomes cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests defense mechanisms against prompt injection attacks rather than evaluating model knowledge on a benchmark. Contamination is structurally inapplicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defense mechanisms, not model capability on knowledge-based benchmarks. Train/test overlap is not a relevant concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates attack success rates against defenses, not model knowledge retrieval. Benchmark contamination is structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The 3 annotators validated the dataset but were not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Latency overhead is reported: 'Latency +18% avg.' for the defense framework. However, no monetary cost or token consumption figures are given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API spend, hardware) is stated for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper runs '3 runs/scenario for variance' and reports results with ± values, capturing run-to-run variation from temperature sampling."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: '3 runs/scenario for variance.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described for the defense framework components. Thresholds and parameters for the 6 defense layers appear untuned or tuned without documentation."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents one configuration of the 6-layer framework without explaining how parameters were selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "Only one formal statistical comparison is reported (framework vs. baseline, p<0.001). Multiple comparison correction is not needed."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed and evaluated their own defense framework without acknowledging author-evaluation bias or having independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Latency overhead (+18%) is mentioned but performance is not shown as a function of compute budget. No analysis of how each layer's cost contributes to the overall protection gain."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 1,000 curated attack scenarios are representative of real-world prompt injection attacks or have construct validity for measuring defense effectiveness."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Three models are tested but it is unclear whether the defense framework was applied identically across all three. No explicit discussion of whether model-specific factors confound the aggregate results."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' safety training already included defenses against these specific attack patterns, which would be a form of temporal leakage for defense evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence among the 1,000 attack scenarios. Scenarios 'augmented from GLUE/Alpaca' may share structural patterns that inflate or deflate success rates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used or described."
       }
     }
   },
   "claims": [
     {
       "claim": "Baseline vendor safeguards provide approximately 58% ± 3.2% protection against prompt injection (42% ASR)",
       "evidence": "Stated in abstract and conclusion. Evaluation on GPT-4 (0613), Claude 3 Opus, Gemini 1.5 Pro with 1,000 attack scenarios. No detailed results table is shown.",
       "supported": "moderate"
     },
     {
       "claim": "State-of-the-art single-layer defenses reach at most ~82% detection but with elevated false positive rates",
       "evidence": "Stated in abstract and conclusion, referencing reported detection rates of '67-82%' from [9,14]. No detailed per-defense results shown.",
       "supported": "weak"
     },
     {
       "claim": "The proposed six-layer defense-in-depth framework reduces ASR to 3.2% ± 1.1% (92.4% relative reduction over baseline, p<0.001)",
       "evidence": "Statistical significance reported (p<0.001, paired t-tests). 95% CIs provided. 3 runs per scenario. However, no detailed results table or per-layer breakdown is shown.",
       "supported": "moderate"
     },
     {
       "claim": "Ablation confirms architectural separation and multi-model verification as key contributors to defense effectiveness",
       "evidence": "One-sentence claim in abstract and conclusion. No ablation data, tables, or per-component contribution figures are presented anywhere in the paper.",
       "supported": "unsupported"
     },
     {
       "claim": "The defense framework maintains acceptable false-positive rates (~5.3%) and moderate latency (+18% average)",
       "evidence": "Stated in conclusion. No breakdown of FP rates by layer or attack type. No latency measurements shown.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Skeletal results section with no data",
       "detail": "The Results section (Section 3) contains only the experimental setup (dataset, models, procedure, stats) but no actual results tables, figures, or detailed findings. All quantitative claims appear only in the abstract and conclusion with no supporting evidence shown in between."
     },
     {
       "flag": "Ablation claimed but not shown",
       "detail": "The paper repeatedly claims ablation studies were conducted and names key components, but presents zero ablation data. For a six-layer architecture, showing per-layer contributions is essential."
     },
     {
       "flag": "Scrambled reference list",
       "detail": "In-text citations do not match the reference list. For example, '[3] Greshake et al.' in the text refers to the indirect prompt injection paper, but reference [3] is Brown et al. 'Language Models are Few-Shot Learners.' Reference [8] is actually Greshake et al. This suggests careless reference management."
     },
     {
       "flag": "Unexplained dataset provenance",
       "detail": "The paper claims 1,000 attack scenarios were 'augmented from GLUE/Alpaca + taxonomy-based attacks.' GLUE and Alpaca are NLP benchmarks/instruction-following datasets, not attack datasets. How they were converted to prompt injection attacks is completely unexplained."
     },
     {
       "flag": "No per-model or per-category results",
       "detail": "Three models are tested and 11 attack vectors are taxonomized, but all results are reported as single aggregate numbers. There is no breakdown showing which models are more vulnerable or which attack types are more successful."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The conclusion makes sweeping claims about 'actionable insights, empirical benchmarks, and a deployable framework' but the paper provides no code, no data, no detailed results, and no deployment guidelines beyond high-level descriptions."
     },
     {
       "flag": "Taxonomy claims 11 vectors but shows 7",
       "detail": "The abstract mentions '7 attack vectors' but Section 2.4 claims '11 attack vectors (Table 1).' Table 1 shows only 7 rows. The discrepancy is not acknowledged or explained."
     }
   ],
   "cited_papers": [
     {
       "title": "More than you've asked for: A Comprehensive Analysis of Novel Prompt Injection Threats to Application-Integrated Large Language Models",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Foundational work on indirect prompt injection via external content in LLM-integrated applications."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "relevance": "Demonstrates practical indirect prompt injection attacks against real-world LLM applications."
     },
     {
       "title": "Jailbreaking ChatGPT via Prompt Engineering: An Empirical Study",
       "authors": ["Y. Liu", "G. Deng", "Z. Xu", "Y. Li", "Y. Zheng", "Y. Zhang", "L. Zhao", "T. Zhang", "Y. Liu"],
       "year": 2023,
       "arxiv_id": "2305.13860",
       "relevance": "Empirical study of jailbreaking techniques against ChatGPT through prompt engineering."
     },
     {
       "title": "A Comprehensive Framework for Understanding and Mitigating Prompt Injection Attacks",
       "authors": ["W. Liu", "Y. Wang", "X. Chen", "H. Li", "J. Zhang", "Y. Zhao"],
       "year": 2024,
       "relevance": "Provides a formal framework and mitigation strategies for prompt injection, published at USENIX Security."
     },
     {
       "title": "Adversarial Worms: Self-Replicating Prompt Injection in Multi-Agent LLM Systems",
       "authors": ["S. Kumar", "R. Patel", "L. Zhang", "M. Williams"],
       "year": 2025,
       "arxiv_id": "2501.00234",
       "relevance": "Explores emerging self-replicating prompt injection threats in multi-agent LLM systems."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["R. Bommasani", "D.A. Hudson", "E. Adeli"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive analysis of foundation model risks including security vulnerabilities."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper demonstrating LLM capabilities that enabled the applications now targeted by prompt injection."
     }
   ]
 }