ai-research-survey

scan.json (21840B)

---

 {
   "paper": {
     "title": "Prompt Injection Attacks on Agentic Coding Assistants: A Systematic Analysis of Vulnerabilities in Skills, Tools, and Protocol Ecosystems",
     "authors": [
       "Narek Maloyan",
       "Dmitry Namiot"
     ],
     "year": 2026,
     "arxiv_id": "2601.17548"
   },
   "scan_version": 2,
   "active_modules": [
     "survey_methodology"
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This SoK synthesizes 78 studies on prompt injection attacks against agentic coding assistants, proposing a three-dimensional taxonomy (delivery vector, modality, propagation). Key findings: 85%+ of attacks compromise at least one major platform, adaptive attacks bypass 90%+ of published defenses, and skill ecosystems (Claude Code, Copilot Extensions, MCP) lack adequate security review. The paper argues prompt injection is architecturally fundamental — unlike SQL injection, no parameterization fix exists.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is mentioned. The paper states 'Attack code is not released; techniques are described at the conceptual level' (Section VIII.D)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of the 78 surveyed papers or extracted data is released. The paper could have released its corpus or analysis data but did not."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/SoK paper with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The search methodology is described (Section I.A) but no scripts, queries, or detailed steps for reproducing the literature review are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that synthesizes reported statistics from other studies; does not run its own experiments."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made by this paper; it reports statistics from primary sources."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper without its own empirical experiments."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no experimental sample sizes."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; does not run its own experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its taxonomy or framework against prior survey/SoK efforts in a structured way. Related work is discussed but not benchmarked against."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experimental evaluation with baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation performed."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a literature synthesis."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-platform vulnerability breakdowns (Table IV), per-defense bypass rates (Table III), and per-attack-category analysis throughout Section V."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses where defenses fail — Table III shows defense bypass rates, and Section VI details fundamental limitations of each defense category."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central finding is negative: all evaluated defenses are bypassable. Section VI.B documents that adaptive attacks bypass all 12 defenses with 78-93% success rates."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '42 distinct attack techniques' and '18 defense mechanisms' but the body describes '31 attack techniques' (Section I contributions item 3) and '12 defense mechanisms' (Section I contributions item 4). These numbers are inconsistent between abstract and body."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims; it synthesizes and categorizes existing findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII.F explicitly bounds scope: 'Rapid Evolution' (findings may be outdated), 'Closed-Source Systems' (limited visibility), 'Benchmark Validity' (may not reflect real-world attacks), 'Selection Bias' in published attacks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII.F discusses alternative explanations: benchmark validity ('Existing benchmarks may not reflect real-world attack sophistication'), selection bias in published attacks, and that adaptive defenses are understudied."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a Systematization of Knowledge (SoK) survey paper that synthesizes findings from 78 studies. It reports statistics from primary sources (e.g., '85%+ attack success rates') rather than making claims based on its own measurements. No proxy-outcome gap applies to a literature synthesis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no models are run by the authors."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting experiments conducted."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic system built or used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section I.A describes the pipeline: 183 initial results → 78 primary sources, with three inclusion criteria listed. However, the filtering criteria are generic ('addresses LLM-integrated systems', 'presents novel attacks/defenses', 'peer-reviewed or established venues') and counts per stage are not provided for intermediate steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII.F 'Limitations of This Study' provides a dedicated limitations section with five specific bullet points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII.F lists specific threats: rapid field evolution, closed-source system opacity, benchmark validity concerns, adaptive defense understudied, and selection bias in published attacks. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: restricted to 2024-2025 publications (Section I.A), focused on agentic AI era, and Section VIII.F states what the results do not show (e.g., real-world attack sophistication, adaptive defense effectiveness)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The list of 78 surveyed papers is not released as a structured dataset. No supplementary materials or data files are mentioned."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section I.A describes data collection: searched arXiv, IEEE Xplore, ACM DL, USENIX with specific query terms, restricted to Jan 2024–Dec 2025, 183 initial results yielding 78 primary sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is published literature."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from 183 to 78 papers is mentioned but intermediate filtering stages with counts are not documented. The abstract claims 78 studies but the body text also references works outside this scope without clarifying which are primary vs. supplementary sources."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author names are listed but no institutional affiliations are provided in the extracted text."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation conducted."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational method to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "Section I.A describes a structured search with query terms and inclusion criteria, but does not follow PRISMA, provide a flow diagram, or reference a registered protocol. The search methodology is described in one paragraph without reproducible query strings or database-specific search strategies."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of its 78 source papers. Attack success rates and defense evaluations are cited directly from primary sources without quality weighting or risk-of-bias assessment. All sources are treated equally regardless of methodology."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII.F explicitly discusses selection bias: 'Published attacks may represent a biased sample. Successful attacks by sophisticated actors may never be disclosed.' This acknowledges publication bias in the attack literature."
       }
     }
   },
   "claims": [
     {
       "claim": "Attack success rates against state-of-the-art defenses exceed 85% when adaptive attack strategies are employed",
       "evidence": "Drawn from MCPSecBench [43] and Nasr et al. [70]; Table III shows adaptive bypass rates of 78-93% across 6 defense systems. Section VII.A reports 85%+ compromise rate.",
       "supported": "moderate"
     },
     {
       "claim": "42 distinct attack techniques are cataloged spanning input manipulation, tool poisoning, protocol exploitation, multimodal injection, and cross-origin context poisoning",
       "evidence": "The abstract claims 42 techniques, but Section I contribution item 3 states '31 attack techniques from the literature.' The taxonomy in Section IV lists categories but the count is inconsistent.",
       "supported": "weak"
     },
     {
       "claim": "Most defenses achieve less than 50% mitigation against sophisticated adaptive attacks",
       "evidence": "Table III shows that under adaptive attacks, reported low attack success rates (<3-12%) jump to 78-93%. However, this data is from Nasr et al. [70], not independent replication.",
       "supported": "moderate"
     },
     {
       "claim": "73% of tested platforms fail to adequately enforce at least one trust boundary",
       "evidence": "Stated in Section III.C but no methodology or data source is cited for this specific statistic. It appears to be the authors' assessment from platform analysis.",
       "supported": "weak"
     },
     {
       "claim": "Skill ecosystems are under-secured with inadequate security review and capability restriction",
       "evidence": "Table I compares platforms showing Claude Code and Cursor have no marketplace review. Section V.C documents concrete exploit chains. Supported by CVE evidence in Table II.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract-body inconsistency",
       "detail": "Abstract claims '42 distinct attack techniques' and '18 defense mechanisms' but the body states '31 attack techniques' and '12 defense mechanisms.' These discrepancies undermine trust in the paper's precision."
     },
     {
       "flag": "No independent replication",
       "detail": "The paper explicitly states 'we did not conduct independent replication experiments' (Section I.A). All quantitative claims about attack success rates are drawn directly from primary sources, particularly MCPSecBench and Nasr et al."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "The 78 source papers are treated equally without methodological quality assessment. Attack success rates from different studies with varying rigor are synthesized as equivalent evidence."
     },
     {
       "flag": "Unsourced statistic",
       "detail": "The claim that '73% of tested platforms fail to adequately enforce at least one trust boundary' (Section III.C) lacks a citation or methodology description."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": [
         "K. Greshake",
         "S. Abdelnabi",
         "S. Mishra",
         "C. Endres",
         "T. Holz",
         "M. Fritz"
       ],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection against LLM-integrated applications."
     },
     {
       "title": "MCPSecBench: A systematic security benchmark and playground for testing Model Context Protocols",
       "authors": [
         "Y. Yang",
         "D. Wu",
         "Y. Chen"
       ],
       "year": 2025,
       "arxiv_id": "2508.13220",
       "relevance": "Standardized benchmark for MCP security evaluation with 17 attack types across 4 surfaces."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": [
         "E. Debenedetti",
         "J. Zhang",
         "M. Balunovic",
         "L. Beurer-Kellner",
         "M. Fischer",
         "F. Tramèr"
       ],
       "year": 2024,
       "relevance": "Dynamic evaluation environment with 97 tasks and 629 security test cases for agent prompt injection."
     },
     {
       "title": "The attacker moves second: Stronger adaptive attacks bypass defenses against LLM jailbreaks and prompt injections",
       "authors": [
         "M. Nasr",
         "N. Carlini",
         "C. Sitawarin",
         "S. Schulhoff",
         "J. Hayes"
       ],
       "year": 2025,
       "arxiv_id": "2510.09023",
       "relevance": "Demonstrates all 12 evaluated defenses bypassed with 90%+ success, establishing lower bound on achievable security."
     },
     {
       "title": "Agent Security Bench (ASB): Formalizing and benchmarking attacks and defenses in LLM-based agents",
       "authors": [
         "H. Zhang",
         "J. Huang",
         "K. Mei"
       ],
       "year": 2025,
       "relevance": "Comprehensive framework for evaluating agent attack/defense effectiveness with attack success rates up to 84.3%."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated LLM agents",
       "authors": [
         "Q. Zhan",
         "R. Fang",
         "R. Bindu",
         "A. Gupta",
         "T. Hashimoto",
         "D. Kang"
       ],
       "year": 2024,
       "relevance": "Benchmark evaluating 30 LLM agents for indirect prompt injection vulnerability rates up to 47%."
     },
     {
       "title": "The instruction hierarchy: Training LLMs to prioritize privileged instructions",
       "authors": [
         "E. Wallace",
         "K. Xiao",
         "R. Leike",
         "L. Weng",
         "J. Heidecke",
         "A. Beutel"
       ],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Proposes training LLMs to prioritize instruction sources as a defense against prompt injection."
     },
     {
       "title": "Defeating prompt injections by design",
       "authors": [
         "E. Debenedetti"
       ],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "CaMeL framework achieving provable security on 77% of AgentDojo tasks through capability-based isolation."
     },
     {
       "title": "Progent: Programmable privilege control for LLM agents",
       "authors": [
         "T. Shi",
         "J. He",
         "Z. Wang"
       ],
       "year": 2025,
       "arxiv_id": "2504.11703",
       "relevance": "Programmable privilege control reducing attack success from 41.2% to 2.2%."
     },
     {
       "title": "XOXO: Stealthy cross-origin context poisoning attacks against AI coding assistants",
       "authors": [
         "A. Storek",
         "M. Gupta",
         "N. Bhatt"
       ],
       "year": 2025,
       "arxiv_id": "2503.14281",
       "relevance": "Semantic attacks through cross-origin context poisoning specifically targeting AI coding assistants."
     },
     {
       "title": "IDEsaster: Security vulnerabilities in AI-powered integrated development environments",
       "authors": [
         "A. Marzouk"
       ],
       "year": 2025,
       "relevance": "Documented 30+ CVEs across major AI IDEs including Cursor, Copilot, and Codex CLI."
     },
     {
       "title": "Agentic AI security: Threats, defenses, evaluation, and open challenges",
       "authors": [
         "S. Datta",
         "S. K. Nahin",
         "A. Chhabra",
         "P. Mohapatra"
       ],
       "year": 2025,
       "arxiv_id": "2510.23883",
       "relevance": "Comprehensive survey of agentic AI security with threat modeling for autonomous systems."
     },
     {
       "title": "AgentHarm: A benchmark for measuring harmfulness of LLM agents",
       "authors": [
         "M. Andriushchenko"
       ],
       "year": 2025,
       "relevance": "Benchmark for evaluating harmful capabilities of LLM agents."
     }
   ]
 }