scan.json (21709B)
1 { 2 "paper": { 3 "title": "Systematization of Knowledge: Security and Safety in the Model Context Protocol Ecosystem", 4 "authors": [ 5 "Shiva Gaire", 6 "Srijan Gyawali", 7 "Saroj Mishra", 8 "Suman Niroula", 9 "Dilip Thakur", 10 "Umesh Yadav" 11 ], 12 "year": 2025, 13 "venue": "arXiv", 14 "arxiv_id": "2512.08290" 15 }, 16 "scan_version": 2, 17 "active_modules": ["survey_methodology"], 18 "methodology_tags": ["meta-analysis", "theoretical"], 19 "key_findings": "This SoK provides the first academic taxonomy of security and safety risks in the Model Context Protocol ecosystem, distinguishing adversarial security threats (tool poisoning, prompt injection, supply-chain attacks) from epistemic safety hazards (alignment failures, hallucination-driven tool execution). The paper identifies that MCP's coupling of context (Resources) and action (Tools) creates a 'semantic attack surface' where indirect prompt injection can escalate into real-world operational damage. It surveys defenses including ETDI (cryptographic tool provenance), capability-based access control, session isolation via gVisor/seL4, and proposes a tiered trust model for future MCP ecosystems.", 20 "checklist": { 21 "artifacts": { 22 "code_released": { 23 "applies": true, 24 "answer": false, 25 "justification": "No source code, analysis scripts, or repository links are provided anywhere in the paper." 26 }, 27 "data_released": { 28 "applies": true, 29 "answer": false, 30 "justification": "No dataset or structured corpus of the surveyed literature is released. The paper does not provide a downloadable bibliography or extracted taxonomy data." 31 }, 32 "environment_specified": { 33 "applies": false, 34 "answer": false, 35 "justification": "This is a survey/SoK paper with no computational experiments requiring an environment specification." 36 }, 37 "reproduction_instructions": { 38 "applies": true, 39 "answer": false, 40 "justification": "No instructions are provided for reproducing the literature search or taxonomy construction process." 41 } 42 }, 43 "statistical_methodology": { 44 "confidence_intervals_or_error_bars": { 45 "applies": false, 46 "answer": false, 47 "justification": "Survey paper with no statistical analysis of quantitative data." 48 }, 49 "significance_tests": { 50 "applies": false, 51 "answer": false, 52 "justification": "No quantitative comparisons requiring significance tests." 53 }, 54 "effect_sizes_reported": { 55 "applies": false, 56 "answer": false, 57 "justification": "No experiments producing effect sizes." 58 }, 59 "sample_size_justified": { 60 "applies": false, 61 "answer": false, 62 "justification": "No experiments or data collection with a sample size to justify." 63 }, 64 "variance_reported": { 65 "applies": false, 66 "answer": false, 67 "justification": "No experimental runs producing variance." 68 } 69 }, 70 "evaluation_design": { 71 "baselines_included": { 72 "applies": true, 73 "answer": false, 74 "justification": "The paper does not compare its taxonomy or SoK approach against prior surveys or systematizations. No baseline survey is identified and compared." 75 }, 76 "baselines_contemporary": { 77 "applies": false, 78 "answer": false, 79 "justification": "No baselines are included, so contemporaneity is not assessable." 80 }, 81 "ablation_study": { 82 "applies": false, 83 "answer": false, 84 "justification": "No system with components to ablate; this is a survey paper." 85 }, 86 "multiple_metrics": { 87 "applies": false, 88 "answer": false, 89 "justification": "No quantitative evaluation is performed." 90 }, 91 "human_evaluation": { 92 "applies": false, 93 "answer": false, 94 "justification": "No system outputs to evaluate; this is a survey." 95 }, 96 "held_out_test_set": { 97 "applies": false, 98 "answer": false, 99 "justification": "No data splits; this is a survey." 100 }, 101 "per_category_breakdown": { 102 "applies": true, 103 "answer": true, 104 "justification": "The paper provides detailed per-category breakdown of threats in Table III (vulnerability taxonomy by category, impact, phase, evidence source) and Table II (security vs safety dimensions across integrity, confidentiality, availability)." 105 }, 106 "failure_cases_discussed": { 107 "applies": true, 108 "answer": true, 109 "justification": "Section VIII presents detailed case studies of failures: the Supabase data leak incident, multi-tenant KV-cache sharing attacks, and policy conflict failures in enterprise AI pipelines." 110 }, 111 "negative_results_reported": { 112 "applies": true, 113 "answer": true, 114 "justification": "The paper reports that current defenses are insufficient (Section IV.I), that over 43% of MCP server implementations execute unsafe shell calls (Section VII.A), and that existing safety alignment techniques fail on long contexts (Section VII.C)." 115 } 116 }, 117 "claims_and_evidence": { 118 "abstract_claims_supported": { 119 "applies": true, 120 "answer": true, 121 "justification": "The abstract claims to provide a comprehensive taxonomy (delivered in Tables II-III), analyze structural vulnerabilities of MCP primitives (Sections IV-V), survey defenses (Section VI), and conclude with a roadmap (Sections VII, IX). These are all present in the paper body." 122 }, 123 "causal_claims_justified": { 124 "applies": false, 125 "answer": false, 126 "justification": "The paper is a survey/taxonomy that does not make causal claims about empirical relationships. It describes threat mechanisms and defense architectures rather than claiming 'X causes Y' from data." 127 }, 128 "generalization_bounded": { 129 "applies": true, 130 "answer": false, 131 "justification": "The paper makes broad claims about MCP security ('the first academic survey to systematize the risks') but does not bound its scope to specific MCP implementations or versions. Section I.C lists scope but the conclusions generalize across the entire MCP ecosystem without acknowledging which implementations were actually analyzed." 132 }, 133 "alternative_explanations_discussed": { 134 "applies": false, 135 "answer": false, 136 "justification": "As a survey/taxonomy paper presenting no empirical results, there are no findings requiring alternative explanations." 137 }, 138 "proxy_outcome_distinction": { 139 "applies": false, 140 "answer": false, 141 "justification": "Theoretical/survey paper with no measurements." 142 } 143 }, 144 "setup_transparency": { 145 "model_versions_specified": { 146 "applies": false, 147 "answer": false, 148 "justification": "No models are used in experiments; this is a survey." 149 }, 150 "prompts_provided": { 151 "applies": false, 152 "answer": false, 153 "justification": "No prompting is used; this is a survey." 154 }, 155 "hyperparameters_reported": { 156 "applies": false, 157 "answer": false, 158 "justification": "No experiments with hyperparameters." 159 }, 160 "scaffolding_described": { 161 "applies": false, 162 "answer": false, 163 "justification": "No agentic scaffolding used; this is a survey." 164 }, 165 "data_preprocessing_documented": { 166 "applies": true, 167 "answer": false, 168 "justification": "The paper does not document how papers were selected for inclusion. There is no search strategy, database queries, inclusion/exclusion criteria, or PRISMA-style flow describing the literature selection pipeline." 169 } 170 }, 171 "limitations_and_scope": { 172 "limitations_section_present": { 173 "applies": true, 174 "answer": false, 175 "justification": "There is no dedicated limitations section. Section I.C defines scope but does not discuss limitations of the survey methodology itself." 176 }, 177 "threats_to_validity_specific": { 178 "applies": true, 179 "answer": false, 180 "justification": "No threats to validity are discussed. The paper does not address potential selection bias in its literature coverage, recency bias, or completeness gaps." 181 }, 182 "scope_boundaries_stated": { 183 "applies": true, 184 "answer": true, 185 "justification": "Section I.C explicitly states scope: 'We specifically exclude general LLM adversarial attacks (e.g., weight poisoning) unless they directly impact the protocol's integrity or execution flow.' The scope is bounded to protocol primitives, topology risks, and their intersection." 186 } 187 }, 188 "data_integrity": { 189 "raw_data_available": { 190 "applies": true, 191 "answer": false, 192 "justification": "No raw data (literature corpus, extraction spreadsheets, coded themes) is made available." 193 }, 194 "data_collection_described": { 195 "applies": true, 196 "answer": false, 197 "justification": "The paper does not describe how the surveyed literature was collected. No search queries, databases, date ranges, or collection methodology is specified." 198 }, 199 "recruitment_methods_described": { 200 "applies": false, 201 "answer": false, 202 "justification": "No human participants; data source is published literature (not a standard benchmark either)." 203 }, 204 "data_pipeline_documented": { 205 "applies": true, 206 "answer": false, 207 "justification": "No pipeline from literature search to final taxonomy is documented. The reader cannot determine how papers were found, screened, or coded." 208 } 209 }, 210 "conflicts_of_interest": { 211 "funding_disclosed": { 212 "applies": true, 213 "answer": false, 214 "justification": "No funding source or acknowledgments section is present in the paper." 215 }, 216 "affiliations_disclosed": { 217 "applies": true, 218 "answer": true, 219 "justification": "Author affiliations are clearly listed: Tribhuvan University, University of North Dakota, Youngstown State University, University of Missouri, University of Toledo. All are academic institutions." 220 }, 221 "funder_independent_of_outcome": { 222 "applies": true, 223 "answer": false, 224 "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not absence of conflict." 225 }, 226 "financial_interests_declared": { 227 "applies": true, 228 "answer": false, 229 "justification": "No competing interests or financial disclosures statement is present in the paper." 230 } 231 }, 232 "contamination": { 233 "training_cutoff_stated": { 234 "applies": false, 235 "answer": false, 236 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 237 }, 238 "train_test_overlap_discussed": { 239 "applies": false, 240 "answer": false, 241 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 242 }, 243 "benchmark_contamination_addressed": { 244 "applies": false, 245 "answer": false, 246 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 247 } 248 }, 249 "human_studies": { 250 "pre_registered": { 251 "applies": false, 252 "answer": false, 253 "justification": "No human participants in this survey paper." 254 }, 255 "irb_or_ethics_approval": { 256 "applies": false, 257 "answer": false, 258 "justification": "No human participants." 259 }, 260 "demographics_reported": { 261 "applies": false, 262 "answer": false, 263 "justification": "No human participants." 264 }, 265 "inclusion_exclusion_criteria": { 266 "applies": false, 267 "answer": false, 268 "justification": "No human participants." 269 }, 270 "randomization_described": { 271 "applies": false, 272 "answer": false, 273 "justification": "No human participants." 274 }, 275 "blinding_described": { 276 "applies": false, 277 "answer": false, 278 "justification": "No human participants." 279 }, 280 "attrition_reported": { 281 "applies": false, 282 "answer": false, 283 "justification": "No human participants." 284 } 285 }, 286 "cost_and_practicality": { 287 "inference_cost_reported": { 288 "applies": false, 289 "answer": false, 290 "justification": "Survey paper with no computational method to cost." 291 }, 292 "compute_budget_stated": { 293 "applies": false, 294 "answer": false, 295 "justification": "Survey paper with no compute-intensive work." 296 } 297 }, 298 "survey_methodology": { 299 "prisma_or_structured_protocol": { 300 "applies": true, 301 "answer": false, 302 "justification": "No PRISMA flow diagram, no registered protocol, no systematic search strategy with reproducible queries. The paper is labeled an SoK but does not describe a structured review methodology for how sources were identified and selected." 303 }, 304 "quality_assessment_of_sources": { 305 "applies": true, 306 "answer": false, 307 "justification": "The paper does not assess the quality of its source papers. Blog posts (Medium, Equixly, Forge Code, Pomerium), industry reports, and peer-reviewed papers are cited alongside each other without any quality differentiation or risk-of-bias assessment." 308 }, 309 "publication_bias_discussed": { 310 "applies": true, 311 "answer": false, 312 "justification": "No discussion of publication bias. The paper does not consider whether the MCP security literature skews toward reporting vulnerabilities over successful defenses, or whether grey literature dominates due to the topic's recency." 313 } 314 } 315 }, 316 "claims": [ 317 { 318 "claim": "MCP is the first academic survey to systematize the risks of the Model Context Protocol", 319 "evidence": "Section I.D states 'To our knowledge, this is the first academic survey to systematize the risks of the Model Context Protocol.'", 320 "supported": "weak" 321 }, 322 { 323 "claim": "Over 43% of MCP server implementations execute unsafe shell calls", 324 "evidence": "Section VII.A cites Equixly [2] for this statistic: 'Over 43% of MCP server implementations tested by Equixly were found to execute unsafe shell calls.'", 325 "supported": "weak" 326 }, 327 { 328 "claim": "MCP's coupling of Context and Action creates a 'semantic attack surface' enabling cross-primitive escalation from read-only access to write-actions", 329 "evidence": "Sections I.D, IV, and V describe this mechanism with references to Guo et al. [5] and the Supabase incident [8]. Section VIII.A provides a detailed reconstruction of the Supabase case.", 330 "supported": "moderate" 331 }, 332 { 333 "claim": "Existing defense mechanisms are ill-equipped for the security-safety duality in MCP", 334 "evidence": "Section I.B argues this conceptually: 'Traditional firewalls cannot inspect the semantic intent of a JSON-RPC message, and LLM safety filters cannot see the downstream consequences of a tool execution.' No empirical evaluation of defense effectiveness is provided.", 335 "supported": "weak" 336 }, 337 { 338 "claim": "ETDI framework with cryptographic signed tool manifests can prevent rug-pull and tool poisoning attacks", 339 "evidence": "Section VI.A describes the ETDI framework from Bhatt et al. [6] with cryptographic verification, immutable versioning, and registry-based approval. No empirical evaluation of ETDI effectiveness is presented in this paper.", 340 "supported": "weak" 341 } 342 ], 343 "red_flags": [ 344 { 345 "flag": "No systematic literature search methodology", 346 "detail": "For a paper claiming to be a 'Systematization of Knowledge,' there is no description of how literature was identified, searched, screened, or selected. No databases, search queries, date ranges, or inclusion/exclusion criteria are specified. This makes the survey non-reproducible." 347 }, 348 { 349 "flag": "Heavy reliance on grey literature without quality differentiation", 350 "detail": "Many key claims are sourced from blog posts (Medium, Equixly, Forge Code, Pomerium, Red Hat, Reco, Promptfoo) rather than peer-reviewed research. These sources are cited alongside academic papers without any quality assessment or acknowledgment of their different evidentiary weight." 351 }, 352 { 353 "flag": "No empirical validation of taxonomy or defenses", 354 "detail": "The proposed vulnerability taxonomy (Table III) and defense architecture are entirely conceptual. No empirical evaluation demonstrates that the taxonomy is complete, that the categories are well-defined, or that the proposed defenses are effective." 355 }, 356 { 357 "flag": "Claims outrun evidence", 358 "detail": "The paper makes strong claims ('first academic survey,' 'comprehensive taxonomy,' 'profound new threat landscape') but provides no empirical basis beyond citing other works and reconstructing one case study (Supabase). The 43% unsafe shell calls statistic is taken from a blog post without independent verification." 359 }, 360 { 361 "flag": "No limitations section", 362 "detail": "Despite being a survey paper making broad claims about the MCP ecosystem, the paper contains no limitations section discussing potential gaps in coverage, selection bias, or the recency/maturity limitations of its sources." 363 } 364 ], 365 "cited_papers": [ 366 { 367 "title": "Model context protocol (MCP): Landscape, security threats, and future research directions", 368 "authors": ["X. Hou", "Y. Zhao", "S. Wang", "H. Wang"], 369 "year": 2025, 370 "arxiv_id": "2503.23278", 371 "relevance": "Companion MCP security survey covering landscape and threat directions." 372 }, 373 { 374 "title": "MCP safety audit: LLMs with the model context protocol allow major security exploits", 375 "authors": ["B. Radosevich", "J. Halloran"], 376 "year": 2025, 377 "arxiv_id": "2504.03767", 378 "relevance": "Empirical safety audit of MCP showing major security exploits through the protocol." 379 }, 380 { 381 "title": "Systematic analysis of MCP security", 382 "authors": ["Y. Guo", "P. Liu", "W. Ma", "Z. Deng", "X. Zhu", "P. Di", "X. Xiao", "S. Wen"], 383 "year": 2025, 384 "arxiv_id": "2508.12538", 385 "relevance": "Systematic analysis of MCP security vulnerabilities including tool poisoning and cross-session contamination." 386 }, 387 { 388 "title": "ETDI: Mitigating tool squatting and rug pull attacks in model context protocol (MCP)", 389 "authors": ["M. Bhatt", "V. S. Narajala", "I. Habler"], 390 "year": 2025, 391 "arxiv_id": "2506.01333", 392 "relevance": "Proposes cryptographic provenance framework (ETDI) for defending against MCP tool attacks." 393 }, 394 { 395 "title": "Enterprise-grade security for the model context protocol (MCP): Frameworks and mitigation strategies", 396 "authors": ["V. S. Narajala", "I. Habler"], 397 "year": 2025, 398 "arxiv_id": "2504.08623", 399 "relevance": "Enterprise security frameworks for MCP deployment." 400 }, 401 { 402 "title": "Agentic AI security: Threats, defenses, evaluation, and open challenges", 403 "authors": ["S. Datta", "S. K. Nahin", "A. Chhabra", "P. Mohapatra"], 404 "year": 2025, 405 "arxiv_id": "2510.23883", 406 "relevance": "Broader agentic AI security survey covering threats and defenses relevant to MCP-connected agent systems." 407 }, 408 { 409 "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection", 410 "authors": ["K. Greshake", "S. Abdelnabi"], 411 "year": 2023, 412 "arxiv_id": "2302.12173", 413 "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications." 414 }, 415 { 416 "title": "MindGuard: Tracking, detecting, and attributing MCP tool poisoning attack via decision dependence graph", 417 "authors": ["Z. Wang"], 418 "year": 2025, 419 "arxiv_id": "2508.20412", 420 "relevance": "Runtime defense system for detecting MCP tool poisoning via decision provenance tracking." 421 }, 422 { 423 "title": "MCP-Guard: A defense framework for model context protocol integrity in LLM applications", 424 "authors": ["W. Xing", "Z. Qi"], 425 "year": 2025, 426 "arxiv_id": "2508.10991", 427 "relevance": "Multi-stage defense framework for MCP protocol integrity." 428 }, 429 { 430 "title": "TRiSM for agentic AI: A review of trust, risk, and security management in LLM-based agentic multi-agent systems", 431 "authors": ["S. Raza", "R. Sapkota", "M. Karkee", "C. Emmanouilidis"], 432 "year": 2025, 433 "arxiv_id": "2506.04133", 434 "relevance": "Trust/risk/security management framework for LLM-based multi-agent systems." 435 }, 436 { 437 "title": "Backdoored retrievers for prompt injection attacks on retrieval augmented generation", 438 "authors": ["C. Clop", "Y. Teglia"], 439 "year": 2024, 440 "arxiv_id": "2410.14479", 441 "relevance": "Demonstrates RAG-based prompt injection via backdoored retrievers, directly relevant to MCP context poisoning." 442 }, 443 { 444 "title": "I know what you asked: Prompt leakage via KV-cache sharing in multi-tenant LLM serving", 445 "authors": ["M. Wu"], 446 "year": 2025, 447 "relevance": "NDSS paper showing side-channel prompt leakage in multi-tenant LLM deployments relevant to MCP multi-tenant risks." 448 } 449 ] 450 }