ai-research-survey

scan.json (18999B)

---

 {
   "paper": {
     "title": "MCP Safety Audit: LLMs with the Model Context Protocol Allow Major Security Exploits",
     "authors": ["Brandon Radosevich", "John T. Halloran"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.03767"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper demonstrates that Claude 3.7 and Llama-3.3-70B-Instruct connected to standard MCP servers can be coerced into executing malicious code, granting remote access control, and stealing credentials. A novel Retrieval-Agent Deception (RADE) attack is introduced where corrupted data in a vector database triggers multi-MCP-server exploits without direct attacker access. The authors release McpSafetyScanner, a multi-agent tool that proactively scans MCP server configurations for vulnerabilities and generates remediation reports.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "McpSafetyScanner is released at https://github.com/johnhalloran321/mcpSafetyScanner, stated in the abstract and confirmed as 'freely available.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Attack prompts are shown in figures/screenshots but no structured dataset of attack prompts, MCP configurations, or results is released as a downloadable artifact."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 specifies: Claude Desktop v0.8.1, macOS Sequoia v15.3.2, Agno v1.2.6, gpt-4o-2024-08-06, mcp v1.1.2, huggingface-hub v0.29.3, GNU netcat v0.7.1. Sufficient to reconstruct the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The MCP config file is given (Appendix A) but there are no scripts or README instructions for replicating the specific attacks."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative success rates are measured. The paper presents individual attack demonstrations (screenshots) with no aggregate statistics or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed. Claims about guardrail reliability ('refuses some of the time') are not quantified with success/failure rates across trials."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes or quantified success rates. The paper shows individual examples of successful and refused attacks but does not measure rates across systematic trials."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of attack prompts tested is not stated systematically. Only individual examples are shown in figures with no justification for why these specific prompts were chosen or how many were tested total."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or spread measures reported. The paper acknowledges guardrail refusals vary but does not quantify this variation across runs or prompt variants."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No comparison to existing security auditing tools or alternative MCP security approaches. McpSafetyScanner is presented without comparison to any baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines of any kind are included."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "McpSafetyScanner has three stages (hacker agent, auditor agent, supervisor agent) but no ablation study examines the contribution of each stage."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are used. Results are presented as qualitative demonstrations (screenshots of successful/failed attacks and scanner reports)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of attack severity, scanner report quality, or remediation usefulness. The scanner reports are shown but not evaluated by independent security experts."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark dataset exists for this work; it is a qualitative security demonstration, not a benchmark evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down findings by attack type (MCE, RAC, CT). Results are shown separately for Claude and Llama-3.3-70B-Instruct, and for direct prompt attacks vs. RADE attacks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses cases where attacks are refused by LLM guardrails (Figures 1a, 7, 8a, 18, 19), analyzing what triggers refusals vs. what bypasses them."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Guardrail refusals are reported as negative results for the attacker. The paper shows that some attack variants trigger guardrails and are refused (Section 3, Appendix D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLMs can be coerced for MCE, RAC, and CT attacks — all demonstrated with screenshots. McpSafetyScanner is described and shown to detect these vulnerabilities (Table 1, Figures 20-21)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claims are demonstrated constructively: the paper shows specific prompts that cause specific attack outcomes. This proof-by-demonstration is appropriate for a security audit showing that attacks are possible."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLMs with the Model Context Protocol Allow Major Security Exploits' but only tests Claude 3.7 and Llama-3.3-70B-Instruct. No explicit bounding of claims to these two models. The paper says 'likely other LLMs' require re-evaluation without evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. For example, the paper does not discuss whether the observed vulnerabilities stem from MCP design specifically vs. general tool-use capabilities, or whether user-confirmation dialogs (which Claude Desktop has) would mitigate the attacks in practice."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper demonstrates actual exploits (code execution, SSH key insertion, credential exfiltration) rather than proxy measures. The claims match the granularity of what was demonstrated."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 specifies Claude Desktop v0.8.1, gpt-4o-2024-08-06 (for McpSafetyScanner), Llama-3.3-70B-Instruct with specific library versions. Claude model version behind Desktop is not specified beyond 'Claude 3.7' in text."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Attack prompts are shown in full in figures (Figures 1, 2, 6-19). The RADE attack files are also shown (Figures 4a, 10a, 12a). Sufficient to reconstruct the attacks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or sampling settings are reported for any model. The McpSafetyScanner agents use gpt-4o but no inference parameters are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "McpSafetyScanner's three-stage architecture is described in detail (Figure 5, Section 5): hacker agent probes MCP features, auditor agent searches knowledge bases, supervisor generates report. Agent roles and tools are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how attack prompts were designed, selected, or iterated upon. The methodology for constructing the RADE attack files is not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 6 ('Discussion, Conclusions, and Future Work') mentions future plans but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. The paper does not address whether results might differ with different prompt phrasings, model updates, MCP server versions, or user-confirmation settings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. The paper does not state what it did NOT test (e.g., other models, other MCP servers, defense mechanisms like user approval dialogs)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data released. The full set of prompts tested, success/failure logs, or complete conversation transcripts are not available — only selected screenshots in figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No systematic methodology for how attack prompts were designed or selected. The paper presents specific examples without describing the process of developing them."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no sampling from a larger population. Attack prompts were crafted by the authors."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how many attacks were attempted, which were selected for inclusion, or any filtering/selection process for the examples shown."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8: 'We thank Leidos for funding this research through the Office of Technology. Approved for public release 25-LEIDOS-0318-29149.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list Leidos as their affiliation on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Leidos is a defense/IT services contractor with no apparent financial stake in MCP being insecure or secure. The funder appears independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests security vulnerabilities in MCP-enabled systems, not model knowledge on benchmarks. Contamination is structurally inapplicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model capability. Train/test overlap is inapplicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation. Contamination is structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states McpSafetyScanner has a 'runtime of less than one minute to scan and generate each report on an M2 Max MacBook Pro.' Wall-clock time is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware (M2 Max MacBook Pro) is mentioned but no total compute budget, API costs, or token consumption is reported for either the attacks or the scanner."
       }
     }
   },
   "claims": [
     {
       "claim": "Industry-leading LLMs (Claude 3.7, Llama-3.3-70B) connected to standard MCP servers can be coerced to execute malicious code, grant remote access control, and steal credentials.",
       "evidence": "Demonstrated via screenshots in Figures 1b, 2, 6, 8b, 9 (direct prompt attacks) and Figures 4, 10-14 (RADE attacks). Section 3 describes MCE, RAC, CT attacks; Section 4 describes RADE.",
       "supported": "moderate"
     },
     {
       "claim": "LLM guardrails are unreliable for preventing MCP-enabled attacks — refusal varies based on prompt phrasing rather than attack severity.",
       "evidence": "Figures 1a vs 1b show the same MCE attack refused when octal-encoded but executed in plaintext. Llama-3.3-70B-Instruct only refuses when explicit words like 'hack' or 'steal' are used (Section 3, Figures 18-19).",
       "supported": "moderate"
     },
     {
       "claim": "The RADE attack enables credential theft and remote access without direct attacker access to the victim's system.",
       "evidence": "Demonstrated end-to-end for CT (Figure 4) and RAC (Figures 10-11) using Claude Desktop with Chroma + filesystem/everything MCP servers.",
       "supported": "moderate"
     },
     {
       "claim": "McpSafetyScanner correctly identifies the vulnerabilities used in the demonstrated attacks and provides actionable remediations.",
       "evidence": "Table 1 summarizes scanner findings matching the three attack types. Figures 20-21 show scanner reports. However, the scanner is only evaluated on the same servers used in the attacks.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic evaluation",
       "detail": "All results are individual demonstrations (screenshots). No success rates are measured across multiple trials, prompt variants, or model configurations. The paper shows selected examples without quantifying how often attacks succeed or fail."
     },
     {
       "flag": "Circular evaluation of McpSafetyScanner",
       "detail": "McpSafetyScanner is evaluated only on the same MCP servers and attack types the authors designed. There is no evaluation on unseen servers, novel attack types, or comparison to other security tools. The scanner 'finding' the vulnerabilities the authors already know about does not demonstrate general detection capability."
     },
     {
       "flag": "Overly broad claims from narrow testing",
       "detail": "Only two LLMs tested (Claude 3.7, Llama-3.3-70B-Instruct) and four MCP servers, but claims are framed as general MCP security issues. The title suggests broad applicability ('LLMs with the Model Context Protocol') beyond what was tested."
     },
     {
       "flag": "User confirmation dialogs not discussed",
       "detail": "Claude Desktop requires user confirmation before tool execution in many cases. The paper does not discuss whether the demonstrated attacks require the user to approve tool calls, which would significantly reduce real-world threat level."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section and does not discuss threats to validity, scope boundaries, or potential weaknesses in the study methodology."
     }
   ],
   "cited_papers": [
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Aaron Grattafiori", "Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Documents safety alignment and cybersecurity evaluation methodology for Llama 3, directly relevant to understanding the guardrails tested in this paper."
     },
     {
       "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations",
       "authors": ["Hakan Inan"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Describes input/output safety guardrails for LLMs, relevant to understanding why MCP attacks can bypass safety mechanisms."
     },
     {
       "title": "Introducing the Model Context Protocol",
       "authors": ["Anthropic"],
       "year": 2025,
       "relevance": "The protocol whose security properties are audited in this paper; foundational to the MCP ecosystem being studied."
     }
   ]
 }