ai-research-survey

scan.json (21517B)

---

 {
   "paper": {
     "title": "Cuckoo Attack: Stealthy and Persistent Attacks Against AI-IDE",
     "authors": ["Xinpeng Liu", "Junming Liu", "Peiyu Liu", "Han Zheng", "Qinying Wang", "Mathias Payer", "Shouling Ji", "Wenhai Wang"],
     "year": 2025,
     "arxiv_id": "2509.15572"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "PoC artifacts are uploaded to Zenodo (https://zenodo.org/records/16757439) and the user study form is publicly available. Section V and Ethics section confirm open-sourced PoC artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The user study raw data is not released. The authors state they 'deleted all the metadata generated in the analysis process' (Ethics section). No dataset of configuration files or analysis scripts is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The PoC environment is described at a high level (MacBook Air, Kali Linux, Cobalt Strike) but no requirements.txt, Dockerfile, or detailed dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While PoC videos are uploaded to Zenodo and the attack flow is described, there are no step-by-step reproduction instructions with specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The user study reports percentages (e.g., 80%, 74%, 73%) but no confidence intervals or error bars are provided for these estimates from 112-124 participants."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative statistical claims between groups. The user study is descriptive, and the security evaluation is pass/fail per IDE."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No comparative claims requiring effect sizes are made. Results are binary (vulnerable/not vulnerable) or descriptive percentages."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The user study has 112-124 participants (discrepancy: 124 in Section IV-A, 112 in Appendix A) with no justification for sample size or power analysis."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "The experiments are deterministic security evaluations (exploit works or doesn't), not repeated stochastic trials requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against prior attack paradigms (e.g., direct MCP credential exfiltration from Invariant Lab, direct command execution attacks) and explains how Cuckoo Attack differs in stealth and persistence."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baseline attacks referenced are recent (2024-2025), including Invariant Lab MCP exploits, AgentPoison, and CVE-2025-54135."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper does not systematically test which attack components (e.g., obfuscation, exec chaining) contribute most to success."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation covers multiple dimensions: exploit file editing success, exploit command execution success, display of startup command, achievement of CI, and manual approval count (Table III)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A user study (112-124 participants) evaluates developer willingness to delegate configuration tasks to agents, which is relevant to the attack's real-world feasibility claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a security research paper demonstrating vulnerabilities, not a benchmark evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides per-IDE breakdown of seven security checkpoints, and Table III provides per-IDE exploit results across multiple dimensions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Cursor is identified as the one IDE that resists the MCP configuration attack (filters && in mcp.json). The paper discusses why it fails and notes Cursor remains vulnerable to broader attack paradigm."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Cursor blocks the specific MCP attack vector, and that Cline with GPT-4.1 is not opaque in information retrieval. Some IDEs resist file editing exploits (Table III)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about stealthy/persistent attacks, nine AI-IDE pairs tested, and PoC validation are supported by Section V results (Table III shows 8/9 achieve CI)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are about exploit mechanisms (e.g., 'embedding payloads into configuration files achieves persistence'). These are demonstrated through direct PoC execution, which constitutes adequate causal evidence for security research."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to the nine specific AI-IDE/Agent pairs tested and notes that Cursor's defense is narrow. Supply chain impact estimates are clearly labeled as potential rather than demonstrated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, the user study willingness scores may not translate to actual behavior; the paper does not consider that users might notice payload in code review or version control diffs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table II lists models as 'GPT-4.1', 'Claude-4', 'doubao-seed-1.6', 'Qwen3' without specific API versions or snapshot dates. 'Claude-4' and 'GPT-4.1' lack version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The malicious README.md content and mcp.json payload examples are shown (Listing 1), but the exact prompts used to instruct the Agent (e.g., the user's natural language request) are not fully provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, etc.) are reported for any of the tested Agent/IDE pairs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party AI-IDEs (Cursor, Cline, Copilot, etc.) as black boxes. The authors cannot describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The user study questionnaire design and analysis pipeline are not fully documented. The configuration file identification process used XAI Grok deep search but the filtering criteria and verification process are only briefly described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion section (IX) covers related findings and community awareness but does not discuss study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address limitations such as user study sample bias, the controlled lab setting vs real-world conditions, or the evolving nature of IDE security."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. For example, it does not acknowledge that the user study measures willingness rather than actual behavior, or that PoC in controlled settings may not reflect real-world exploit success rates."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "User study raw data is explicitly deleted per the Ethics section. PoC videos are on Zenodo but raw experimental logs are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "User study: participants recruited from developer community and academic forums, 112 participants, online questionnaire, willingness scale 0-10. PoC: isolated network, MacBook Air victim, Kali Linux attacker (Section V-A)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states participants were 'recruited from both industrial and academic backgrounds through internal professional networks and public forums' with demographic breakdown by role (Figure 6)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The user study analysis pipeline from raw responses to reported percentages is not documented. The paper jumps from '124 AI-IDE users' (or 112 in Appendix) to summary statistics without showing intermediate steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Zhejiang University, EPFL) are clearly listed. None of the authors appear affiliated with the evaluated AI-IDE vendors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests security vulnerabilities in AI-IDEs, not model capability on benchmarks. Training cutoff is irrelevant to the attack evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model knowledge. The paper tests whether agents can be manipulated via prompt injection, not model recall."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. This is a security vulnerability demonstration."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the user study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics section explicitly states 'this institution does not have an IRB' but claims to follow the Menlo Report principles. No formal ethics approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides participant breakdown by professional role (software engineers, architects, researchers, students) with Figure 6, and states over 55% had 4+ years programming experience."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion or exclusion criteria are stated for the user study. Participants are described as 'AI-IDE users' recruited from 'developer community and academic forums' without selection criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with treatment/control conditions. Randomization does not apply."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not an experimental study. Blinding does not apply."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Inconsistent participant counts (124 in Section IV-A vs 112 in Appendix A) with no explanation for the discrepancy, suggesting possible attrition or filtering that is not documented."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a security research paper demonstrating vulnerabilities, not proposing a method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "The experiments involve manual PoC demonstrations on consumer hardware, not compute-intensive workloads."
       }
     }
   },
   "claims": [
     {
       "claim": "All nine mainstream AI-IDE/Agent pairs tested are vulnerable to the Cuckoo Attack paradigm, with 8 of 9 achieving arbitrary command injection via MCP configuration files.",
       "evidence": "Table III shows exploit results: all 9 pairs support command execution exploit, and 8/9 achieve CI (Cursor is the exception). Section V-C details per-IDE results.",
       "supported": "strong"
     },
     {
       "claim": "Over 55% of surveyed developers delegate configuration-related tasks to AI agents, with 80% willing to configure environments from README.md files.",
       "evidence": "User study of 112-124 participants (Section IV-A and Appendix A) reports willingness scores. 80% for README-based configuration, 74% for build configs, 73% for IDE settings.",
       "supported": "moderate"
     },
     {
       "claim": "The attack achieves stealth by decoupling the malicious action from the Agent's visible operations, and persistence through configuration file embedding.",
       "evidence": "Section III-B formalizes the two-stage paradigm. PoC in Section V-B demonstrates the attack chain. Table III 'Display Startup Cmd' row shows most IDEs hide the execution.",
       "supported": "strong"
     },
     {
       "claim": "The attack can propagate through the open-source ecosystem as a supply chain attack via shared configuration files.",
       "evidence": "Section IV-D describes the propagation mechanism. Impact estimates in Table I (e.g., 5.7M GitHub Actions repositories) are based on repository counts, not demonstrated propagation.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper introduces Cuckoo Attack, a two-stage attack paradigm against AI-IDEs that achieves stealth and persistence by embedding malicious payloads into configuration files (e.g., mcp.json, Makefile). Testing across nine mainstream AI-IDE/Agent pairs (including Cursor, Cline, Copilot, Windsurf, Trae, Zed, Augment, Lingma), the PoC demonstrates arbitrary command injection in 8 of 9 platforms. A user study of 112-124 developers confirms that configuration tasks are routinely delegated to AI agents, validating the attack surface. The paper identifies seven actionable security checkpoints for vendors and has responsibly disclosed vulnerabilities, with Microsoft and ByteDance confirming the findings.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity section. For a security paper making broad claims about real-world impact, the absence of discussion about controlled lab settings vs real deployments, user study generalizability, or evolving vendor defenses is a significant omission."
     },
     {
       "flag": "Inconsistent participant counts",
       "detail": "Section IV-A reports 124 participants while Appendix A reports 112, with no explanation for the discrepancy. This raises questions about data handling."
     },
     {
       "flag": "Supply chain impact claims exceed evidence",
       "detail": "Claims about potential supply chain propagation affecting millions of repositories (Table I) are based on repository counts containing configuration files, not demonstrated propagation. The 'worm-like effect' is hypothetical."
     },
     {
       "flag": "User study measures willingness, not behavior",
       "detail": "The user study measures self-reported willingness to delegate tasks on a 0-10 scale, which may not reflect actual behavior. This distinction is not acknowledged in the paper."
     }
   ],
   "cited_papers": [
     {
       "title": "MCP Safety Audit: LLMs with the Model Context Protocol Allow Major Security Exploits",
       "authors": ["B. Radosevich", "J. Halloran"],
       "year": 2025,
       "arxiv_id": "2504.03767",
       "relevance": "Directly relevant security audit of MCP protocol vulnerabilities in LLM agents."
     },
     {
       "title": "Model Context Protocol (MCP): Landscape, Security Threats, and Future Research Directions",
       "authors": ["X. Hou", "Y. Zhao", "S. Wang", "H. Wang"],
       "year": 2025,
       "relevance": "First systematic analysis of MCP attack surface including Name Collision and Code Injection threats."
     },
     {
       "title": "RedCode: Risky Code Execution and Generation Benchmark for Code Agents",
       "authors": ["C. Guo", "X. Liu", "C. Xie", "A. Zhou", "Y. Zeng", "Z. Lin", "D. Song", "B. Li"],
       "year": 2024,
       "relevance": "Benchmark for evaluating safety of code agents, relevant to AI-IDE security evaluation."
     },
     {
       "title": "Prompt Injection Attack Against LLM-Integrated Applications",
       "authors": ["Y. Liu", "G. Deng", "Y. Li"],
       "year": 2024,
       "arxiv_id": "2306.05499",
       "relevance": "Foundational work on prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "Les Dissonances: Cross-Tool Harvesting and Polluting in Multi-Tool Empowered LLM Agents",
       "authors": ["Z. Li", "J. Cui", "X. Liao", "L. Xing"],
       "year": 2025,
       "arxiv_id": "2504.03111",
       "relevance": "Demonstrates cross-tool data exfiltration attacks in multi-tool LLM agents."
     },
     {
       "title": "Demystifying RCE Vulnerabilities in LLM-Integrated Apps",
       "authors": ["T. Liu", "Z. Deng", "G. Meng", "Y. Li", "K. Chen"],
       "year": 2024,
       "relevance": "Systematic detection of remote code execution vulnerabilities in LLM frameworks, published at ACM CCS."
     },
     {
       "title": "AgentPoison: Red-Teaming LLM Agents via Poisoning Memory or Knowledge Bases",
       "authors": ["Z. Chen", "Z. Xiang", "C. Xiao", "D. Song", "B. Li"],
       "year": 2024,
       "relevance": "Red-teaming attacks against LLM agents through knowledge base poisoning, published at NeurIPS."
     },
     {
       "title": "Beyond the Protocol: Unveiling Attack Vectors in the Model Context Protocol Ecosystem",
       "authors": ["H. Song", "Y. Shen", "W. Luo"],
       "year": 2025,
       "relevance": "Validates MCP attack vectors with real-world malicious MCP servers."
     },
     {
       "title": "Spaiware: Uncovering a Novel Artificial Intelligence Attack Vector Through Persistent Memory in LLM Applications and Agents",
       "authors": ["M. Herrador", "J. Rehberger"],
       "year": 2025,
       "relevance": "Demonstrates persistent attacks through LLM application memory, related to persistence in AI agent attacks."
     },
     {
       "title": "Research Directions in Software Supply Chain Security",
       "authors": ["L. Williams", "G. Benedetti", "S. Hamer"],
       "year": 2025,
       "relevance": "Survey of software supply chain security threats relevant to the supply chain propagation aspect of Cuckoo Attack."
     }
   ]
 }