ai-research-survey

scan.json (19064B)

---

 {
   "paper": {
     "title": "Cybersecurity AI: Hacking the AI Hackers via Prompt Injection",
     "authors": ["Víctor Mayoral-Vilches", "Per Mannermaa Rynning", "Ameer Pornillos"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.21669"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub pull request link is provided (https://github.com/aliasrobotics/cai/pull/249) with detailed technical solutions. The CAI framework itself is open source."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides PoC attack payloads directly in the text and appendix (malicious server code, index.html payloads), and references the CAI repository for reproduction."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions Python and specific tools (curl, nmap, socat) but does not specify versions or a reproducible environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While PoC code is shown, there are no step-by-step reproduction instructions. The paper mentions 'test environments provided in our research repository' but does not include a README or explicit reproduction guide in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 91.4% overall success rate, 100% for direct execution). No confidence intervals or error bars are provided despite the relatively small sample sizes (10 attempts per variant)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares success rates across attack categories and claims differences in effectiveness but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Success rates are reported with baseline context: e.g., '91.4% (128/140 attempts)' unprotected vs '0% (0/140)' with guardrails. The absolute counts and rates provide sufficient effect size context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 10 attempts per variant (140 total) is stated but not justified. No power analysis or rationale for why 10 attempts is sufficient to establish 'statistical significance' as the paper claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across the 10 attempts per variant. Only aggregate success rates and mean time-to-compromise are given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The unprotected CAI system serves as the baseline, compared against the guardrail-protected version. Table 4 shows before/after guardrail results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Only the CAI framework is tested. No comparison against other AI security tools (PentestGPT, etc.) or other defense mechanisms is provided."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The four-layer defense is presented as a complete system. No ablation study shows the contribution of each individual layer (sandboxing alone, tool-level protection alone, etc.)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: success rate, time-to-compromise, latency overhead (12.3ms), memory footprint (47.2MB), CPU utilization (1.7% increase), and false positive rate (<0.1%)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; the attacks and defenses are evaluated by whether system compromise occurs, which is objectively verifiable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a security evaluation with proof-of-concept attacks, not a benchmark evaluation with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-category breakdown across all seven attack categories with individual success rates and time-to-compromise for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss cases where attacks failed (e.g., the 1/40 failure in multi-layer encoding, 1/20 in variable indirection). No analysis of why some attempts within categories failed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every defense layer is presented as successful. No discussion of defense approaches that were tried and didn't work, or attack variants that the guardrails initially failed to catch."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims prompt injection attacks against AI security tools, PoC exploits against CAI, and multi-layered defense. All are supported by the experimental results in Sections 2 and 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that the guardrail system blocks attacks is supported by controlled before/after testing (same attacks with and without guardrails). The single-variable manipulation (enabling/disabling guardrails) is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only the CAI framework but makes broad claims about 'AI security agents' and 'all LLM-based security tools' being 'fundamentally unsafe.' The title and conclusions generalize far beyond the single system tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the paper does not consider whether CAI's specific architecture makes it more vulnerable than other tools, or whether the 100% success rate is an artifact of the specific LLM used."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not specify which LLM is used by the CAI framework for the experiments. No model name, version, or API details are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The attack prompts/payloads are provided in full (Code listings 1-9). The system prompt context ('Bug Bounter' agent) is shown in the tool output logs."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for the CAI agent's underlying model."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The CAI framework's agentic scaffolding is described: tool execution via generic_linux_command, data wrapping with [TOOL OUTPUT] markers, and the four-layer defense architecture is detailed in Section 3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how attack variants were selected, how the 14 PoC attacks were developed, or what criteria determined the seven categories."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. A footnote on page 9 briefly mentions sandbox limitations, but this does not constitute a limitations section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The footnote about sandbox escape is the only acknowledgment of a limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what it did NOT test. It generalizes from testing one framework (CAI) to all LLM-based security tools without bounding the scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental logs (the 140 individual trial outcomes) are not available. Only aggregate success rates are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the experimental protocol: '14 attack variants × 10 attempts each = 140 total exploitation attempts against unprotected systems.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a technical security evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from individual trial outcomes to aggregate success rates is not documented. How 'success' vs 'failure' was determined for each attempt is not explicitly defined."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'This research was partly funded by the European Innovation Council (EIC) accelerator project \"RIS\" (GA 101161136).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Alias Robotics, Oracle Corporation, and independent security researcher. The first author is affiliated with Alias Robotics, which develops the CAI framework being tested."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The EIC funder is a government agency with no commercial stake in the CAI framework's security evaluation results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. The first author's company (Alias Robotics) develops and maintains the CAI framework being evaluated, which is a significant undeclared conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests prompt injection attacks against a security agent, not model capability on a benchmark. The attacks test the agent's behavior, not its knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation; contamination in the traditional sense is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Token costs are shown in individual code listing outputs (e.g., 'Session: $0.0376') but no aggregate cost analysis for the full experimental campaign is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Performance overhead of guardrails is reported (12.3ms latency, 47.2MB memory, 1.7% CPU) but not the total compute used for experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "All 14 attack variants achieved 100% exploitation success against unprotected AI security agents, with a mean time-to-compromise of 20.1 seconds.",
       "evidence": "Section 4 and Table 4 report 128/140 successful attempts (91.4% overall) across 14 variants with 10 attempts each. Individual category rates range from 70% to 100%.",
       "supported": "moderate"
     },
     {
       "claim": "The four-layer defense architecture achieves complete mitigation (0% attack success) while maintaining operational efficiency (<12ms latency, <0.1% false positives).",
       "evidence": "Section 4 reports 0/140 successful attacks with guardrails enabled, with 12.3ms mean latency, 47.2MB memory, and 1.7% CPU overhead.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt injection is a systemic architectural flaw in transformer-based LLMs, not an implementation bug.",
       "evidence": "Section 5 provides a theoretical argument based on the attention mechanism treating all tokens identically regardless of trust level. Empirical support is limited to testing one framework (CAI).",
       "supported": "weak"
     },
     {
       "claim": "Current LLM-based security agents are fundamentally unsafe for deployment in adversarial environments without comprehensive defensive measures.",
       "evidence": "Based on testing 14 attack variants against the CAI framework only. No other security agents were tested.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study", "benchmark-eval"],
   "key_findings": "The paper demonstrates that AI-powered cybersecurity agents (specifically the CAI framework) are highly vulnerable to prompt injection attacks, with a 91.4% overall success rate across 14 attack variants in 7 categories. A four-layer defense architecture (sandboxing, tool-level protection, file write protection, multi-layer validation) achieved complete mitigation in testing. The authors argue this vulnerability is architecturally inherent to transformer-based LLMs due to the attention mechanism's inability to distinguish trusted from untrusted tokens.",
   "red_flags": [
     {
       "flag": "Self-evaluation conflict",
       "detail": "The first author (Alias Robotics) develops and maintains the CAI framework being both attacked and defended. The paper evaluates their own product's vulnerability and their own defense, without acknowledging this as a conflict of interest."
     },
     {
       "flag": "Overgeneralization from single system",
       "detail": "Only the CAI framework is tested, but conclusions are stated as applying to 'all LLM-based security tools' and 'current LLM-based security agents' broadly. No other tools were evaluated."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity discussion despite making very strong claims ('fundamentally unsafe', 'unequivocal conclusion')."
     },
     {
       "flag": "Suspiciously clean defense results",
       "detail": "The guardrail system achieves exactly 0% attack success across all 140 attempts with no false negatives discussed. No ablation of individual defense layers is provided."
     },
     {
       "flag": "Unspecified LLM",
       "detail": "The underlying LLM used by the CAI agent is never identified. Results could vary dramatically with different models, yet the paper generalizes to all LLM-based systems."
     },
     {
       "flag": "Small sample size claimed as statistically significant",
       "detail": "The paper claims 'statistical significance' from 10 attempts per variant without any statistical tests or power analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "PentestGPT: Evaluating and Harnessing Large Language Models for Automated Penetration Testing",
       "authors": ["Gelei Deng", "Yi Liu", "Víctor Mayoral-Vilches"],
       "year": 2024,
       "relevance": "Directly relevant as a major AI-powered security tool evaluated for LLM capability in penetration testing."
     },
     {
       "title": "CAI: An Open, Bug Bounty-Ready Cybersecurity AI",
       "authors": ["Víctor Mayoral-Vilches"],
       "year": 2025,
       "arxiv_id": "2504.06017",
       "relevance": "The framework under evaluation in this paper; describes the agentic AI architecture for cybersecurity."
     },
     {
       "title": "Ignore Previous Prompt: Attack Techniques for Language Models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Foundational work on prompt injection attacks against LLMs, directly relevant to AI safety."
     },
     {
       "title": "Cybersecurity AI: The Dangerous Gap Between Automation and Autonomy",
       "authors": ["Víctor Mayoral-Vilches"],
       "year": 2025,
       "arxiv_id": "2506.23592",
       "relevance": "Discusses the automation-autonomy gap in AI security agents, relevant to agentic AI safety."
     },
     {
       "title": "CAI Fluency: A Framework for Cybersecurity AI Fluency",
       "authors": ["Víctor Mayoral-Vilches"],
       "year": 2025,
       "arxiv_id": "2508.13588",
       "relevance": "Framework for evaluating cybersecurity AI capabilities, relevant to AI agent evaluation methodology."
     }
   ]
 }