ai-research-survey

scan.json (27167B)

---

 {
   "paper": {
     "title": "The Hidden Dangers of Browsing AI Agents",
     "authors": [
       "Mykyta Mudryi",
       "Markiyan Chaklosh",
       "Grzegorz Marcin Wójcik"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.13076",
     "doi": "10.48550/arXiv.2505.13076"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper presents a white-box security assessment of Browser Use, a popular open-source LLM-powered browsing agent. Two vulnerabilities were identified: a critical domain validation bypass (CVE-2025-47241, CVSS 9.3) enabling SSRF through URL credential injection, and a high-severity credential exfiltration via prompt injection (CVSS 8.8) exploiting the agent's practice of appending parsed webpage HTML to prompts. The authors propose a defense-in-depth framework organized around initial access and post-exploitation mitigations, including planner-executor isolation, formal security analyzers, and session safeguards.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references a public GitHub repository (arimlabs/agent-hijacking-poc) containing the proof-of-concept exploit code. Listing 2 provides the full PoC Python script, and the malicious payload is documented in the repository's Issues tab."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The target system (Browser Use) is open source, and the exploit payloads and PoC code are published in the authors' GitHub repository. For a security assessment, the 'data' is the exploit code and target codebase, both publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Package Version: 0.1.44' for Browser Use and uses langchain_openai with ChatOpenAI(model='gpt-4o'), but provides no requirements.txt, Dockerfile, or detailed dependency listing. Library versions beyond the Browser Use version are not stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Listing 2 provides the PoC script and the paper describes the attack flow, there are no explicit step-by-step reproduction instructions. The reader must infer setup steps (install Browser Use 0.1.44, set up the malicious GitHub issue, configure OpenAI API key)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative security assessment with no quantitative experimental results. The findings are binary vulnerability demonstrations (exploit succeeds/fails), not statistical measurements."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative quantitative claims are made. The paper demonstrates vulnerability existence through PoC exploits, not statistical comparisons."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effects are measured. The contribution is identifying specific security vulnerabilities, not measuring effect magnitudes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No statistical sample is drawn. The paper performs a white-box code review and demonstrates two specific vulnerabilities."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No repeated measurements or experimental runs are conducted. Each vulnerability is demonstrated with a single proof-of-concept."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions other browsing agents (Operator, Computer Use, Skyvern, Proxy) in Section 2.3 for benchmark performance comparison, but does not compare their security posture or test whether the same vulnerabilities exist in other agents."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No security baselines are compared. The paper lists contemporary agents in Table 1 but only assesses Browser Use. There is no comparison against how other agents handle domain validation or prompt injection."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a security assessment identifying vulnerabilities, not a system with components to ablate. There is no proposed system whose components could be individually tested."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Vulnerabilities are scored only via CVSS (9.3/10 and 8.8/10) with severity labels (Critical, High). No attack success rates, time-to-exploit, or other quantitative metrics are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to demonstrating the existence of code-level security vulnerabilities. The PoC exploits are deterministic."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark or test set is used. The paper performs targeted code review and PoC demonstrations, not benchmark evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive per-layer threat breakdowns using the MAESTRO framework (Tables 4-11 in Appendix A), covering Foundation Models, Data Operations, Agent Frameworks, Deployment & Infrastructure, Evaluation & Observability, Security & Compliance, and Agent Ecosystem layers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper only reports successful exploits. There is no discussion of attack attempts that failed, partial mitigations that reduced attack effectiveness, or conditions under which the vulnerabilities could not be exploited."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper does not mention attacks that did not work, mitigations that were effective against their exploits, or configurations where Browser Use was resilient."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's main claims are supported: the threat model is presented in Section 3, defense-in-depth strategy in Section 4, and the CVE (CVE-2025-47241) and PoC are demonstrated in Section 5. The 'first end-to-end threat model' novelty claim is unverified but the paper's core empirical claims are substantiated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that 'untrusted web content can hijack agent behavior' is demonstrated through a working proof-of-concept exploit (Listing 2, Section 5.3.2). For security vulnerability claims, a working PoC is adequate causal evidence. The domain bypass is similarly demonstrated with concrete URL manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The Hidden Dangers of Browsing AI Agents' (plural, general) and abstract claim of 'comprehensive security evaluation of such agents' substantially overclaim. The actual assessment covers only Browser Use v0.1.44 with GPT-4o. Section 5.1 acknowledges that 'vulnerabilities discovered in Browser Use are likely to have practical relevance' to other agents, but this is speculative generalization, not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the vulnerabilities. For example, whether different LLMs would resist the prompt injection, whether the vulnerabilities are specific to Browser Use's architecture or inherent to all agent architectures, or whether configuration changes could mitigate the issues without code changes."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific vulnerabilities (domain validation bypass, credential exfiltration via prompt injection) and claims specific security issues. The measurements directly match the claims — there is no proxy gap between what was measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The PoC uses ChatOpenAI(model='gpt-4o') (Listing 2) without specifying a snapshot date or API version. 'gpt-4o' is a marketing name that maps to different model versions over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides the prompt template structure (Section 5.3.2), the injected malicious payload verbatim, and the full PoC task prompt in Listing 2. The attacker-controlled payload and the agent's base prompt construction are described in sufficient detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for the GPT-4o calls. The PoC uses default ChatOpenAI parameters without stating what those defaults are."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides a detailed description of the browsing agent architecture including Perception, Reasoning, Planning, and External Tool Calls components. The prompt construction pipeline, feedback loop mechanism, and sensitive data handling are documented in Sections 3.1 and 5.2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe the systematic methodology of the security assessment — how files were selected for review, what parts of the codebase were examined, or the process from initial code review to vulnerability identification. Specific files are named (browser_use/browser/context.py, browser_use/agent/prompts.py) but the assessment procedure is not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion briefly mentions 'the security landscape for AI browsing agents is still evolving' but does not discuss limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not acknowledge that findings may be specific to Browser Use v0.1.44, to GPT-4o, or to the specific prompt construction pattern used."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 4.1 states that traditional cybersecurity for external tools is 'beyond the scope of this research,' the paper does not state what its security findings do NOT show — e.g., that results may not generalize to other agents, other LLMs, or production deployments with additional safeguards."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The target system (Browser Use) is fully open source, and the PoC exploit code is published in the authors' GitHub repository (arimlabs/agent-hijacking-poc). The CVE (CVE-2025-47241) is publicly documented. All 'raw data' for this security assessment is independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe the systematic code review methodology — what tools were used, how the codebase was explored, or what criteria guided the vulnerability search. The specific vulnerability findings are presented without documenting the assessment process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is an open-source codebase (Browser Use), which does not require recruitment methodology."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from initial code review to vulnerability identification to PoC development is not documented. The paper jumps from 'we conducted a security evaluation' to presenting specific findings without describing intermediate steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: ARIMLABS.AI, Polish-Japanese Academy of Information Technology, Maria Curie-Sklodowska University, and University of the National Education Commission in Kraków."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Without disclosed funding, funder independence cannot be assessed. ARIMLABS.AI is a security-oriented company that could benefit commercially from publicizing AI agent vulnerabilities, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The authors' affiliation with ARIMLABS.AI (which appears to be a security consulting/research firm) is not discussed as a potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It is a security assessment testing defense mechanisms and identifying vulnerabilities in agent infrastructure, not model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed. The paper tests security vulnerabilities in agent architecture, not model performance on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The WebVoyager results cited in Table 1 are from other papers, not the authors' own evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a white-box security assessment of software."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study analyzes open-source code and demonstrates software vulnerabilities."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this security assessment."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this security assessment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this security assessment."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a security assessment demonstrating vulnerability existence, not proposing a compute-intensive method. The cost of running the PoC is irrelevant to the contribution's validity."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a security assessment, not a method requiring significant compute. No computational budget is relevant to the paper's claims."
       }
     }
   },
   "claims": [
     {
       "claim": "Browser Use's FQDN validation can be bypassed using URL credential injection, enabling navigation to unauthorized domains (CVE-2025-47241, CVSS 9.3).",
       "evidence": "Section 5.3.1 demonstrates that the URL https://example.com:pass@localhost:8080 bypasses the allowed_domains check because the parser treats 'example.com' as the domain when the actual target is 'localhost'. The vulnerable code in browser_use/browser/context.py is shown in Listing 1.",
       "supported": "strong"
     },
     {
       "claim": "Stored credentials can be exfiltrated via prompt injection through malicious webpage content (CVSS 8.8).",
       "evidence": "Section 5.3.2 provides a full proof-of-concept (Listing 2) where a malicious GitHub issue injects a HumanMessage override into the agent's prompt, causing it to extract <secret> tokens and submit them to an attacker-controlled endpoint (c2-demo.arimlabs.ai).",
       "supported": "strong"
     },
     {
       "claim": "This paper presents the first end-to-end threat model for browsing agents.",
       "evidence": "Section 3 presents a MAESTRO-based threat model with per-layer analysis (Tables 4-11). However, the 'first' claim is asserted without systematic evidence that no prior end-to-end threat model exists for this class of agents.",
       "supported": "weak"
     },
     {
       "claim": "The proposed defense-in-depth strategy (input sanitization, planner-executor isolation, formal analyzers, session safeguards) can protect against both initial access and post-exploitation attack vectors.",
       "evidence": "Section 4 discusses these mitigations with citations to prior work (e.g., f-secure LLM system reducing injection success to 0%), but the authors do not empirically test any of these defenses against their own discovered vulnerabilities.",
       "supported": "weak"
     },
     {
       "claim": "Placing untrusted content at the end of prompts increases prompt injection risk because LLMs disproportionately attend to tokens at the beginning and end.",
       "evidence": "Section 5.2 cites Liu et al. [10] ('Lost in the Middle') and demonstrates that Browser Use appends parsed HTML at the prompt's end. The cited paper supports the attention pattern claim, and the PoC confirms the exploit succeeds in this configuration.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single target, broad title",
       "detail": "The paper is titled 'The Hidden Dangers of Browsing AI Agents' (general) but only tests Browser Use v0.1.44 with GPT-4o. No other agents are security-tested. The claim of 'comprehensive security evaluation' overclaims the actual scope."
     },
     {
       "flag": "No failure reporting",
       "detail": "Only successful exploits are reported. There is no discussion of attacks that failed, LLMs that resisted the prompt injection, or configurations where Browser Use was resilient. This creates selection bias in the findings."
     },
     {
       "flag": "Mitigations proposed but not tested",
       "detail": "Section 4 proposes an extensive defense-in-depth strategy citing prior work, but none of the proposed mitigations are tested against the paper's own discovered vulnerabilities. The gap between the threat analysis and the mitigation recommendations is unbridged by evidence."
     },
     {
       "flag": "Potential commercial interest",
       "detail": "Two authors are affiliated with ARIMLABS.AI, which appears to be a security consulting firm. The PoC exfiltrates data to c2-demo.arimlabs.ai. No competing interests statement is provided despite the potential commercial benefit of publicizing AI agent vulnerabilities."
     },
     {
       "flag": "No systematic assessment methodology",
       "detail": "The paper does not describe how the security assessment was conducted — what tools were used, what parts of the codebase were examined, or what methodology guided the analysis. The reader cannot assess the completeness or rigor of the code review."
     }
   ],
   "cited_papers": [
     {
       "title": "WebGPT: Browser-assisted Question-Answering with Human Feedback",
       "authors": ["R. Nakano"],
       "year": 2021,
       "arxiv_id": "2112.09332",
       "relevance": "Early LLM-based web browsing agent that established the paradigm of browser-assisted question answering with human feedback."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational reasoning-and-acting framework for LLM agents that underpins the sense-plan-act loop discussed in the paper."
     },
     {
       "title": "Mind2Web: Towards a Generalist Agent for the Web",
       "authors": ["X. Deng"],
       "year": 2023,
       "arxiv_id": "2306.06070",
       "relevance": "Benchmark dataset for web agent evaluation using human action traces across real websites."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["S. Zhou"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic browser environment benchmark for testing web agent capabilities with complex dynamic web tasks."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["H. He"],
       "year": 2024,
       "arxiv_id": "2401.13919",
       "relevance": "End-to-end multimodal web agent benchmark used as the primary performance comparison for browsing agents in Table 1."
     },
     {
       "title": "AI Agents Under Threat: A Survey of Key Security Challenges and Future Pathways",
       "authors": ["Zehang Deng"],
       "year": 2024,
       "arxiv_id": "2406.02630",
       "relevance": "Survey of security challenges facing AI agents, covering knowledge gaps in multi-step user inputs and untrusted external entity interactions."
     },
     {
       "title": "Security of AI Agents",
       "authors": ["Yifeng He"],
       "year": 2024,
       "arxiv_id": "2406.08689",
       "relevance": "Discusses session management and isolation as security mechanisms for AI agents, relevant to post-exploitation defenses."
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
       "authors": ["Edoardo Debenedetti"],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Framework for evaluating prompt injection attacks and defenses in LLM agents; cited for defense baseline reducing attack success from 25% to 8%."
     },
     {
       "title": "AI Agents with Formal Security Guarantees",
       "authors": ["Mislav Balunović"],
       "year": 2024,
       "arxiv_id": "2409.19091",
       "relevance": "Proposes formal security analyzers that enforce hard constraints on agent actions using domain-specific security rules."
     },
     {
       "title": "System-Level Defense against Indirect Prompt Injection Attacks: An Information Flow Control Perspective",
       "authors": ["Fangzhou Wu"],
       "year": 2024,
       "arxiv_id": "2409.19091",
       "relevance": "f-secure LLM system with planner-executor isolation that reportedly reduces prompt injection success to 0% while preserving functionality."
     },
     {
       "title": "Goal-guided Generative Prompt Injection Attack on Large Language Models",
       "authors": ["Chong Zhang"],
       "year": 2024,
       "arxiv_id": "2404.07234",
       "relevance": "Studies prompt injection attacks on LLMs, cited for the finding that larger models tend to perform better at mitigating prompt injection."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "authors": ["Nelson F. Liu"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Demonstrates that LLMs attend disproportionately to tokens at the beginning and end of prompts, directly relevant to the prompt injection attack vector."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly actionable for Browser Use users (upgrade to v0.1.45) and provides a threat model framework applicable to agent developers building similar systems."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Prompt injection vulnerabilities in LLM agents are widely expected; the URL parsing bypass is a known class of vulnerability (SSRF via credential injection in URLs)."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates credential exfiltration from a 60K-star AI agent via a working PoC with a published CVE (CVSS 9.3), directly feeding AI safety and security anxieties."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Responsible disclosure was followed and the vulnerability was patched; no adversarial relationship or controversy with Browser Use maintainers."
     },
     "demo_ability": {
       "score": 2,
       "justification": "PoC code is provided in-paper and in a public GitHub repo, but requires installing a specific (now outdated) Browser Use version and OpenAI API access to reproduce."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Browser Use has 60K GitHub stars but is not a mainstream consumer product; ARIMLABS.AI is not widely recognized."
     }
   }
 }