ai-research-survey

scan.json (21001B)

---

 {
   "paper": {
     "title": "AGENTVIGIL: Generic Black-Box Red-teaming for Indirect Prompt Injection against LLM Agents",
     "authors": ["Zhun Wang", "Vincent Siu", "Zhe Ye", "Tianneng Shi", "Yuzhou Nie", "Xuandong Zhao", "Chenguang Wang", "Wenbo Guo", "Dawn Song"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.05849"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper. No mention of code release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two public benchmarks: AgentDojo and VWA-adv, both publicly available. The initial corpus templates are described but not explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. Model checkpoints are listed in Appendix A but no software dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method is described algorithmically but without runnable reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 71%, 70% success rates) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AGENTVIGIL outperforms baselines (e.g., 'nearly doubling the performance') based solely on comparing success rate numbers with no statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute success rates with baseline context throughout (e.g., baseline 38% vs AGENTVIGIL 71%; baseline 36% vs 70%), providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The fuzzing and test sets are 142 and 173 tasks for AgentDojo, 99 and 100 for VWA-adv. No justification for these sizes or power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across runs is reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against handcrafted baseline attacks from AgentDojo and VWA-adv, plus OpenPromptInjection and InjecAgent baselines in Appendix B.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include AgentDojo (2024), VWA-adv (2024), OpenPromptInjection (2024), and InjecAgent (2024), all contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 presents ablation removing (1) the initial corpus and (2) adaptive seed scoring + MCTS-based selection, shown in Figure 3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both attack success rate (ASR) and coverage as metrics, plus evaluates transferability across models and effectiveness against defenses."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of attack quality or outputs is included. All evaluation is automated via benchmark frameworks. The paper makes claims about real-world applicability that could benefit from human assessment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Tasks are explicitly split into fuzzing sets and test sets (142/173 for AgentDojo, 99/100 for VWA-adv) with transferability evaluated on the held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 (Table 5) provides per-suite breakdowns for AgentDojo (Slack, Workspace, Travel, Banking) and per-goal breakdowns for VWA-adv (illusioning vs goal misdirection)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that attacks are ineffective against Claude-3.5-Sonnet, that defenses reduce success rates, and that VWA-adv defenses cause AGENTVIGIL to converge with baseline."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AGENTVIGIL performs worse than baseline against Claude-3.5-Sonnet on AgentDojo (0.04 vs 0.08), and that defenses on VWA-adv cause degradation to baseline levels."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 71% and 70% success rates on AgentDojo and VWA-adv are supported by Tables 1-3 and Figures 3-4. The 'nearly doubling' claim is supported (38%→71%, 36%→70%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Section 5.3 provides controlled single-variable manipulation to support claims about which components matter (initial corpus, seed scoring/selection)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Generic Black-Box Red-teaming' but results are on only two benchmarks. The real-world case study is a single scenario on a local WebArena deployment. The paper does not bound its generalization claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why AGENTVIGIL outperforms baselines. For example, the improvement could stem primarily from the curated initial corpus rather than the fuzzing framework itself. No threats-to-validity or confound discussion is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A lists exact checkpoints: o3-mini-2024-12-17, gpt-4o-mini-2024-07-18, gpt-4o-2024-08-06, claude-3-5-sonnet-20241022, gemini-2.0-flash-exp."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The mutation prompt templates (Shorten, Expand, Rephrase, Crossover, GenerateSimilar) and the initial corpus templates are described in natural language but the actual prompt text is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: 10 fuzzing iterations, 3 mutated prompts per iteration (AgentDojo), 10 per iteration (VWA-adv), top-5 seeds for transfer, quarter sampling of tasks. Coverage factor C and exploration factor C in algorithms are named but values not specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The fuzzing framework architecture is described in detail in Section 4, including seed selection (MCTS), scoring, mutation, and the iterative loop. Algorithms 1-3 formalize the process."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and 5.2 describe how tasks are randomly split into fuzzing/test sets with specific counts (142/173 and 99/100), and how quarter-sampling is used during fuzzing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Impact Statement discusses ethical considerations but not methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. The threat model (Section 3) defines scope of the attacker but not limitations of the evaluation or claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual task results, generated adversarial prompts, logs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes corpus collection from human heuristics, online resources, and existing research. Sections 5.1-5.2 describe the benchmark setup and task splitting."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from standard benchmarks (AgentDojo, VWA-adv)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from corpus collection → seed generation → fuzzing → evaluation is documented in Section 4 and the experimental setup sections, with task counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed with numbered superscripts, though the specific institutions are not spelled out in the extracted text. Authors are from UC Berkeley (Dawn Song's group) and other institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attack effectiveness against LLM agents, not model capability on benchmarks. The models are targets being attacked, not being evaluated for their knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates attack success against agents, not model benchmark performance where training data contamination would be relevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the benchmarks test agent security vulnerabilities, not model knowledge that could be contaminated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock time for the fuzzing process are reported, despite the framework making many API calls across iterations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (API spend, GPU hours, total time) is stated despite the iterative fuzzing requiring substantial API usage."
       }
     }
   },
   "claims": [
     {
       "claim": "AGENTVIGIL achieves 71% attack success rate on AgentDojo against o3-mini agents, nearly doubling the baseline's 38%.",
       "evidence": "Table 1 and Section 5.1 fuzzing results show 71% ASR on fuzzing set vs 38% baseline.",
       "supported": "strong"
     },
     {
       "claim": "AGENTVIGIL achieves 70% success rate on VWA-adv against GPT-4o agents, compared to 36% baseline.",
       "evidence": "Figure 4 and Section 5.2 show 70% on fuzzing set vs 36% baseline. Table 1 confirms 60% on fuzzing set (discrepancy: 70% is coverage, 60% is ASR in Table 1).",
       "supported": "moderate"
     },
     {
       "claim": "Generated adversarial prompts exhibit strong transferability across unseen tasks and LLMs.",
       "evidence": "Table 1 test set results show 65% on unseen tasks for o3-mini, 67% on Gemini-2-flash-exp (unseen LLM). However, transfer to Claude-3.5-Sonnet is poor (4% on AgentDojo).",
       "supported": "moderate"
     },
     {
       "claim": "All three core components (initial corpus, seed scoring, MCTS selection) are critical to performance.",
       "evidence": "Ablation study in Section 5.3 and Figure 3 shows degraded coverage when each is removed.",
       "supported": "moderate"
     },
     {
       "claim": "AGENTVIGIL can successfully attack agents in real-world environments, misleading them to navigate to arbitrary URLs.",
       "evidence": "Section 5.4 describes a single case study on a local WebArena deployment with one attack scenario.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "AGENTVIGIL is a black-box fuzzing framework for indirect prompt injection attacks against LLM agents, combining curated initial seed corpora, adaptive scoring, and MCTS-based seed selection. It achieves 71% and 60-70% attack success rates on AgentDojo (o3-mini) and VWA-adv (GPT-4o) respectively, roughly doubling baseline handcrafted attacks. Generated adversarial prompts transfer across unseen tasks and models (except Claude-3.5-Sonnet), but existing defenses can reduce the advantage to near-baseline levels on VWA-adv.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or variance across runs. Given the stochastic nature of both LLM outputs and the fuzzing process, this is a significant omission."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity discussion despite making broad claims about generic applicability."
     },
     {
       "flag": "Overly broad title claim",
       "detail": "The title claims 'Generic Black-Box Red-teaming' but the framework is evaluated on only two benchmarks and one real-world scenario (local WebArena). Transfer to Claude fails, undermining the genericity claim."
     },
     {
       "flag": "No cost reporting",
       "detail": "The fuzzing process involves many LLM API calls across iterations but no cost or time information is reported, making practical applicability hard to assess."
     },
     {
       "flag": "Inconsistent VWA-adv numbers",
       "detail": "The abstract and Section 5.2 claim 70% success on VWA-adv but Table 1 shows 60% ASR on the fuzzing set for GPT-4o. The 70% may refer to coverage rather than ASR, which is not clearly distinguished."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramer"],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Primary benchmark used for evaluating indirect prompt injection attacks against personal assistant agents."
     },
     {
       "title": "Dissecting Adversarial Robustness of Multimodal LM Agents",
       "authors": ["Chih-Hui Wu", "Rushil R. Shah", "Jing Yu Koh", "Ruslan Salakhutdinov", "Daniel Fried", "Aditi Raghunathan"],
       "year": 2024,
       "relevance": "Second benchmark (VWA-adv) used for evaluating attacks against web agents with multimodal input."
     },
     {
       "title": "GPTFuzzer: Red Teaming Large Language Models with Auto-Generated Jailbreak Prompts",
       "authors": ["Jiahao Yu", "Xingwei Lin", "Xinyu Xing"],
       "year": 2023,
       "arxiv_id": "2309.10253",
       "relevance": "Key prior work applying fuzzing to direct prompt injection/jailbreaking; AGENTVIGIL extends this to indirect injection in agents."
     },
     {
       "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
       "authors": ["Eric Wallace", "Kai Xiao", "Reimar Leike", "Lilian Weng", "Johannes Heidecke", "Alex Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Defense mechanism evaluated in the paper; results suggest instruction hierarchy is insufficient against AGENTVIGIL attacks."
     },
     {
       "title": "Formalizing and Benchmarking Prompt Injection Attacks and Defenses",
       "authors": ["Yi Liu", "Yujia Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Provides formalization of prompt injection attacks and baseline used for comparison (OpenPromptInjection)."
     },
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for indirect prompt injection in tool-integrated agents; used as additional baseline comparison."
     },
     {
       "title": "AgentPoison: Red-teaming LLM Agents via Poisoning Memory or Knowledge Bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang", "Chaowei Xiao", "Dawn Song", "Bo Li"],
       "year": 2024,
       "arxiv_id": "2407.12784",
       "relevance": "Related attack method that poisons agent memory/knowledge bases; requires white-box access unlike AGENTVIGIL."
     },
     {
       "title": "Defending Against Indirect Prompt Injection Attacks with Spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati", "Yonatan Zunger", "Emre Kiciman"],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Defense strategy (delimiter-based spotlighting) evaluated against AGENTVIGIL's attacks."
     },
     {
       "title": "PromptFuzz: Harnessing Fuzzing Techniques for Robust Testing of Prompt Injection in LLMs",
       "authors": ["Jiahao Yu", "Yuhang Shao", "Hanwen Miao", "Jianwei Shi", "Xinyu Xing"],
       "year": 2024,
       "arxiv_id": "2409.14729",
       "relevance": "Related fuzzing approach for testing prompt injection robustness in LLMs."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou", "Frank F. Xu", "Hao Zhu", "Xuhui Zhou", "Robert Lo", "Abishek Sridhar", "Xianyi Cheng", "Tianyue Ou", "Yonatan Bisk", "Daniel Fried"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Provides the real-world web environment used for the case study evaluation in Section 5.4."
     }
   ]
 }