ai-research-survey

scan.json (27736B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Prompt Injection Attack to Tool Selection in LLM Agents",
     "authors": ["Jiawen Shi", "Zenghui Yuan", "Guiyao Tie", "Pan Zhou", "Neil Zhenqiang Gong", "Lichao Sun"],
     "year": 2025,
     "venue": "NDSS 2026",
     "arxiv_id": "2504.19793"
   },
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ToolHijacker achieves high attack success rates (up to 99.6%) against tool selection in LLM agents by injecting a single malicious tool document, even in no-box scenarios where the attacker has no access to the target system. The attack transfers across architecturally different LLMs (e.g., 96.7% ASR from Llama-3.3-70B shadow to GPT-4o target). Existing defenses — both prevention-based (StruQ, SecAlign) and detection-based (known-answer, DataSentinel, PPL) — are insufficient, with gradient-free attacks achieving 99.6% ASR even under StruQ.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We will release code and data under restricted access — interested parties must request permission' in the Ethics section. This is not a public release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two public benchmark datasets: MetaTool and ToolBench, both publicly available. The 10 target tasks and 1,000 task descriptions per dataset are described in detail in Appendix C (Figures 14-15)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes methods but does not include scripts or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '96.7% ASR') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'outperforms baselines') based solely on comparing percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: e.g., 'gradient-free attack achieves a higher ASR by 4.5% when targeting GPT-4o' and 'Llama-3-8B increases the ASR by 15.12% over Llama-2-7B' (Section IV-C). Absolute percentages with baselines provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 target task descriptions per task and 10 tasks per dataset but provides no justification for why these numbers are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single experimental runs averaged across tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven baselines are compared: five manual attacks (naive, escape characters, context ignore, fake completion, combined) and two automated attacks (JudgeDeceiver, PoisonedRAG). Results in Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent automated attacks: JudgeDeceiver (CCS 2024) and PoisonedRAG (2024). Also includes StruQ and SecAlign as defense baselines. These represent current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies in Section IV-C: impact of R vs S components (Table V), impact of k, k', shadow task descriptions (Figure 6), shadow LLM choice (Tables VI-VII), similarity metrics (Table VIII), loss terms (Table XV), and hyperparameters α and β (Figure 9)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics used: ACC (accuracy without attack), ASR (attack success rate), HR (hit rate for retrieval), and AHR (attack hit rate). Defined in Section IV-A5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A user study with 6 participants was conducted to evaluate whether humans can detect malicious tool documents (Table XVII in Appendix B)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Shadow task descriptions Q' used for optimization are explicitly disjoint from target task descriptions Q (Q ∩ Q' = ∅, Section II-B). The 100 target task descriptions per task are distinct from the 5 shadow descriptions used for optimization."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per task (Figure 3 shows 10 tasks), per LLM (Table I shows 8 LLMs), per retriever (Table IV shows 4 retrievers), and per dataset (MetaTool vs ToolBench throughout)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the attack is less effective: Claude-3-Haiku is 'the least sensitive' (Section IV-B), gradient-based attack with Llama-2-7B shadow drops to 34% ASR on Llama-3-70B (Table VII), and low k values reduce effectiveness (Figure 5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings reported: defenses are insufficient (Section V), gradient-based attack has lower transferability with weak shadow LLMs (Table VII), small k' leads to declining ASR (Figure 5), and removing any loss term significantly hurts performance (Table XV)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about high ASR, outperforming baselines, and defenses being insufficient are all supported by Tables I, III, and IX-X respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily from ablation studies (e.g., 'removing L3 reduces ASR from 95% to 5%' in Table XV) which use controlled single-variable manipulation. The two-phase design isolates R's effect on retrieval and S's effect on selection (Table V)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Prompt Injection Attack to Tool Selection in LLM Agents' broadly, but the evaluation is limited to a specific two-step retrieval+selection pipeline. Many LLM agents use different tool selection mechanisms (e.g., function calling APIs) that are not tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why attacks transfer ('shared alignment objectives and training paradigms make LLMs inherently vulnerable' and 'LLM homogenization'), why gradient-free outperforms gradient-based on closed-source models, and why Claude-3-Haiku is more resistant (Section IV-B)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes what it measures (ASR = selecting the malicious tool name) from the broader security implication (executing harmful tools). The metrics are precisely defined (Section IV-A5) and the paper explicitly discusses the gap between tool selection manipulation and actual harm potential."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed: Llama-2-7B-chat, Llama-3-8B-Instruct, Llama-3-70B-Instruct, Llama-3.3-70B-Instruct, Claude-3-Haiku, Claude-3.5-Sonnet, GPT-3.5, GPT-4o (Section IV-A3). However, no snapshot dates for closed-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided: the selection prompt (Figure 2), shadow task description generation prompt (Figure 10), shadow tool document generation prompt (Figure 11), attacker LLM system instruction (Figure 13), and initial R and S (Figure 12)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters reported in Section IV-A4: m'=5, k'=5, Titer=10, B=2, W=10 for gradient-free; α=2.0, β=0.1, R iterations=3, S iterations=400 for gradient-based."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-step tool selection pipeline (retrieval + selection) is formally described in Section II-A with mathematical formulations. The shadow framework construction is detailed in Section III."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "For ToolBench: 'After removing duplicate tools and empty descriptions, the tool library contains 9,650 benign tool documents' from 16,464 originals (Section IV-A1). Task description generation process described with templates."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusion mentions future work directions but does not discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential confounds or weaknesses in the experimental design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. It doesn't acknowledge that the attack is only tested on one specific tool selection architecture (retrieval + selection) and may not apply to other agent frameworks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental results (individual trial outcomes, per-query results) are not available. Only aggregated percentages are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: MetaTool (21,127 instances, 199 tools from OpenAI Plugins) and ToolBench (126,486 samples, 16,464 tools from RapidAPI). Task descriptions generated via LLM + human evaluation (Section IV-A1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The user study mentions '6 participants' but provides no details on how they were recruited, their background, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from dataset selection through shadow framework construction, optimization, and evaluation is documented in Sections III and IV. ToolBench filtering from 16,464 to 9,650 tools is explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Huazhong University of Science and Technology, Duke University, and Lehigh University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests prompt injection attacks against tool selection, not model knowledge on benchmarks. The LLMs are used as tool selectors, not evaluated for their pre-trained knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests attack effectiveness on tool selection, not model capability on benchmark tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — contamination of benchmark solutions in training data is irrelevant to this attack evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The user study with 6 participants evaluating malicious tool documents is not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics approval is mentioned for the user study. The Ethics section discusses responsible disclosure and informed consent but not IRB review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The 6 participants are not characterized — no information about their expertise, background, or demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for the 6 participants are stated."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a between-subjects experiment requiring randomization — all participants evaluated the same tool documents."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding details are provided for the user study. It's unclear whether participants knew the ratio of malicious to benign tools."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about whether all 6 participants completed the study or if any were excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports attack costs: gradient-free R requires 1 LLM query and S requires ~18 LLM queries; gradient-based R requires ~1 GPU-minute and S requires ~8 GPU-hours on one NVIDIA A800 GPU."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compute budget stated in Appendix B: gradient-based requires ~8 GPU-hours on one NVIDIA A800 GPU for S optimization. Gradient-free requires ~19 LLM queries total."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. It is unclear whether results are from single or multiple optimization/evaluation runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While hyperparameter sensitivity is explored (α, β in Figure 9), no search budget is reported for finding the default values (α=2.0, β=0.1)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper shows ablation results across multiple α and β values (Figure 9) and reports that ASR remains above 95% for a range of values, justifying the chosen defaults."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their attack and the baselines (JudgeDeceiver, PoisonedRAG) without acknowledging potential bias in re-implementing competitors' methods."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of performance at matched compute budgets between ToolHijacker and baselines. The gradient-based method uses 8 GPU-hours while baselines may use significantly less."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether MetaTool and ToolBench adequately represent real-world tool selection scenarios. No discussion of construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a specific tool selection pipeline, not comparing models across different scaffolds. The pipeline IS the thing being attacked."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attack effectiveness, not model knowledge. Temporal leakage of benchmark solutions is not relevant to measuring whether an LLM selects a malicious tool."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation measures attack success on tool selection, not model capability."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly ensures independence between shadow and target descriptions (Q ∩ Q' = ∅) and evaluates on target tasks not used during optimization. Table XII shows 0% ASR on non-target tasks."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper is not evaluating pre-trained model knowledge on benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "ToolHijacker achieves high attack success rates across different LLMs, with gradient-free achieving 96.7% ASR on GPT-4o (MetaTool) using Llama-3.3-70B as shadow LLM.",
       "evidence": "Table I shows ASRs across 8 LLMs and 2 datasets. GPT-4o gradient-free ASR = 96.7% on MetaTool, 88.2% on ToolBench.",
       "supported": "strong"
     },
     {
       "claim": "ToolHijacker significantly outperforms existing prompt injection attacks when applied to tool selection.",
       "evidence": "Table III: gradient-free achieves 96.7% vs best baseline PoisonedRAG at 39.3% on MetaTool with GPT-4o. All 7 baselines are far below ToolHijacker.",
       "supported": "strong"
     },
     {
       "claim": "Prevention-based defenses (StruQ, SecAlign) fail to defend against ToolHijacker.",
       "evidence": "Table IX: gradient-free achieves 99.6% ASR under StruQ on MetaTool. SecAlign reduces ASR to 97.5% on MetaTool. Both defenses are insufficient.",
       "supported": "strong"
     },
     {
       "claim": "Detection-based defenses are insufficient, with known-answer detection and DataSentinel having FNRs exceeding 90%.",
       "evidence": "Table X: known-answer detection has 100% FNR for both attacks. DataSentinel has 100% FNR for gradient-free, 90% for gradient-based. PPL detects some gradient-based but misses 90%.",
       "supported": "strong"
     },
     {
       "claim": "The attack is targeted with minimal impact on non-target tasks.",
       "evidence": "Table XII: gradient-free achieves 0% ASR and 0.22% AHR on non-target tasks; gradient-based achieves 0.11% ASR and 4% AHR.",
       "supported": "strong"
     },
     {
       "claim": "Humans struggle to detect malicious tool documents generated by ToolHijacker.",
       "evidence": "Table XVII: 6 participants failed to detect ≥71% of malicious tools (FNR 71-100%) while incorrectly flagging 5.6-30.35% of benign tools.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny user study sample",
       "detail": "The human detection study uses only 6 participants with no reported demographics, recruitment criteria, or statistical analysis. This is insufficient to draw conclusions about human detection ability."
     },
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "All results are point estimates with no error bars, confidence intervals, or variance across runs. Given the stochastic nature of LLM inference, results could vary across runs."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section and does not discuss scope boundaries. The attack is only tested on one specific tool selection architecture (retrieval + selection with dual encoders) but is presented broadly."
     },
     {
       "flag": "Self-comparison bias in baseline implementations",
       "detail": "The authors implement competitor baselines (JudgeDeceiver, PoisonedRAG) themselves without acknowledging potential bias. These methods were designed for different problem settings and may be disadvantaged."
     },
     {
       "flag": "Closed-source model versions unspecified",
       "detail": "GPT-3.5, GPT-4o, Claude-3-Haiku, Claude-3.5-Sonnet are listed without snapshot dates or API versions. Model behavior can change across versions, affecting reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Key agent framework for code-level software engineering tasks."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["S. Hong", "X. Zheng", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent collaborative framework for software development."
     },
     {
       "title": "Gorilla: Large language model connected with massive APIs",
       "authors": ["S. G. Patil", "T. Zhang", "X. Wang"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "LLM agent connected with massive APIs for tool use."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Y. Qin", "S. Liang", "Y. Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Major benchmark for LLM tool use with 16K+ APIs, used as evaluation dataset in this paper."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection in LLM-integrated applications."
     },
     {
       "title": "PoisonedRAG: Knowledge poisoning attacks to retrieval-augmented generation of large language models",
       "authors": ["W. Zou", "R. Geng", "B. Wang", "J. Jia"],
       "year": 2024,
       "arxiv_id": "2402.07867",
       "relevance": "Key baseline: adversarial text injection into RAG knowledge bases."
     },
     {
       "title": "Optimization-based prompt injection attack to LLM-as-a-Judge",
       "authors": ["J. Shi", "Z. Yuan", "Y. Liu"],
       "year": 2024,
       "relevance": "JudgeDeceiver - gradient-optimized prompt injection baseline for LLM judging."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["S. Chen", "J. Piet", "C. Sitawarin", "D. Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Prevention-based defense against prompt injection evaluated in this paper."
     },
     {
       "title": "Aligning LLMs to be robust against prompt injection",
       "authors": ["S. Chen", "A. Zharmagambetov", "S. Mahloujifar"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "SecAlign defense against prompt injection via preference optimization."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": ["Y. Liu", "Y. Jia", "J. Jia", "D. Song", "N. Z. Gong"],
       "year": 2025,
       "relevance": "State-of-the-art detection-based defense against prompt injection."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["E. Debenedetti", "J. Zhang", "M. Balunovic"],
       "year": 2024,
       "relevance": "Comprehensive evaluation framework for prompt injection in LLM agents."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Q. Zhan", "Z. Liang", "Z. Ying", "D. Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for indirect prompt injection in tool-using LLM agents."
     },
     {
       "title": "From allies to adversaries: Manipulating LLM tool-calling through adversarial injection",
       "authors": ["H. Wang", "R. Zhang", "J. Wang"],
       "year": 2024,
       "arxiv_id": "2412.10198",
       "relevance": "Adversarial manipulation of LLM tool calling mechanisms."
     },
     {
       "title": "A critical evaluation of defenses against prompt injection attacks",
       "authors": ["Y. Jia", "Z. Shao", "Y. Liu"],
       "year": 2025,
       "arxiv_id": "2505.18333",
       "relevance": "Critical evaluation showing defenses sacrifice general capabilities and remain vulnerable to adaptive attacks."
     },
     {
       "title": "Defeating prompt injections by design",
       "authors": ["E. Debenedetti", "I. Shumailov", "T. Fan"],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "Security policy enforcement approach to prevent prompt injection in LLM agents."
     }
   ]
 }