ai-research-survey

scan.json (32051B)

---

 {
   "paper": {
     "title": "QueryIPI: Query-agnostic Indirect Prompt Injection on Coding Agents",
     "authors": [
       "Yuchong Xie",
       "Zesen Liu",
       "Mingyu Luo",
       "Zhixiang Zhang",
       "Kaikai Zhang",
       "Yuanyuan Yuan",
       "Zongjie Li",
       "Ping Chen",
       "Shuai Wang",
       "Dongdong She"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.23675",
     "doi": "10.48550/arXiv.2510.23675"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "QueryIPI introduces query-agnostic indirect prompt injection for coding agents, achieving 87% attack success rate on simulated agents with just 8 training samples, far exceeding the best baseline (50%). The method leverages leaked system prompts ('system invariants') to guide black-box optimization of malicious tool descriptions. Attacks transfer from simulation to real-world coding agents at 50% ASR, and resist both prevention-based (Sandwich Defense, Spotlighting) and detection-based (perplexity) defenses.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided: 'Source code: https://github.com/QueryIPI/QueryIPI.git' on the first page of the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: the Command Injection Payload List (https://github.com/payloadbox/command-injection-payload-list) and LMSYS-Chat-1M (Zheng et al., 2024). Both are public and accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper. Only the backend LLM (Claude-Sonnet-4) and temperature settings are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The algorithm is described (Algorithm 1) and parameter settings given (Section 5.1), but there are no explicit instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are point estimates (ASR values) with no confidence intervals or error bars reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'QueryIPI significantly outperforms the baselines' (Section 5.2) based solely on comparing ASR numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute ASR values are reported with baseline context throughout (e.g., 'average ASR of 0.70 even when trained with just two samples' vs baseline 0.50). Relative improvements are contextualized with the baseline reference points in Tables 1-4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set of 100 cases (10 commands × 10 queries) is used without any justification for why this sample size is adequate. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. Temperature is set to 0 for determinism, but the mutator LLM uses temperature 1, introducing stochasticity in the optimization process whose variance is never measured."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are included: AgentDojo, InjecAgent, MCPTox, and TIP (TIPExploit). Results compared in Table 1. The paper also justifies excluding ToolHijacker (Section 5.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2024-2025: AgentDojo (2024), InjecAgent (2024), MCPTox (2025), and TIPExploit (2025). These represent recent work in the IPI space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 5.3, Table 2) ablates the internal prompt component, testing 'w/o Internal Prompt' (average ASR drops from 70% to 20%) and 'Partial Prompt' (71% ASR). This demonstrates the contribution of the system invariant."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: Attack Success Rate (ASR). The perplexity analysis in Section 5.5 is for defense evasion evaluation, not for measuring attack effectiveness. No secondary metrics like attack latency, optimization cost, or attack specificity are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via an LLM judge (Section B). While the judge was calibrated against human annotators (152 data points, Spearman's ρ = 0.78), human evaluation was not used as a primary evaluation method for the attack results."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses 2/4/8 queries for optimization, while ASR is measured on a separate set of 100 test cases (10 commands × 10 queries) randomly sampled from LMSYS-Chat-1M (Section 5.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per agent (Cursor, Copilot, Cline, Trae, Windsurf) in Tables 1-4, and per training sample count (n=2, 4, 8). Per-command breakdowns are not provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses failure modes when the internal prompt is unavailable (ASR drops to 20%). Section 5.4 acknowledges performance degradation in real-world transfer. The mutation prompt design (Section 4.2) explicitly addresses two failure cases: irrelevant responses and safety refusals."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D (RQ5) reports poor cross-LLM transferability for Copilot (ASR drops from 85% to near 0% on non-Claude backends). Real-world transfer (Table 3) shows degradation from 87% to 50%. The w/o Internal Prompt ablation shows 20% ASR."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 70%/82%/87% ASR with 2/4/8 samples are supported by Table 1. The claim that the best baseline achieves 50% is supported by TIP's average in Table 1. Real-world transferability claim is supported by Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim — that the internal prompt 'is a key driver of the attack's success' (Section 5.3) — is supported by a controlled ablation: removing internal prompt access drops average ASR from 70% to 20% (Table 2). This is a single-variable manipulation with the rest of the framework held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims query-agnostic IPI is 'highly effective in practice,' but main results are from simulated agents (not real-world), and real-world transfer (Table 3) shows substantial degradation (87%→50%). The title says 'Coding Agents' broadly, but only 5 IDE agents were tested. Cross-LLM transfer (Table 4) shows highly variable results (0% to 95%)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for QueryIPI's success. Could the improvement over baselines be due to the iterative optimization alone rather than the system invariant insight? The paper does not consider confounding factors or alternative mechanisms."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures ASR (whether a malicious command is executed) and claims this demonstrates attack effectiveness. The measurement directly matches the claim — there is no proxy gap between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The backend LLM is specified as 'Claude-Sonnet-4 (Anthropic, 2025)' without a snapshot date or API version. Cross-LLM experiments use 'GPT-5', 'Grok-4', and 'Gemini-2.5-pro' — all marketing names without version specifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E.1 and E.2 describe the mutation and judge prompts in a modular, third-person summary format ('Role Module: Defines the operational position...'), not the actual prompt text sent to the LLM. The reader cannot reconstruct the exact prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports: temperature 0 for target agent and judge, temperature 1 for mutator, 20 generations, 2 variants per generation. These are the key hyperparameters for the optimization process."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The QueryIPI framework is described in detail: Algorithm 1 outlines the iterative optimization, Section 4.2 describes the mutation function (initial seed generation and reflective optimization phases), and the scoring function is detailed in Section B. The simulated agent setup (leaked system prompts + Claude-Sonnet-4, single-turn evaluation) is described in Section 5.1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper says queries are 'randomly sampled' from LMSYS-Chat-1M and 10 commands are 'selected' from the Command Injection Payload List. No filtering criteria, sampling methodology, or preprocessing steps are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is a dedicated section discussing scope restrictions (coding agents only), architectural prerequisites (tool integration support), and operational requirements (capable backend LLM)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses scope boundaries (coding agents, tool integration support) but does not address specific threats to the validity of the experimental findings, such as the representativeness of the simulated environment, adequacy of 100 test cases, or reliability of the LLM judge for final ASR determination."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'we limit our scope to coding agents in this work,' the target system 'must support user-defined tools,' and the attack 'is not limited to coding agents but theoretically extends to any framework satisfying these conditions.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual agent responses, judge scores per test case, generated malicious descriptions) is provided or linked. Only aggregated ASR values are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes data collection: 10 commands selected from the Command Injection Payload List, queries randomly sampled from LMSYS-Chat-1M, test cases constructed as 10 commands × 10 queries = 100 test cases."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (Command Injection Payload List, LMSYS-Chat-1M)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from agent responses to binary ASR is not fully documented. The scoring function (Section B) produces continuous 0-100 scores, but the paper does not clearly state how final ASR binary success/failure is determined for the test evaluation (e.g., whether score=100 is required or some threshold is used)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: HKUST, Fudan University, and Tsinghua University. The paper evaluates third-party products (Cursor, Copilot, etc.) with no apparent affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not evidence of absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming/security study that tests attack effectiveness against coding agents, not a benchmark evaluation of model knowledge. Contamination of training data is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests attack methods, not model knowledge on benchmarks. Train/test contamination in the traditional sense is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates attack success rates, not model performance on knowledge benchmarks. Benchmark contamination is not a relevant concern for this study type."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments are automated evaluations of attack methods against LLM-based coding agents."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Section 8 discusses ethical considerations related to dual-use risk of the attack methodology, but no IRB review is applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time for the attack optimization or evaluation are reported. The method requires multiple LLM calls (mutator, target agent, judge) across 20 generations but no cost quantification is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (API spend, GPU hours, total calls) is stated for the experiments despite extensive use of Claude-Sonnet-4 across 5 agents × multiple configurations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Target agent and judge use temperature 0 for determinism, but the mutator uses temperature 1, introducing stochasticity. No results across multiple random seeds are reported to assess optimization variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "It is not stated how many times the full optimization was run per agent-command configuration. Results appear to be from single optimization runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (20 generations, 2 variants, temperature settings) are stated but no search budget or justification for these values is provided. No description of how these hyperparameters were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The optimization process selects the best-performing description via training set scores (Algorithm 1, SelectTop function), and final evaluation is on a separate test set of 100 cases. For RQ3, they select 'the most effective malicious tool description from our RQ1 analysis' (Section 5.4)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own QueryIPI against baselines without acknowledging author-evaluation bias. Baselines were manually adapted by the authors ('we manually adapted the malicious payloads to suit each of the 10 malicious commands'), potentially disadvantaging them."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "QueryIPI requires iterative optimization (20 generations with LLM calls for mutation, scoring, and judging) while baselines are non-iterative. This compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASR on simulated agents with 100 test cases validly measures real-world attack threat. The sim-to-real gap is acknowledged (Section 5.4) but construct validity of the simulated benchmark is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All five simulated agents use Claude-Sonnet-4 as backend but differ in system prompts and tool definitions. The paper does not discuss whether performance differences across agents are attributable to scaffold differences vs. other factors. RQ5 varies the backend LLM but does not disentangle model vs. scaffold effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LMSYS-Chat-1M queries or command injection payloads could have been seen during Claude-Sonnet-4's training, potentially affecting the model's response patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The malicious tool description is directly injected into the agent's context, which is the intended attack vector, but no analysis of whether the test setup creates unrealistic advantages."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training and test queries are both randomly sampled from LMSYS-Chat-1M. No analysis of whether independence between training and test sets is maintained (e.g., duplicate or near-duplicate queries)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No deduplication, temporal splitting, or contamination checking between training and test sets."
       }
     }
   },
   "claims": [
     {
       "claim": "QueryIPI achieves average ASR of 70%, 82%, and 87% with 2, 4, and 8 training samples across five simulated coding agents.",
       "evidence": "Table 1 (Section 5.2) shows per-agent ASR results across training sample configurations. Average ASR: 0.70 (n=2), 0.82 (n=4), 0.87 (n=8).",
       "supported": "strong"
     },
     {
       "claim": "QueryIPI significantly outperforms baselines, with the best baseline (TIP) achieving only 50% average ASR.",
       "evidence": "Table 1 shows AgentDojo (0.02), InjecAgent (0.00), MCPTox (0.00), TIP (0.50) average ASR vs QueryIPI's 0.70-0.87.",
       "supported": "moderate"
     },
     {
       "claim": "The internal prompt is a key driver of attack success.",
       "evidence": "Table 2 (Section 5.3) shows removing internal prompt access drops ASR from 70% to 20%. Partial prompt access yields 71% ASR.",
       "supported": "moderate"
     },
     {
       "claim": "Malicious tool descriptions transfer from simulated to real-world coding agents.",
       "evidence": "Table 3 (Section 5.4) shows QueryIPI achieves 50% average ASR on real-world agents vs TIP's 2%. Trae (72%) and Cursor (63%) show strongest transfer.",
       "supported": "moderate"
     },
     {
       "claim": "QueryIPI is robust against prevention-based and detection-based defenses.",
       "evidence": "Section 5.5: Sandwich Defense yields 82% ASR, Spotlighting variants 52-84% ASR. Perplexity (59.04) is far below real MCP tools (1423.27).",
       "supported": "moderate"
     },
     {
       "claim": "QueryIPI has notable cross-LLM transferability with 26% average ASR across different backends.",
       "evidence": "Table 4 (Appendix D) shows variable results: 40% average on GPT-5, 21% on Grok-4, 16% on Gemini-2.5-pro. Copilot drops to near 0% for all non-Claude backends.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Simulated environment as primary evaluation",
       "detail": "Main results (Table 1) use simulated agents built from leaked system prompts + Claude-Sonnet-4, not actual coding agents. Real-world transfer (Table 3) shows substantial degradation from 87% to 50% ASR, indicating the simulation may overestimate attack effectiveness."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results are point estimates from apparently single optimization runs. The mutator LLM uses temperature 1, introducing stochasticity, yet no variance across multiple runs is reported. Different optimization runs could yield very different malicious descriptions."
     },
     {
       "flag": "Small and unjustified test set",
       "detail": "ASR is measured on only 100 test cases (10 commands × 10 queries) with no justification for this sample size. With such small N per agent-command combination, the reported ASR values have wide implicit confidence intervals."
     },
     {
       "flag": "Baseline adaptation by authors",
       "detail": "The authors manually adapted baseline payloads for each of the 10 malicious commands (Section 5.1). This introduces potential bias — the baselines were designed for different settings and may perform worse when adapted by the competing method's authors."
     },
     {
       "flag": "Unfair compute comparison with baselines",
       "detail": "QueryIPI uses iterative optimization (20 generations × multiple LLM calls) while baselines are static injection strategies. The compute advantage is neither quantified nor controlled for, making it unclear how much of the improvement comes from the method's insight vs. simply more computation."
     },
     {
       "flag": "LLM judge reliability for final evaluation",
       "detail": "The scoring function uses a continuous 0-100 scale, but the paper reports binary ASR without clearly specifying the threshold. The judge was validated on 152 data points (Spearman's ρ = 0.78), which indicates good but not excellent alignment with human judgment."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr"],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Benchmark for evaluating indirect prompt injection attacks and defenses on LLM agents."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for indirect prompt injection in tool-integrated LLM agents, used as baseline."
     },
     {
       "title": "AgentPoison: Red-teaming LLM agents via poisoning memory or knowledge bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang", "Chaowei Xiao", "Dawn Song", "Bo Li"],
       "year": 2024,
       "arxiv_id": "2407.12784",
       "relevance": "Red-teaming approach for LLM agents through poisoning memory and knowledge bases."
     },
     {
       "title": "MCPTox: A benchmark for tool poisoning attack on real-world MCP servers",
       "authors": ["Zhiqiang Wang", "Yichao Gao", "Yanting Wang"],
       "year": 2025,
       "relevance": "Benchmark for tool poisoning attacks on Model Context Protocol servers, used as baseline."
     },
     {
       "title": "AGENTVIGIL: Generic black-box red-teaming for indirect prompt injection against LLM agents",
       "authors": ["Zhun Wang", "Vincent Siu", "Zhe Ye"],
       "year": 2025,
       "relevance": "Generic black-box red-teaming method for indirect prompt injection against LLM agents."
     },
     {
       "title": "Exploit tool invocation prompt for tool behavior hijacking in LLM-based agentic system",
       "authors": ["Yuchong Xie", "Mingyu Luo", "Zesen Liu"],
       "year": 2025,
       "relevance": "Prior work on tool behavior hijacking in LLM agents via tool invocation prompts, used as primary baseline (TIP)."
     },
     {
       "title": "Prompt injection attack to tool selection in LLM agents",
       "authors": ["Jiawen Shi", "Zenghui Yuan", "Guiyao Tie"],
       "year": 2025,
       "relevance": "ToolHijacker attack on tool selection in LLM agents, discussed as related but excluded baseline."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati", "Yonatan Zunger", "Emre Kiciman"],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Defense mechanism against indirect prompt injection using data marking and delimiting, evaluated as defense in this paper."
     },
     {
       "title": "Tree of attacks: Jailbreaking black-box LLMs automatically",
       "authors": ["Anay Mehrotra", "Manolis Zampetakis", "Paul Kassianik"],
       "year": 2024,
       "relevance": "Automated jailbreaking framework (TAP) whose scoring approach inspired QueryIPI's scoring function."
     },
     {
       "title": "Adaptive attacks break defenses against indirect prompt injection attacks on LLM agents",
       "authors": ["Qiusi Zhan", "Richard Fang", "Henil Shalin Panchal", "Daniel Kang"],
       "year": 2025,
       "relevance": "Demonstrates that adaptive attacks can bypass defenses against indirect prompt injection in LLM agents."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "relevance": "Key finding about LLMs ignoring information in long contexts, motivating QueryIPI's system invariant approach."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analysis of LLM safety training failure modes, relevant to understanding why prompt injection attacks succeed."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "relevance": "Foundational work on universal adversarial attacks against aligned LLMs."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security researchers and AI tool developers can use the methodology to test coding agent defenses, though it requires significant setup to replicate."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Demonstrates that query-agnostic attacks are feasible, challenging the assumption that IPI attacks are limited to specific trigger queries."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Shows that widely-used coding agents (Cursor, Copilot, Windsurf, Cline, Trae) can be reliably compromised to execute arbitrary commands regardless of user intent."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implicates major coding tools as vulnerable and highlights that leaked system prompts enable attacks, touching on supply chain and MCP security concerns."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code is released on GitHub, though the attack requires setting up simulated agents and the paper deliberately does not release ready-to-use attack payloads."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Targets well-known products (Cursor, GitHub Copilot, Windsurf, Cline, Trae) but authored by academic researchers without major industry affiliation."
     }
   }
 }