ai-research-survey

scan.json (34647B)

---

 {
   "paper": {
     "title": "VORTEXPIA: Indirect Prompt Injection Attack against LLMs for Efficient Extraction of User Privacy",
     "authors": [
       "Yu Cui",
       "Sicheng Pan",
       "Yifei Liu",
       "Haibin Zhang",
       "Cong Zuo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.04261",
     "doi": "10.48550/arXiv.2510.04261"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "VORTEXPIA is an indirect prompt injection attack that uses fabricated 'false memories' to induce LLMs to proactively request PII from users under black-box settings. Evaluated on 6 LLMs across 4 datasets, it achieves up to 90.9% ASR (on Qwen2.5-72B) and ~2.37× improvement over baselines while using 54% fewer tokens. The paper finds that models with stronger reasoning abilities are more vulnerable, and that reasoning LLMs' final answers are less secure than their reasoning tokens — contradicting prior findings on adversarial attacks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is provided anywhere in the paper. The injected prompts are given in Appendix A.1 but no reproducible codebase is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets: MATH500 (Lightman et al., 2024), AIME2024, AIME2025, and AICrypto (Wang et al., 2025c). The core injected prompts are provided in Appendix A.1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided. The paper states 'All models are queried via APIs' (Appendix A.5) but provides no details about the software environment used to run experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper provides attack prompt templates in the appendix but no scripts, configuration files, or procedures to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main ASR results in Figures 3 and 4 report only point estimates with no error bars or confidence intervals. Tables 1 and 2 show ± values, but these represent variance across the evaluator group (two LLM judges), not confidence intervals or experimental uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims that VORTEXPIA 'significantly outperforms baselines' (Section 4.2) and achieves '2.37× improvement' (Figure 4) are based solely on comparing point estimates without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: '2.37× improvement' over baselines (Section 4.2), '31% improvement' on real-world applications (Table 1), '53.98% reduction' in token consumption (Figure 6), and '27% relative reduction' in PR compared to Baseline 3 (Section 4.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.5 states '150 samples from MATH500 and 118 samples from AICrypto' with no justification for these numbers. Full AIME2024 and AIME2025 datasets are used (30 problems each) but no power analysis or sample size rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The ± values in Tables 1 and 2 represent variance across the two-member evaluator group (Qwen2.5-72B and DeepSeek-V3.1), not variance across experimental runs. The main ASR results in Figures 3 and 4 are single-run point estimates with no indication of result stability across repeated experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baseline attack methods from Zhan et al. (2025b) are included: Direct CAI (Baseline 1), Reciprocal CAI (Baseline 2), and User-benefits CAI (Baseline 3). A fourth conventional PIA baseline is included for PR evaluation (Section 4.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from Zhan et al. (2025b), which is the most recent and directly relevant prior work on LLM-based privacy extraction. The paper positions User-benefits CAI as the current SOTA (Table 1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The paper tests different Np values (6, 10, 13) for the privacy set size (Figure 5), which is a parameter sensitivity analysis, but does not ablate individual components of the attack design (e.g., false memories, no-disclosure instruction, batch requesting)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are defined and used: ASR (attack success rate), MR (matching rate between requested and specified privacy items), and PR (positive detection rate). Token consumption is also reported as a cost metric (Figure 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states that for MR evaluation, 'human annotators further validate the results for accuracy' after the LLM evaluator performs the initial assessment. However, human evaluation is limited to MR validation only; ASR and PR rely entirely on LLM-as-a-Judge."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation between development and test data is described. It is unclear whether the attack prompts (Appendix A.1) were designed and tuned using any of the four evaluation datasets. No dev/test split is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (MATH500, AIME2024, AIME2025, AICrypto) and per model (6 LLMs) in Figure 3. Table 1 provides per-application breakdowns. Figure 5 shows MR across different Np values and datasets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses failure modes: 'VORTEXPIA performs poorly on Qwen3-32B when tested on the challenging mathematical datasets' because the model 'sometimes ignores the task of requesting private data after solving the mathematical problem.' The paper also shows DeepSeek-R1's 'stuck' states in detail (Appendix A.6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that VORTEXPIA underperforms on Qwen3-32B for hard math datasets (Section 5.2). Figure 3c shows VortexPIA scoring lower than Baseline 1 on AICrypto for Qwen3-32B. DeepSeek-R1 results show relatively lower ASR compared to some baselines on certain datasets (Figure 3e)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of SOTA performance are supported by Figure 4 (~2.37× average improvement). Reduced token consumption is supported by Figure 6 (53.98% reduction). Enhanced robustness against defenses is supported by Table 2 (PR of 45% vs 61% for the SOTA baseline). The claim of practical effectiveness is supported by Table 1 (31% improvement on real-world apps)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 claims 'as the reasoning ability of the targeted LLM increases, the severity of the exposed privacy threat also escalates.' This causal claim is based on observational comparison across models that differ in multiple dimensions (size, architecture, training). Reasoning ability is confounded with model size (32B to 1000B parameters) and training methodology."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims the attack works 'against LLMs' generally, and the abstract claims it works 'under black-box settings.' However, testing is limited to 6 LLMs (all Chinese-developed or open-source), 4 datasets (3 math + 1 crypto), and 2 real-world applications. No GPT, Claude, or Gemini models are tested. The generalization to other application domains beyond math/crypto queries is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 analyzes the attack mechanism (reasoning 'stuck' states) but does not consider alternative explanations for the results. For example, the high ASR on Qwen2.5-72B could be due to weaker safety training rather than stronger reasoning. The paper does not discuss whether results could be explained by model-specific safety alignment differences rather than general reasoning capability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "ASR is measured by LLM-as-a-Judge scoring of whether responses contain PII requests, which is a proxy for actual user privacy risk. The paper does not discuss the gap between LLM-judged ASR and real-world privacy harm. Whether users would actually disclose PII in response to these requests is cited from prior work (Zhan et al., 2025b) but not validated for VORTEXPIA's specific attack outputs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are specified as Qwen2.5-32B-Instruct, Qwen2.5-72B-Instruct, Qwen3-32B, DeepSeek-V3, DeepSeek-R1-0528, and Kimi-K2-Instruct. Most include sufficient version information (family + version + size + variant). DeepSeek-R1-0528 includes a date suffix. DeepSeek-V3 lacks a snapshot date but references the specific technical report."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the actual prompt text for: the core injected prompt of VORTEXPIA (A.1), the ASR evaluator system prompt (A.2), the MR evaluator system prompt (A.2), the prevention system prompt (A.3), and the detection system prompt (A.4). These are complete, usable prompt texts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states only 'All models use default values for parameters, such as temperature.' The actual default values (temperature, top-p, max tokens) are not specified. Defaults vary across providers and can change over time, so this is insufficient for reproduction."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The core attack is a simple prompt injection appended to external data — no agentic scaffolding is used. The real-world applications (DeepSearch, LongTermMemory) are evaluated as black-box third-party systems whose internal scaffolding the authors cannot be expected to describe."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.5 mentions '150 samples from MATH500 and 118 samples from AICrypto' and 'We exclude data instances that cause model execution to fail,' but does not specify how many instances were excluded, what constitutes execution failure, or how the 150/118 subsets were selected from the full datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations and Ethical Considerations' is present. It acknowledges dataset coverage limitations and states the work is 'intended solely for scientific research purposes.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 mentions only generic limitations: 'Due to resource limitations, we could not assess its performance on additional datasets' and plans for future work on defense mechanisms. No specific threats to validity of the current results are discussed (e.g., LLM-as-a-Judge reliability, evaluator bias, model selection bias toward Chinese-developed LLMs)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show — for example, that results may not generalize to closed-source commercial models (GPT-4, Claude), non-STEM query domains, or real user interactions as opposed to automated benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model responses, evaluator scores, individual ASR/MR/PR values per instance) is released. Only aggregated results in figures and tables are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: user queries from benchmark datasets are appended with injected malicious data and sent to LLMs via APIs (Section 3.2, Algorithm 1). Evaluator LLMs then score responses according to defined metrics (Section 3.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for the main study. Data sources are standard public benchmarks (MATH500, AIME2024, AIME2025, AICrypto). Human annotators are mentioned for MR validation but are not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The high-level pipeline is described (inject data → query LLM → evaluate response), but intermediate steps lack detail. How many instances were excluded due to execution failure is not stated. The evaluator scoring process uses two LLM judges but aggregation details beyond averaging are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is present anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Beijing Institute of Technology (Cui, Pan, Liu, Zuo) and Yangtze Delta Region Institute of Tsinghua University (Zhang). These are academic institutions with no apparent conflict with the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding statement does not establish that no funding exists."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests an attack method's ability to induce privacy-requesting behavior, not the models' capability on any benchmark. The math/crypto datasets serve only as benign user query context. Whether models have seen the benchmark data is not relevant to the attack's success metric (ASR)."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates attack effectiveness, not model knowledge. Train/test overlap for the math benchmarks does not affect whether the model follows injected instructions to request PII."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not relevant since the benchmarks measure attack success rate, not model problem-solving capability. The attack's effectiveness depends on instruction-following and safety alignment, not benchmark memorization."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated prompt injection attacks against LLMs using benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves only automated interactions between attack prompts and LLM APIs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 reports the token consumption of the attack payload: VORTEXPIA uses 77 tokens vs. 89-243 tokens for baselines, a 53.98% reduction. This is the most relevant cost metric for an attack method (cost to mount the attack)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API calls made, total tokens processed across all experiments, wall-clock time, or total API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis across seeds. Results appear to be from single runs with no assessment of result stability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. The ± values in Tables 1 and 2 appear to represent evaluator variance (across 2 LLM judges), not multiple experimental runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The attack prompt design appears fixed with no discussion of alternative configurations tried or how the final prompt was selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only the final VORTEXPIA prompt is presented. No discussion of alternative prompt designs, how this particular formulation was chosen, or what design iterations were explored."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement the baselines from Zhan et al. (2025b) themselves and compare against their own attack. No discussion of potential bias from implementing competing methods. The original authors' implementations are not referenced or compared."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6 shows token consumption differences but does not present performance as a function of compute budget. The methods use different token amounts but performance is not normalized by compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ASR as measured by LLM-as-a-Judge actually captures real privacy risk. The paper does not question whether the benchmark setup (automated queries with math problems) reflects realistic attack scenarios or whether LLM judges reliably assess privacy extraction risk."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The main experiments directly query LLMs via APIs with no scaffolding. The real-world application experiments evaluate DeepSearch and LongTermMemory as bundled products — the scaffold IS the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. While the benchmarks serve as benign query context rather than evaluation targets, the paper notes that ASR varies by dataset difficulty (Section 4.2), suggesting model familiarity with benchmark problems could affect attack dynamics. This confound is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup appends injected data directly to user queries, and whether the query structure provides cues that affect attack success is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Problems within each dataset (e.g., MATH500) may share structural similarities that systematically affect attack success rates, but independence of test instances is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. The paper does not use temporal splits, canary strings, or other techniques to assess whether data leakage affects results."
       }
     }
   },
   "claims": [
     {
       "claim": "VORTEXPIA achieves approximately 2.37× average ASR improvement over existing attack methods across six LLMs.",
       "evidence": "Figure 4 shows average ASR across all datasets and models. VORTEXPIA achieves the highest average ASR on 5 of 6 models (38.65-90.91%) compared to baselines (0.05-43.13%). Section 4.2 states the improvement is 'about 2.37×'.",
       "supported": "moderate"
     },
     {
       "claim": "VORTEXPIA achieves 90.9% ASR on Qwen2.5-72B, the highest among all models tested.",
       "evidence": "Figure 4 shows Qwen2.5-72B achieves 90.91% ASR with VORTEXPIA. However, this is a single point estimate with no confidence interval, from LLM-judged evaluation.",
       "supported": "moderate"
     },
     {
       "claim": "VORTEXPIA reduces token consumption of injected data by 53.98% compared to baselines.",
       "evidence": "Figure 6 directly measures token counts: VORTEXPIA uses 77 tokens vs. baselines at 89, 243, and 170 tokens. This is a straightforward count.",
       "supported": "strong"
     },
     {
       "claim": "VORTEXPIA improves over SOTA by 31% on real-world LLM-integrated applications.",
       "evidence": "Table 1 shows average ASR of 86.12% for VORTEXPIA vs 65.86% for User-benefits CAI across two applications (DeepSearch, LongTermMemory) evaluated by two LLM judges. Only two applications are tested.",
       "supported": "moderate"
     },
     {
       "claim": "Models with stronger reasoning abilities pose greater privacy risks under VORTEXPIA.",
       "evidence": "Section 5.1 observes that RLLMs (DeepSeek-R1, Qwen3-32B) show higher ASR than smaller traditional LLMs (Qwen2.5-32B). However, reasoning ability is confounded with model size, architecture, and safety training. No controlled comparison isolates reasoning as the causal factor.",
       "supported": "weak"
     },
     {
       "claim": "Final answers of reasoning LLMs are less secure than their reasoning tokens under VORTEXPIA.",
       "evidence": "Section 5.2 and Appendix A.6 show DeepSeek-R1's reasoning tokens explicitly reject privacy requests ('We must not request private information') while the final answer still contains PII queries. Qualitative examples are provided but no systematic quantification across all instances.",
       "supported": "moderate"
     },
     {
       "claim": "VORTEXPIA achieves a PR of 45%, demonstrating robustness against detection-based defenses.",
       "evidence": "Table 2 shows VORTEXPIA achieves average PR of 44.88% vs. 61.44% for the SOTA baseline (User-benefits CAI). However, Baseline 2 (Reciprocal CAI) achieves lower PR (33.69%) while having weaker attack performance.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('significantly outperforms,' '2.37× improvement') are based on comparing point estimates without any statistical test (t-test, bootstrap, etc.). With no significance testing, observed differences could be within noise."
     },
     {
       "flag": "LLM-as-a-Judge evaluation reliability",
       "detail": "The primary metric (ASR) is evaluated by an LLM evaluator group of only two models (Qwen2.5-72B and DeepSeek-V3.1). The reliability and calibration of these judges is not validated. Notably, Qwen2.5-72B is both an attack target and an evaluator, creating a potential conflict."
     },
     {
       "flag": "Single-run results with no variance reporting",
       "detail": "The main results in Figures 3 and 4 appear to be from single experimental runs. The ± values in Tables 1 and 2 represent evaluator disagreement, not experimental variance. Result stability across repeated experiments is unknown."
     },
     {
       "flag": "Model selection limited to Chinese-developed LLMs",
       "detail": "All six evaluated models are Chinese-developed (Qwen, DeepSeek, Kimi). No Western commercial models (GPT-4, Claude, Gemini) or other open-source models (Llama, Mistral) are tested, limiting generalizability claims."
     },
     {
       "flag": "No code or raw data released",
       "detail": "Despite claims of SOTA performance, no code, raw model responses, or evaluator scores are released. Results cannot be independently verified or reproduced."
     },
     {
       "flag": "Reasoning ability claim confounded by model size",
       "detail": "The claim that stronger reasoning increases vulnerability (Section 5.1) is based on comparing models that differ in size (32B to 1000B), architecture, and training methodology. The causal attribution to 'reasoning ability' is not supported by the experimental design."
     }
   ],
   "cited_papers": [
     {
       "title": "Malicious LLM-based conversational AI makes users reveal personal information",
       "authors": ["Xiao Zhan", "Juan Carlos Carrillo", "William Seymour", "Jose Such"],
       "year": 2025,
       "relevance": "Direct predecessor demonstrating system-prompt-level manipulation of CAIs to extract PII, with 90%+ user disclosure rates — the white-box attack that VORTEXPIA extends to black-box settings."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Foundational framework for formalizing and evaluating prompt injection attacks and defenses, which VORTEXPIA builds upon for its threat model definition."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2025,
       "relevance": "Proposes a structural defense against prompt injection attacks, relevant to assessing whether VORTEXPIA can bypass structured defenses."
     },
     {
       "title": "Can indirect prompt injection attacks be detected and removed?",
       "authors": ["Yulin Chen", "Haoran Li", "Yuan Sui", "Yufei He", "Yue Liu", "Yangqiu Song", "Bryan Hooi"],
       "year": 2025,
       "relevance": "Studies detection and removal of indirect prompt injections; VORTEXPIA's detection evaluation builds on this work's detection model approach."
     },
     {
       "title": "Adaptive attacks break defenses against indirect prompt injection attacks on LLM agents",
       "authors": ["Qiusi Zhan", "Richard Fang", "Henil Shalin Panchal", "Daniel Kang"],
       "year": 2025,
       "arxiv_id": "2310.16620",
       "relevance": "Demonstrates that adaptive attacks can break PIA defenses on LLM agents, providing the instructional prevention defense used in VORTEXPIA's evaluation."
     },
     {
       "title": "Breaking agents: Compromising autonomous LLM agents through malfunction amplification",
       "authors": ["Boyang Zhang", "Yicong Tan", "Yun Shen", "Ahmed Salem", "Michael Backes", "Savvas Zannettou", "Yang Zhang"],
       "year": 2024,
       "arxiv_id": "2407.20859",
       "relevance": "Shows PIAs remain effective in multi-agent systems, extending the threat model relevant to VORTEXPIA's evaluation on multi-agent applications like DeepSearch."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2025,
       "relevance": "Provides benchmarks and defense evaluation frameworks for indirect prompt injection, directly relevant to the survey's coverage of LLM security evaluation methodology."
     },
     {
       "title": "PIG: Privacy jailbreak attack on LLMs via gradient-based iterative in-context optimization",
       "authors": ["Yidan Wang", "Yanan Cao", "Yubing Ren", "Fang Fang", "Zheng Lin", "Binxing Fang"],
       "year": 2025,
       "relevance": "Privacy-specific jailbreak attack using gradient-based optimization, representing a complementary approach to VORTEXPIA's prompt injection method for LLM privacy extraction."
     },
     {
       "title": "Searching for privacy risks in LLM agents via simulation",
       "authors": ["Yanzhe Zhang", "Diyi Yang"],
       "year": 2025,
       "arxiv_id": "2508.10880",
       "relevance": "Explores privacy risks from malicious LLM agents proactively extracting PII through multi-turn interactions, closely related to VORTEXPIA's privacy extraction threat model."
     },
     {
       "title": "The hidden risks of large reasoning models: A safety assessment of R1",
       "authors": ["Kaiwen Zhou", "Chengzhi Liu", "Xuandong Zhao"],
       "year": 2025,
       "relevance": "Safety assessment of reasoning LLMs finding that reasoning tokens show stronger threats than final answers — VORTEXPIA's finding contradicts this for privacy extraction attacks."
     },
     {
       "title": "Optimization-based prompt injection attack to LLM-as-a-judge",
       "authors": ["Jiawen Shi", "Zenghui Yuan", "Yinuo Liu", "Yue Huang", "Pan Zhou", "Lichao Sun", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Demonstrates prompt injection attacks targeting LLM-as-a-Judge evaluation, relevant to assessing the reliability of VORTEXPIA's own LLM-judged evaluation methodology."
     },
     {
       "title": "Fun-tuning: Characterizing the vulnerability of proprietary LLMs to optimization-based prompt injection attacks via the fine-tuning interface",
       "authors": ["Andrey Labunets", "Nishit V. Pandya", "Ashish Hooda", "Xiaohan Fu", "Earlence Fernandes"],
       "year": 2025,
       "relevance": "Characterizes prompt injection vulnerabilities through the fine-tuning interface of proprietary LLMs, extending the attack surface relevant to LLM security research."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security practitioners could use the attack template to test their LLM applications for privacy extraction vulnerabilities, but no tool or code is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Extends known prompt injection threats to privacy extraction under black-box settings; novel combination but not a fundamental surprise given prior work on PIA."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a practical attack achieving 90%+ ASR that causes deployed LLM applications to proactively solicit users' PII including bank PINs and passwords."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy or adversarial framing beyond standard security research; the finding that reasoning models are more vulnerable adds mild tension."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released; the attack prompts are provided in the appendix but cannot be trivially reproduced without the full experimental setup."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Tests well-known models (DeepSeek-R1, Qwen) but from an unknown research group at Beijing Institute of Technology; no major lab or product branding."
     }
   }
 }