ai-research-survey

scan.json (32819B)

---

 {
   "paper": {
     "title": "SecInfer: Preventing Prompt Injection via Inference-time Scaling",
     "authors": [
       "Yupei Liu",
       "Yanting Wang",
       "Yuqi Jia",
       "Jinyuan Jia",
       "Neil Zhenqiang Gong"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2509.24967",
     "doi": "10.48550/arXiv.2509.24967"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SecInfer defends against prompt injection by sampling multiple LLM responses via diverse chain-of-thought system prompts and selecting the response most aligned with the target task using semantic clustering and an LLM-as-a-judge. Across 4 LLMs, 6 target tasks, 7 existing attacks, and 6 adaptive attacks, SecInfer reduces attack success rate to near zero while preserving task utility comparable to undefended baselines. The method outperforms both existing prevention-based defenses and prior inference-time scaling approaches, though it is less effective when the injected and target tasks are of the same type.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper. The paper references open-source implementations of attacks (e.g., nano-GCG) but does not release its own SecInfer implementation."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available standard benchmarks: AG News, MMLU, GPQA, NarrativeQA, SQuAD_v2, SimpleQA, Open-Prompt-Injection, InjecAgent, and AgentDojo. The paper did not collect proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using NVIDIA GH200 GPUs and specific LLMs but provides no requirements.txt, Dockerfile, or detailed environment specification listing library versions sufficient to recreate the experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithms are described pseudocode-level but a researcher would need to implement the full pipeline from scratch."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-8 are reported as point estimates only. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SecInfer 'outperforms' baselines and achieves 'higher effectiveness' throughout, but no statistical significance tests (t-tests, bootstrap, etc.) are reported. All comparisons are based on raw numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context enabling effect size assessment. For example, Table 1 shows ASR drops from 0.64 to 0.00 under Combined Attack on AG News, and utility preservation is shown as 87% vs 86% for GPT-4.1. Absolute differences are computable from all tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 100 contaminated samples per target-injected task pair but provides no justification for why 100 is sufficient and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, interquartile ranges, or variance measures are reported across any results. It is unclear whether results are from single runs or averaged over multiple runs, and no spread measure accompanies any reported number."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against 8 prevention-based defenses (Delimiters, Sandwich, Instructional, Paraphrasing, Retokenization, DS+PL, PromptArmor, MetaSecAlign). Table 3 compares against 6 inference-time scaling methods (ICL-2, ICL-4, CoT, IR, BoN, SC)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent methods: MetaSecAlign (Chen et al., 2025c), PromptArmor (Shi et al., 2025c), DataSentinel+PromptLocate (Liu et al., 2025; Jia et al., 2026). These represent the state of the art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 presents detailed ablations: Step I sampling variants (Figure 3), impact of N (Figure 4), Step II aggregation variants (Figure 5), and choice of judge LLM (Table 7). Each ablation isolates specific components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used throughout: U (utility under no attack), UA (utility under attack), and ASR (attack success rate). These capture different aspects of defense effectiveness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. No human evaluation of system outputs is performed. For open-domain tasks, automated metrics (ROUGE-1, Pass@1) are used exclusively."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test splits are used. Appendix E notes that ICL demonstration examples are guaranteed not to be in the evaluation set. SecInfer's system prompts are fixed (not tuned on test data)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by target task (6 datasets), attack type (7 existing + 6 adaptive), and backend LLM (4 models) in Tables 1, 2, 3, and 8. InjecAgent results are broken down by attack variant and harm type (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides a detailed discussion of when SecInfer fails: same-type target/injected tasks with concrete examples (sentiment analysis). Table 12 quantifies the failure (ASR 0.90→0.79 for SST2). Adaptive Attack V is shown to degrade UA substantially."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 shows SecInfer achieves only marginal ASR reduction for same-type tasks (0.90→0.79). Table 8 shows adaptive attacks V and VI reduce UA significantly. Section 6 openly discusses these limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SecInfer 'effectively mitigates both existing and adaptive prompt injection attacks, outperforming state-of-the-art defenses as well as existing inference-time scaling approaches.' Tables 1-3 and 8 support all three claims with near-zero ASR across conditions and superior UA compared to baselines."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies in Section 5.4: each Step I variant and Step II variant is tested in isolation (Figures 3, 5). The ablation design uses single-variable manipulation, adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across 4 LLMs, 6 target tasks, 8 injected tasks, 13 attacks, and 2 agent benchmarks, which is extensive. Section 6 explicitly bounds generalization by identifying same-type task limitations. Claims are generally well-matched to the breadth of testing."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 conducts a memorization test to rule out the alternative explanation that the judge LLM memorized benchmark answers rather than genuinely evaluating responses. Over 94% of samples had similarity ratios below 0.6, ruling out this confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. ASR directly measures whether the model produces the attacker-desired response. U and UA directly measure task performance using standard metrics (accuracy, ROUGE-1, Pass@1). No proxy gap exists between measurements and framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-weight models are specified precisely (LLaMA3.1-8B-Instruct, Qwen3-8B). However, closed-source models are identified only by marketing names: 'GPT-4o' and 'GPT-4.1' without snapshot dates. Appendix C mentions Azure API version '2024-12-01-preview' but this is the API version, not the model snapshot."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides all 5 system prompts verbatim. Figure 7 shows the full judge instruction. Appendix G provides agent-specific system prompts. Appendix H provides adaptive attack prompts. The actual prompt text is reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C specifies temperature, max output tokens, and top-k for each model: LLaMA (temp=0.1, max=150, top-k=20), Qwen3 (temp=0.6, max=32768, top-k=20), GPT-4o/4.1 (temp=0.7, max=500, top-k=20)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "SecInfer's two-step pipeline is described in full algorithmic detail (Algorithms 1-3), including system-prompt-guided sampling with CoT prompts and target-task-guided aggregation using semantic clustering (Agglomerative Clustering, all-MiniLM-L6-v2 embeddings) and LLM-as-a-judge."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes how contaminated data is constructed for each attack type. Appendix D details attack implementations. Appendix B describes the mapping function for closed-domain tasks. Dataset sources and how injected prompts are integrated are clearly specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion and Limitations' is a dedicated section with substantive discussion of two specific limitations: same-type task failure and a memorization test."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: (1) SecInfer fails when injected and target tasks are the same type, with quantitative evidence from Table 12; (2) the memorization concern is tested with string similarity analysis showing >94% of samples below 0.6 similarity. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what SecInfer does NOT handle: same-type target/injected tasks, where 'prompt injection in this scenario reduces to traditional adversarial examples.' Section 5.5 notes Adaptive Attack V defeats utility preservation when the entire data is modified. Footnote 1 in Section 4.1 also flags the same-type limitation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual model outputs, per-sample results, intermediate clustering outputs) is released. Only aggregated metrics in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes all datasets used, their sources, and how contaminated samples are generated (100 per target-injected task pair). Attack construction details are in Section 2.2 and Appendix D."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments use automated evaluation on standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from data preparation (selecting target/injected tasks, constructing contaminated samples) through SecInfer's two steps (sampling N responses, clustering, judge selection) to metric computation is documented in Sections 4-5 and Appendices B-F."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Penn State University (Liu, Wang, Jia J.) and Duke University (Jia Y., Gong). Authors are academic researchers not affiliated with the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, independence cannot be assessed. The absence of funding disclosure prevents evaluation of this criterion."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four LLMs used (LLaMA3.1-8B-Instruct, Qwen3-8B, GPT-4o, GPT-4.1). This is relevant because utility metrics (U) are measured on standard benchmarks that may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between LLM training data and the benchmark test sets (AG News, MMLU, GPQA, etc.) used for utility evaluation. The memorization test in Section 6 only addresses the judge LLM, not the backend LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks used (AG News from 2020, MMLU from 2021, SQuAD_v2 from 2018) predate all models' training periods and are widely available online, creating high contamination risk for utility measurements. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports average inference time per sample for all methods. SecInfer takes 0.612 seconds. The paper also notes parallelization across 5 GH200 GPUs reduces this to 0.221s (Section 5.2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-sample inference time is reported (Table 4) but the total computational budget for all experiments (total GPU hours, total API spend for GPT-4o/4.1 experiments) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seed sensitivity analysis is reported. The paper does not mention running experiments with multiple seeds or analyzing seed-dependent variability in results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of contaminated samples (100 per target-injected task pair) is stated, but the number of independent experimental runs per configuration is not specified. It is unclear if results are from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Default hyperparameters (N=5, specific temperatures, top-k values) are used without describing how they were selected. The ablation study explores alternatives post-hoc but no systematic search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 provides ablation studies justifying the default configuration: Figure 3 validates the combined sampling strategy, Figure 4 shows N=5 is sufficient, Figure 5 validates LLM-as-a-judge, and Table 7 evaluates judge LLM choices. These are performed on SQuAD_v2/SST2, separate from the main evaluation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement baselines following existing works (Appendix E) but do not acknowledge or discuss the potential bias of implementing and evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports inference time for all methods, enabling compute-matched comparisons. Figure 4 explicitly shows performance (U, UA, ASR) as a function of N alongside computation time. SecInfer's overhead (0.612s) is compared to BoN (0.719s) and SC (0.622s)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmark tasks (AG News classification, MMLU, etc.) are valid proxies for real-world prompt injection scenarios like AI Overviews or Review Highlights, despite motivating the work with these applications."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing backend LLMs in Table 1, all use the same SecInfer method. When comparing defenses in Tables 2-3, the same backend LLM (LLaMA3.1-8B-Instruct) is used throughout. The experimental design controls for scaffold confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmark datasets (AG News 2020, MMLU 2021, SQuAD_v2 2018) were created years before the models' training periods, meaning models may have seen solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage in the evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential non-independence between training and evaluation data for the LLMs used."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 applies a memorization test following Staab et al. (2024) methodology on the judge LLM, measuring string similarity between outputs and ground-truth answers. Over 94% of samples had similarity ratios below 0.6, providing concrete leakage detection evidence for the judge LLM."
       }
     }
   },
   "claims": [
     {
       "claim": "SecInfer reduces ASR to nearly zero across all tested attacks, LLMs, and datasets.",
       "evidence": "Table 1(b) shows ASR of 0.00 across nearly all combinations of 4 LLMs, 6 target tasks, and 7 attacks (with minor exceptions of 0.01-0.02 for GCG attack on some tasks).",
       "supported": "strong"
     },
     {
       "claim": "SecInfer preserves task utility (U) comparable to the undefended baseline.",
       "evidence": "Table 1 shows U with SecInfer is within 1-2 percentage points of no-defense U for most LLM-task combinations (e.g., 0.87 vs 0.86 for GPT-4.1 on AG News).",
       "supported": "strong"
     },
     {
       "claim": "SecInfer outperforms all existing prevention-based defenses.",
       "evidence": "Table 2 shows SecInfer achieves 0.00 ASR under Combined Attack while most baselines have ASR 0.37-0.86. MetaSecAlign matches on CA but fails under GCG (ASR 0.25-0.63). DS+PL also shows non-negligible ASR under GCG for some tasks.",
       "supported": "strong"
     },
     {
       "claim": "SecInfer outperforms prior inference-time scaling methods against prompt injection.",
       "evidence": "Table 3 shows all prior methods (ICL, CoT, IR, BoN, SC) have substantially higher ASR (0.06-0.45) and lower UA compared to SecInfer (0.00 ASR) under Combined Attack.",
       "supported": "strong"
     },
     {
       "claim": "SecInfer is robust against six types of adaptive attacks specifically targeting its components.",
       "evidence": "Table 8(b) shows ASR remains near zero (0.00-0.06) across all six adaptive attacks when SecInfer is deployed, compared to high ASR without defense (Table 8(a)). However, UA degrades notably under Adaptive Attacks V and VI.",
       "supported": "moderate"
     },
     {
       "claim": "SecInfer is effective in LLM agent settings (InjecAgent and AgentDojo).",
       "evidence": "Table 5 shows ASR drops from 0.37-0.68 to 0.00-0.02 on InjecAgent. Table 6 shows ASR drops from 0.21 to 0.02 on AgentDojo while utility is maintained (0.40→0.39 U).",
       "supported": "strong"
     },
     {
       "claim": "SecInfer is less effective when the injected and target tasks are of the same type.",
       "evidence": "Table 12 shows ASR only drops from 0.90 to 0.79 for SST2 and from 0.38 to 0.28 for Gigaword when target and injected tasks match. Section 6 explains this reduces to traditional adversarial examples.",
       "supported": "strong"
     },
     {
       "claim": "The judge LLM's effectiveness does not stem from memorization of target task datasets.",
       "evidence": "Section 6 memorization test shows over 94% of samples have string similarity ratios below 0.6 between judge outputs and ground-truth answers, following Staab et al. (2024) methodology.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across all tables (1-8, 12) are reported as single point estimates with no error bars, confidence intervals, standard deviations, or variance measures. Given that SecInfer involves stochastic sampling (temperature sampling, random system prompt selection), results could vary meaningfully across runs, making it impossible to assess the reliability of reported differences."
     },
     {
       "flag": "No code release",
       "detail": "Despite the algorithmic complexity of SecInfer (two-step pipeline with semantic clustering, multiple system prompts, judge LLM integration), no source code is released, significantly limiting reproducibility and independent verification of results."
     },
     {
       "flag": "Benchmark contamination risk for utility metrics",
       "detail": "Several benchmarks used for utility evaluation (AG News 2020, MMLU 2021, SQuAD_v2 2018) predate all model training periods. High U scores on these benchmarks may partly reflect training data memorization rather than genuine task competence. The memorization test in Section 6 only addresses the judge LLM, not the backend LLMs."
     },
     {
       "flag": "Construct validity gap between evaluation and motivation",
       "detail": "The paper motivates SecInfer with real-world applications (AI Overviews, Review Highlights) but evaluates exclusively on academic benchmarks (AG News classification, MMLU multiple-choice). No evaluation is conducted on the motivating use cases, leaving a gap between claimed applicability and demonstrated effectiveness."
     }
   ],
   "cited_papers": [
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Foundational benchmark (Open-Prompt-Injection) for evaluating prompt injection attacks and defenses, directly used in this paper's experimental setup."
     },
     {
       "title": "Meta secalign: A secure foundation llm against prompt injection attacks",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "David Wagner", "Chuan Guo"],
       "year": 2025,
       "relevance": "State-of-the-art LLM fine-tuning defense against prompt injection, used as a key baseline showing SecInfer's advantage against optimization-based attacks."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Early formalization of indirect prompt injection attacks against LLM-integrated applications, establishing the threat model this paper defends against."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed Chi"],
       "year": 2023,
       "relevance": "Key inference-time scaling baseline (self-consistency via majority voting) that SecInfer extends and outperforms for prompt injection defense."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG optimization attack used in this paper's evaluation as an optimization-based prompt injection method."
     },
     {
       "title": "Trading inference-time compute for adversarial robustness",
       "authors": ["Wojciech Zaremba", "Evgenia Nitishinskaya", "Boaz Barak"],
       "year": 2025,
       "arxiv_id": "2501.18841",
       "relevance": "Concurrent work exploring inference-time scaling for adversarial robustness including prompt injection; provides the COMPROMISE injected task used in this paper."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "relevance": "Agent prompt injection benchmark used to evaluate SecInfer in LLM agent settings, demonstrating ASR reduction from 0.37-0.68 to 0.00-0.02."
     },
     {
       "title": "Agentdojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic"],
       "year": 2024,
       "relevance": "Agent prompt injection benchmark used to evaluate SecInfer, showing ASR reduction from 0.21 to 0.02 while preserving agent utility."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": ["Yupei Liu", "Yuqi Jia", "Jinyuan Jia", "Dawn Song", "Neil Zhenqiang Gong"],
       "year": 2025,
       "relevance": "State-of-the-art prompt injection detection method used as baseline (DS+PL); shows complementary detection-based approach to SecInfer's prevention-based defense."
     },
     {
       "title": "Neural exec: Learning (and learning from) execution triggers for prompt injection attacks",
       "authors": ["Dario Pasquini", "Martin Strohmeier", "Carmela Troncoso"],
       "year": 2024,
       "arxiv_id": "2403.03792",
       "relevance": "Optimization-based prompt injection attack (NeuralExec) used as a strong attack baseline in the evaluation."
     },
     {
       "title": "A critical evaluation of defenses against prompt injection attacks",
       "authors": ["Yuqi Jia", "Zedian Shao", "Yupei Liu", "Jinyuan Jia", "Dawn Song", "Neil Zhenqiang Gong"],
       "year": 2025,
       "arxiv_id": "2505.18333",
       "relevance": "Comprehensive evaluation of prompt injection defenses showing limitations of fine-tuning approaches against optimization-based attacks, motivating SecInfer's inference-time approach."
     },
     {
       "title": "Optimization-based prompt injection attack to LLM-as-a-judge",
       "authors": ["Jiawen Shi", "Zenghui Yuan", "Yinuo Liu"],
       "year": 2024,
       "relevance": "JudgeDeceiver attack method used in Adaptive Attack IV to optimize injected prompts that mislead SecInfer's judge LLM component."
     },
     {
       "title": "PromptArmor: Simple yet effective prompt injection defenses",
       "authors": ["Tianneng Shi", "Kaijie Zhu", "Zhun Wang"],
       "year": 2025,
       "arxiv_id": "2507.15219",
       "relevance": "Reasoning-LLM-based prompt injection defense used as a baseline, showing limited effectiveness compared to SecInfer."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "SecInfer is a deployable defense for production LLM applications with concrete implementation details, though it requires additional inference compute (~5x overhead)."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Extends the existing inference-time scaling paradigm to security rather than challenging conventional wisdom; the finding that vanilla scaling methods fail for prompt injection is moderately surprising."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses prompt injection, ranked as the top LLM security threat by OWASP, and demonstrates that existing defenses including fine-tuning are defeated by optimization-based attacks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward technical contribution with no controversy or conflict; presents a new defense rather than critiquing existing approaches or organizations."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code release, no demo, no pip-installable tool. Implementation would need to be done from scratch based on the paper's descriptions."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Penn State and Duke, respected universities but not top AI labs; evaluates GPT-4o/GPT-4.1 which adds some brand recognition."
     }
   }
 }