ai-research-survey

scan.json (25585B)

---

 {
   "paper": {
     "title": "CachePrune: Neural-Based Attribution Defense Against Indirect Prompt Injection Attacks",
     "authors": [
       "Rui Wang",
       "Junda Wu",
       "Yu Xia",
       "Tong Yu",
       "Ruiyi Zhang",
       "Ryan Rossi",
       "Subrata Mitra",
       "Lina Yao",
       "Julian McAuley"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.21228"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided in the paper. No mention of code release or availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: SQuAD (Rajpurkar, 2016), HotpotQA (Yang et al., 2018), and WildChat (Zhao et al., 2024), with splits from Abdelnabi et al. (2024). These are all publicly accessible benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section listing library versions is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "CachePrune results are reported with ± notation across 3 trials (e.g., '7.44 ± 0.22' for ASR on SQuAD with LLama3-8B in Table 1). However, baselines do not include error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (t-tests, Mann-Whitney, etc.) are used to validate claims that CachePrune outperforms baselines. Comparisons are based solely on point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context. For example, ASR drops from 27.86% (Vanilla) to 7.44% for CachePrune on SQuAD with LLama3-8B (Table 1), giving clear magnitude context for the improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses N=8 samples for neural attribution and tests on 400 prompts per dataset, but provides no justification for why these sample sizes are adequate. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results for CachePrune are reported as averages with standard deviation across 3 trials (e.g., '7.44 ± 0.22' in Table 1). Table 6 also shows variance across different N values."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: Vanilla, Delimiting, Datamarking, Sandwich, and Encode_Base64, as described in Appendix A and Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines come from recent work (Wu et al., 2023; Hines et al., 2024; Schulhoff et al., 2023). The paper explicitly justifies not comparing to training-based or test-time-workflow approaches due to computational cost differences (Appendix A)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 ablates the selective thresholding subset Phi (w/ vs w/o Phi). Table 3 ablates the number of trigger tokens k. Table 4 ablates the masking parameter alpha. Table 6 ablates the number of attribution samples N."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ASR, F1 (clean), F1 (attack), and GPT-Score for WildChat. Appendix H additionally reports ROUGE-1/2/L and BERTScore (Tables 13, 14)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of defense quality or response quality is performed. All evaluation is automated via F1, ROUGE, BERTScore, and LLM-as-judge (GPT-Score)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we randomly select 8 samples from a pool of 400 prompts that are not overlapped with the testing data' (Section 4.1), indicating separation between attribution samples and test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (LLama3-8B, Mistral-7B, Phi-3.5-mini), per dataset (SQuAD, HotpotQA, WildChat), and per attack type (Table 2, Figure 5). Figure 6 shows per-layer neuron distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the adaptive attack is effective (LLama3-8B near 100% ASR on baselines in Table 7), and notes HotpotQA is harder than SQuAD. The Limitations section discusses the lack of comparison with training-based approaches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows that using more tokens (k=4, Lattr_full) for attribution actually degrades performance compared to k=1. Table 5 shows that pruning without Phi at p=5% causes severe quality degradation (F1 drops to 6.48). Delimiting and Sandwich sometimes increase ASR over Vanilla (Table 1, HotpotQA)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CachePrune 'significantly reduce attack success rates while not compromising the response quality.' Table 1 supports this across three models and three datasets, showing ASR reductions with maintained F1/GPT-Score."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that pruning task-triggering neurons causes reduction in ASR. The ablation studies (Tables 3-6) provide controlled single-variable manipulations of k, alpha, N, and Phi, adequately supporting these causal claims through systematic intervention design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims defense against 'Indirect Prompt Injection Attacks' generally, but experiments are limited to three relatively small open-source models (3.8B-8B parameters) on question-answering and dialogue summarization tasks. No commercial models or diverse task types are tested. The paper does not bound these generalization limits."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why CachePrune works. For example, it does not consider whether the ASR reduction could be partly due to general response degradation rather than specifically addressing instruction-following neurons, or whether the improvements are an artifact of the specific attack injection patterns used."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'LLama3-8B', 'Mistral-7B-Instruct-V3.0', and 'Phi-3.5-mini-instruct (3.8B)' but does not provide exact model snapshot dates or version hashes. 'Mistral-7B-Instruct-V3.0' is the most specific, but LLama3-8B and Phi-3.5-mini-instruct lack precise version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I provides the full prompt templates used for the GPT-4 judge evaluation, including the dialogue summarization quality prompt and the attack detection prompt. The attack injection formats are also described in Appendix F."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: N=8 attribution samples, p=0.5% pruning ratio, k=1 trigger tokens, alpha=1.0 masking parameter (Section 4.1). The GPT judge uses 'gpt-4-1106-preview' (Appendix I)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "CachePrune is not an agentic scaffolding system. It is a single-pass neural attribution and pruning mechanism applied to the KV cache, with no multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using splits from Abdelnabi et al. (2024), the injection method (random injection into beginning, middle, and end of context), the WildChat modification ('You should primarily focus on this question'), and the sampling procedure (8 from 400 non-overlapping prompts)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and discusses the lack of comparison with training-based defense approaches and the computational focus of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section is brief (one paragraph) and only mentions not exploring training-based defenses. It does not discuss specific threats such as the small number of models tested, the specific attack patterns used, or potential overfitting of the attribution mask to the few attribution samples."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the tested models, datasets, or attack types. The title and framing suggest general applicability without acknowledging the limited scope of evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, attribution masks, per-example results) is available for independent verification. Only aggregate results are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using existing public datasets (SQuAD, HotpotQA, WildChat) with splits from Abdelnabi et al. (2024). The injection procedure and sample selection (8 from 400 non-overlapping prompts, 3 trials) are described in Section 4.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from dataset selection to injection to evaluation is documented: datasets from Abdelnabi et al. (2024), random injection of attack instructions, sampling 8 from 400 non-overlapping prompts, evaluation with F1/ROUGE/BERTScore/GPT-Score. Evaluation code snippets are provided in Listings 1-3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No mention of grants or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Adobe Research, UC San Diego, UNSW, and CSIRO's Data61. The paper header shows institutional affiliations for all authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are from Adobe Research, which has a commercial interest in LLM security. No funding disclosure is made, so independence cannot be assessed. The paper does not evaluate Adobe products specifically, but the affiliation is relevant."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper proposes a defense mechanism rather than evaluating model knowledge on benchmarks. The evaluation measures attack success rates and response quality, not model knowledge. Contamination of training data with benchmark questions is not relevant to the defense evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense mechanism, not model capability on benchmarks. Train/test contamination of the LLMs with SQuAD/HotpotQA questions is not relevant to whether the defense reduces attack success rates."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above: the paper evaluates a defense mechanism's effectiveness at reducing attack success rates, not model knowledge or capability on the benchmark tasks themselves."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CachePrune is 'computationally lightweight' and does not require 'extra test-time computation per response or LLM calls' but does not report actual inference cost, latency, wall-clock time, or tokens consumed for the attribution step or pruning."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, hardware used, training time for attribution) is reported. The paper mentions using only N=8 samples and a single forward pass but does not quantify the actual compute requirements."
       }
     }
   },
   "claims": [
     {
       "claim": "CachePrune significantly reduces attack success rates while not compromising response quality.",
       "evidence": "Table 1 shows ASR reduction from 27.86% to 7.44% on SQuAD with LLama3-8B, from 69.01% to 15.23% on HotpotQA, and from 14.50% to 2.00% on WildChat, while maintaining comparable F1 scores and GPT-Scores to Vanilla.",
       "supported": "strong"
     },
     {
       "claim": "Neural attribution can be effectively performed with only N=8 samples.",
       "evidence": "Table 6 shows that N=4, 8, and 12 yield similar performance across all three models, with N=8 being the default. The ASR difference between N=4 and N=8 is small (e.g., 8.15 vs 7.44 for LLama3-8B).",
       "supported": "strong"
     },
     {
       "claim": "Only the first k=1 token is sufficient for effective neural attribution (the triggering effect).",
       "evidence": "Table 3 and Figure 3 show that k=1 achieves lowest ASR (7.44%) compared to k=2 (5.57% lower ASR but worse F1) and k=4 or Lattr_full (higher ASR). Figure 3 shows clean/poisoned responses can be triggered by 1-2 tokens.",
       "supported": "moderate"
     },
     {
       "claim": "The learnt pruning mask is transferable between different attack types.",
       "evidence": "Table 2 shows masks learned from code-based injection can defend text-based injection and vice versa. Figure 5 shows cross-attack transferability across 4 different attack instructions, with darkest colors not necessarily on the diagonal.",
       "supported": "moderate"
     },
     {
       "claim": "CachePrune is robust to adaptive attacks (GCG-based).",
       "evidence": "Tables 7-12 show that CachePrune maintains low ASR under adaptive attacks (e.g., 7.71% on LLama3-8B vs near 100% for baselines). However, the adaptive attack is less effective on Mistral and Phi models even without CachePrune.",
       "supported": "moderate"
     },
     {
       "claim": "The selective thresholding subset Phi preserves response quality during pruning.",
       "evidence": "Table 5 shows that at p=5%, pruning without Phi drops F1 (clean) from 27.48 to 6.48, while pruning with Phi maintains F1 at 27.48. The effect is less pronounced at lower pruning ratios.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CachePrune defends against indirect prompt injection attacks by identifying and pruning task-triggering neurons from the KV cache using a preferential attribution loss related to DPO. The approach reduces attack success rates by several times compared to prompt-engineering baselines across three models (LLama3-8B, Mistral-7B, Phi-3.5-mini) and three datasets (SQuAD, HotpotQA, WildChat), while preserving response quality. The paper demonstrates that effective attribution requires only 8 samples and 1 trigger token, and that learned pruning masks transfer across different attack types. The approach shows robustness to GCG-based adaptive attacks, especially on LLama3-8B where baselines fail completely.",
   "red_flags": [
     {
       "flag": "No code release",
       "detail": "Despite proposing a novel defense mechanism with specific implementation details, no source code is provided. This makes independent reproduction and verification difficult."
     },
     {
       "flag": "No significance testing",
       "detail": "Comparative claims between CachePrune and baselines are made without statistical significance tests. Baselines are reported as single-run point estimates while CachePrune reports means over 3 trials, making direct comparison asymmetric."
     },
     {
       "flag": "Limited model scale",
       "detail": "All experiments use relatively small open-source models (3.8B-8B parameters). No experiments on larger models or commercial APIs are conducted, yet the paper's framing implies general applicability."
     },
     {
       "flag": "No compute cost reported",
       "detail": "Despite claiming the approach is 'computationally lightweight', no actual compute costs, GPU hours, or wall-clock times are reported for the attribution step."
     },
     {
       "flag": "Corporate affiliation without disclosure",
       "detail": "Multiple authors are from Adobe Research. No funding disclosure, competing interests statement, or acknowledgments section is present."
     },
     {
       "flag": "Minimal limitations section",
       "detail": "The Limitations section (Section 6) is a single short paragraph that only mentions not exploring training-based approaches, without discussing specific threats to the validity of the results."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLMs, directly relevant to LLM security research."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Training-based defense against prompt injection using structured queries, relevant baseline for LLM security methods."
     },
     {
       "title": "Aligning LLMs to be robust against prompt injection",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "Saeed Mahloujifar", "Kamalika Chaudhuri", "Chuan Guo"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "Alignment-based approach to defending against prompt injection, relevant to LLM safety and alignment research."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati", "Yonatan Zunger", "Emre Kiciman"],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Prompt-engineering defense against indirect prompt injection, directly comparable baseline method."
     },
     {
       "title": "FATH: Authentication-based test-time defense against indirect prompt injection attacks",
       "authors": ["Jiongxiao Wang", "Fangzhou Wu", "Wendi Li", "Jinsheng Pan", "Edward Suh", "Z Morley Mao", "Muhao Chen", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2410.21492",
       "relevance": "Test-time defense approach against prompt injection using authentication, relevant to the defense methodology landscape."
     },
     {
       "title": "The Task Shield: Enforcing task alignment to defend against indirect prompt injection in LLM agents",
       "authors": ["Feiran Jia", "Tong Wu", "Xin Qin", "Anna Squicciarini"],
       "year": 2024,
       "arxiv_id": "2412.16682",
       "relevance": "Defense mechanism for LLM agents against indirect prompt injection through task alignment enforcement."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen", "Zeming Wei", "Elizabeth Sun", "Basel Alomair", "David Wagner"],
       "year": 2024,
       "relevance": "Task-specific finetuning approach to prompt injection defense, relevant to training-based defense methods."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "arxiv_id": "2312.14197",
       "relevance": "Benchmark for indirect prompt injection attacks and defenses, relevant to evaluation methodology in LLM security."
     },
     {
       "title": "Are you still on track!? Catching LLM task drift with activations",
       "authors": ["Sahar Abdelnabi", "Aideen Fay", "Giovanni Cherubin", "Ahmed Salem", "Mario Fritz", "Andrew Paverd"],
       "year": 2024,
       "arxiv_id": "2406.00799",
       "relevance": "Activation-based detection of prompt injection task drift, the source of datasets used in this paper's evaluation."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2024,
       "relevance": "Foundation for the preferential attribution loss used in CachePrune, relevant to LLM alignment techniques."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack used for adaptive attack evaluation in this paper, fundamental to LLM adversarial robustness research."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang", "Ning Zhang", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Automatic prompt injection attack method used as the adaptive attack baseline in this paper."
     }
   ]
 }