ai-research-survey

scan.json (26747B)

---

 {
   "paper": {
     "title": "Backdoored Retrievers for Prompt Injection Attacks on Retrieval Augmented Generation of Large Language Models",
     "authors": ["Cody Clop", "Yannick Teglia"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.14479"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. No mention of code release in the paper or appendix."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets from the BEIR benchmark: Natural Questions, MS-MARCO, HotpotQA, and NFCorpus. These are all standard public benchmarks that the authors did not modify (aside from inserting poisoned documents, which are described in the methodology)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or environment setup section is provided. The paper mentions model names but not library versions, Python version, or hardware details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology section describes the experimental setup at a conceptual level but does not provide commands or configurations needed to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., ASR values like 0.91, 0.27, 1.0). No confidence intervals, error bars, or ± notation appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Llama-3 exhibited the highest susceptibility', 'Vicuna demonstrated the highest resistance', backdoor attacks achieve 'even higher success rates') but uses no statistical significance tests. Comparisons are made by directly comparing ASR numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ASR (Attack Success Rate) values with sufficient context: e.g., 'Llama-3 exhibited the highest susceptibility to prompt injection, achieving an ASR of up to 0.91' and 'Vicuna demonstrated the highest resistance to prompt injection attacks, with a maximum ASR of 0.27.' Tables 3 and 4 provide detailed ASR values across conditions. The reader can assess magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '100 queries per dataset' for LLM vulnerability experiments (Section 3.1, Figure 4 caption) but provides no justification for why 100 was chosen, no power analysis, and no discussion of whether this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across any experimental runs. All results appear to be single-run numbers. No mention of multiple runs with different seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares corpus poisoning against their novel backdoor attack approach. Table 2 compares pretrained, fine-tuned, and backdoored retrievers. Table 4 compares corpus poisoning vs. backdoor attacks across models and configurations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The corpus poisoning baseline is based on PoisonedRAG [3] from 2024, and the paper references Long et al. [4] (2024) and AgentPoison [30] (2024). These are contemporary works in the same problem space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies multiple factors: injection position (10 positions), directive strength (6 levels), number of retrieved documents (k=1,3,5,10,20), attack type (corpus poisoning vs. backdoor), target domain (Alzheimer's Disease vs. Nutrition), and LLM model. These variations function as ablations testing which components matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only a single metric: Attack Success Rate (ASR). While precision@k is reported for the retriever (Table 2), the attack evaluation itself relies solely on ASR. No other metrics such as attack detectability, output quality degradation, or false positive rates are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The attack success is determined automatically (presence of attacker's link, coupon code, or message in the output). Human evaluation of output quality, naturalness of injected content, or detectability of attacks would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe a separation between development and test sets. The 100 queries per dataset used for evaluation are not described as being separate from any queries used during attack development or tuning of directive strength levels."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by attack type, target domain (Alzheimer's Disease vs. Nutrition), LLM model (Llama-3, Vicuna, Mistral), dataset (NFCorpus, HotpotQA), injection position, directive strength level, and number of retrieved documents k. Tables 3 and 4 provide per-configuration breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: Vicuna's high resistance to attacks, corpus poisoning's lower success for broader domains like nutrition (ASR 0.14-0.46), and how increasing k can dilute the injected prompt effect. Section 4 discusses limitations of the trigger approach."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Vicuna showed strong resistance (max ASR 0.27), corpus poisoning on nutrition domain yielded poor results (ASR 0.14-0.44), and the dilution effect of increasing k on backdoor attacks with Vicuna is visible in Table 4 (ASR dropping from 0.83 to 0.46)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) corpus poisoning can achieve significant ASR through small number of compromised documents -- supported by Tables 3 and 4; (2) backdoor attacks demonstrate even higher success rates -- supported by comparison in Tables 3 and 4; (3) backdoor requires more complex setup (victim must fine-tune) -- discussed in Section 3.2 and 4. All abstract claims are supported by the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about attack mechanisms (e.g., 'injected prompt is more effective when it appears among the first retrieved documents', Section 3.1). These are supported by controlled experiments that systematically vary injection position while holding other factors constant. The experimental design (controlled manipulation of attack parameters) is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim broad vulnerability of 'Retrieval Augmented Generation of Large Language Models' but experiments use only 3 open-source 7-8B parameter models, one embedding model (GTE-large-en-v1.5), and 2-4 datasets. No commercial models (GPT-4, Claude) are tested. The paper does not explicitly bound its claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, it does not consider whether the high ASR might be specific to the safety training of the tested models, whether different system prompts could mitigate the attacks, or whether the results reflect artifacts of the specific prompt templates used."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Llama-3-8B', 'Vicuna-7B', 'Mistral-7B', and 'GTE-large-en-v1.5' but does not provide specific version snapshots, dates, or checkpoint identifiers. For example, there are multiple Llama-3-8B variants (base, instruct, different releases) and no specific version is identified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full injection prompts for all 6 directive strength levels are provided in Table 1 (Link Insertion), Table 5 (Denial of Service), and Table 6 (Advertising) in the appendix. The system prompt template for the RAG system is not explicitly shown, but the attack prompts themselves are fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported: no temperature, top-p, or max tokens for LLM inference; no learning rate, batch size, or number of epochs for retriever fine-tuning; no details on the contrastive loss training beyond the loss formula."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a standard RAG pipeline (retriever + LLM) without agentic scaffolding such as tool use, retry logic, or multi-step reasoning. No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how queries were selected from the BEIR datasets (100 queries per dataset -- how were they chosen?), how the poisoned documents were generated beyond a brief mention of using ChatGPT, or the exact process for constructing backdoor training datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion section (Section 4) briefly mentions that the study 'centers on attacks where an unsuspecting user inadvertently triggers behaviors' but does not provide substantive limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues like the limited model set, single embedding model, arbitrary directive strength levels, or the gap between lab settings and real-world RAG deployments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the specific models tested, acknowledge that commercial models with better safety training might behave differently, or state that results may not generalize to production RAG systems."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (query results, model outputs, individual ASR measurements) is made available. Only aggregated ASR values in tables and heatmaps are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the data sources: BEIR benchmark datasets (NQ, MS-MARCO, HotpotQA, NFCorpus) with their references and sizes (e.g., NFCorpus has 3,633 documents, HotpotQA has over 5 million). The attack setup describes how poisoned documents are generated and how backdoor training data is constructed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks, so recruitment methods do not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "Key pipeline details are missing: how 100 queries per dataset were selected, how ChatGPT-generated poisoned documents were filtered or validated, the exact construction of backdoor training datasets (number of poisoned vs. legitimate pairs), and how ASR was computed from raw model outputs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present. Both authors are affiliated with Thales DIS (a defense/cybersecurity company) but there is no disclosure of whether this work was funded by Thales or any other entity."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors' affiliations with Thales DIS Cybersecurity Artificial Intelligence Team are clearly listed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The authors work at Thales DIS, a cybersecurity company that has a commercial interest in demonstrating AI security vulnerabilities (which could drive demand for security products/services). Without explicit funding disclosure, independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of a declaration is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attack vectors (prompt injection, corpus poisoning, backdoor attacks) rather than evaluating pre-trained model knowledge on a benchmark. The models are used as targets for attack, not assessed on their pre-existing knowledge. Contamination in the traditional sense is not relevant to the attack success metrics."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above -- the paper evaluates attack effectiveness on RAG systems, not model knowledge. Train/test overlap for the underlying benchmarks is not relevant to whether the LLM follows injected malicious instructions."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above -- benchmark contamination in the traditional sense does not apply to attack success rate measurements. The question is whether the LLM follows injected prompts, not whether it has memorized benchmark answers."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All experiments are computational."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses automated experiments on publicly available models and datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported for any of the LLM or retriever operations, despite the experiments involving thousands of LLM queries across multiple models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, hardware specifications, or total experiment time is reported. The fine-tuning of the retriever and the thousands of LLM inference calls presumably required meaningful compute, but this is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Llama-3 exhibits the highest susceptibility to prompt injection attacks, achieving an ASR of up to 0.91.",
       "evidence": "Section 3.1 and Figure 4 show heatmaps of ASR across injection positions and directive strengths for Llama-3, Vicuna, and Mistral, based on 100 queries per dataset across three BEIR corpora.",
       "supported": "moderate"
     },
     {
       "claim": "Vicuna demonstrates the highest resistance to prompt injection attacks, with a maximum ASR of 0.27.",
       "evidence": "Section 3.1 and Figure 4 show Vicuna's ASR peaking at 0.27 across all conditions tested.",
       "supported": "moderate"
     },
     {
       "claim": "Backdoor attacks on the dense retriever achieve near-perfect retrieval of poisoned documents (ASR up to 1.0) without degrading performance on benign queries.",
       "evidence": "Table 3 shows backdoor ASR@k of 0.97-1.0 across conditions. Table 2 shows that backdoored retriever precision on benign queries is essentially identical to the clean fine-tuned model.",
       "supported": "strong"
     },
     {
       "claim": "Corpus poisoning is effective for narrow domains (Alzheimer's Disease) but yields lower success rates for broader domains (Nutrition).",
       "evidence": "Table 3: AD corpus poisoning ASR@1 is 0.63 (NFCorpus) and 0.17 (HotpotQA), while Nutrition ASR@1 is only 0.14 and 0.06 respectively. Table 4 confirms this pattern in end-to-end results.",
       "supported": "strong"
     },
     {
       "claim": "The effectiveness of injected instructions may be related to their alignment with the information being processed by the LLM.",
       "evidence": "Section 3.3 notes higher end-to-end ASR for link insertion on Alzheimer's domain and advertising on nutrition domain compared to the LLM-only evaluation, suggesting coherence between injected content and retrieved context matters.",
       "supported": "weak"
     },
     {
       "claim": "ASR decreases as the position of the poisoned document moves further down the retrieval list.",
       "evidence": "Section 3.1 describes this trend based on Figure 4, which shows heatmaps across 10 injection positions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RAG systems are vulnerable to prompt injection attacks via both corpus poisoning and backdoor attacks on the dense retriever, with effects varying by LLM model and attack domain. Backdoor attacks on the retriever achieve near-perfect attack success rates (0.97-1.0) while maintaining normal retrieval performance on benign queries, making them difficult to detect. Corpus poisoning is more practical but less reliable, particularly for broad target domains. Among the three tested LLMs, Llama-3-8B was most susceptible (ASR up to 0.91) while Vicuna-7B showed the strongest resistance (max ASR 0.27).",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as single-point ASR values with no confidence intervals, error bars, or variance across runs. With 100 queries per dataset, sampling variance could meaningfully affect the reported numbers, but this is never addressed."
     },
     {
       "flag": "No code or raw data released",
       "detail": "Despite describing novel attack methods, no code, scripts, or raw experimental data are released, making independent verification impossible."
     },
     {
       "flag": "Broad claims from narrow model set",
       "detail": "The paper tests only three 7-8B open-source models and one embedding model, but the title and abstract imply broad vulnerability of RAG systems generally. No commercial models with stronger safety training (GPT-4, Claude) are tested."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations, threats to validity, or explicit scope boundaries. This is a significant omission for a security paper making vulnerability claims."
     },
     {
       "flag": "Missing hyperparameters and training details",
       "detail": "No training hyperparameters (learning rate, epochs, batch size) are reported for the retriever fine-tuning, and no inference parameters (temperature, top-p) are stated for the LLMs, despite these significantly affecting results."
     }
   ],
   "cited_papers": [
     {
       "title": "PoisonedRAG: Knowledge poisoning attacks to retrieval-augmented generation of large language models",
       "authors": ["Wei Zou", "Runpeng Geng", "Binghui Wang", "Jinyuan Jia"],
       "year": 2024,
       "relevance": "Direct baseline for corpus poisoning attacks on RAG systems, evaluating security vulnerabilities of LLM-integrated retrieval."
     },
     {
       "title": "Backdoor attacks on dense passage retrievers for disseminating misinformation",
       "authors": ["Quanyu Long", "Yue Deng", "LeiLei Gan", "Wenya Wang", "Sinno Jialin Pan"],
       "year": 2024,
       "relevance": "Prior work on backdoor attacks targeting dense retrievers in RAG, focusing on misinformation dissemination."
     },
     {
       "title": "AgentPoison: Red-teaming LLM agents via poisoning memory or knowledge bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang", "Chaowei Xiao", "Dawn Song", "Bo Li"],
       "year": 2024,
       "relevance": "Trigger-based attacks on retrieval-augmented LLM agents, directly relevant to RAG security evaluation."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "Certifiably robust RAG against retrieval corruption",
       "authors": ["Chong Xiang", "Tong Wu", "Zexuan Zhong", "David Wagner", "Danqi Chen", "Prateek Mittal"],
       "year": 2024,
       "relevance": "Defense mechanism for RAG systems against retrieval corruption, relevant to evaluating robustness of AI systems."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang", "Zihao Wang", "Xiaofeng Wang", "Tianwei Zhang", "Yepang Liu", "Haoyu Wang", "Yan Zheng", "Yang Liu"],
       "year": 2024,
       "relevance": "Systematic study of prompt injection attacks on LLM-integrated applications."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2024,
       "relevance": "Benchmark and defense mechanisms for indirect prompt injection, directly relevant to RAG security evaluation."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi", "Rashi Rungta", "Krithika Iyer", "Yuning Mao", "Michael Tontchev", "Qing Hu", "Brian Fuller", "Davide Testuggine", "Madian Khabsa"],
       "year": 2023,
       "relevance": "LLM-based safety guardrail system for sanitizing inputs and outputs, relevant to defense against prompt injection."
     },
     {
       "title": "Tensor Trust: Interpretable prompt injection attacks from an online game",
       "authors": ["Sam Toyer", "Olivia Watkins", "Ethan Adrian Mendes", "Justin Svegliato", "Luke Bailey", "Tiffany Wang", "Isaac Ong", "Karim Elmaaroufi", "Pieter Abbeel", "Trevor Darrell", "Alan Ritter", "Stuart Russell"],
       "year": 2023,
       "relevance": "Large-scale study of prompt injection attacks and defenses through an interactive game setting."
     },
     {
       "title": "A survey on large language model (LLM) security and privacy: The good, the bad, and the ugly",
       "authors": ["Yifan Yao", "Jinhao Duan", "Kaidi Xu", "Yuanfang Cai", "Zhibo Sun", "Yue Zhang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM security and privacy concerns, providing context for the attack landscape."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban", "Hamed Hassani", "George J. Pappas", "Eric Wong"],
       "year": 2024,
       "relevance": "Automated jailbreaking technique for LLMs relevant to understanding LLM vulnerability to adversarial inputs."
     }
   ]
 }