ai-research-survey

scan-v5.json (26033B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "DeRAG: Black-box Adversarial Attacks on Multiple Retrieval-Augmented Generation Applications via Prompt Injection",
     "authors": ["Jerry Wang", "Fang Yu"],
     "year": 2025,
     "venue": "KDD Workshop on Prompt Optimization",
     "arxiv_id": "2507.15042",
     "doi": "10.48550/arXiv.2507.15042"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by experimental results: DE vs. GGPP/PRADA comparisons in Tables 1-2, ≤5 token budgets confirmed, Welch's t-test for readability in Table 16, and AUROC 0.2023 for detector evasion in Table 4.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims such as 'early stopping cuts query cost by ~40%' are supported by controlled comparisons of DE variants across multiple datasets with consistent ablations (Figure 2, Table 2).",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and conclusion claim DeRAG attacks 'multiple RAG applications,' but experiments only cover BERT-base-uncased (dense) and BM25 (sparse) on 1,000-document subsets; modern embedding models and production-scale corpora are untested.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes MS MARCO's high success to corpus redundancy but does not explore whether BERT-base-uncased is unusually vulnerable compared to modern retrievers, nor whether the artificially small corpus (1,000 docs) inflates success rates.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes retrieval rank manipulation (Success@K) from downstream answer quality and validates the connection in Table 5, showing EM/F1/ROUGE-L/BERTScore degradation stratified by attack outcome.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the conclusion briefly mentions future defenses but does not enumerate limitations of the current work.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed — e.g., the impact of corpus size (1,000 docs vs. production scale), retriever model choice, or the small query count (100) on result generalizability.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly bound what the results do NOT show — for example, whether attacks transfer to instruction-tuned embedding models, larger corpora, or API-based retrieval services.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors' affiliations (Department of Management Information Systems, National ChengChi University, Taipei, Taiwan) are disclosed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is identified, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: RAG (Section 1), Differential Evolution (Section 2.2), dense/sparse retrievers (Section 2.4), and all evaluation metrics (Success@K, ΔMRR, ΔnDCG, Δcos) are formally defined in Section 4.1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states its contribution: a gradient-free, black-box adversarial attack (DeRAG) using Differential Evolution to generate short adversarial suffixes that manipulate RAG retrieval rankings without model internals.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "A four-subsection related work covers adversarial prompts, evolutionary optimization, detection methods, and retrieval mechanisms; the paper explicitly compares against GGPP (white-box) and PRADA (sparse black-box) throughout.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code is released at https://github.com/pen9rum/Rag_attack_DeRag, explicitly referenced in Section 4.1.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All datasets used (MS MARCO, SciFact, FiQA, FEVER, SQuAD, NQ-Open) are standard publicly available benchmarks accessible via the BEIR framework.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or version-pinned dependency specifications are provided; only the model name (BERT-base-uncased) is mentioned without library or Python version details.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The algorithm is described via pseudocode (Algorithm 1) but no step-by-step instructions for reproducing the specific experimental results (table entries, figures) are provided in the paper.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Main attack success results (Tables 1-2) report point estimates only; standard deviations appear only for MLM NLL in Table 15, not for primary Success@K metrics.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Welch's t-test is used only for the MLM NLL readability comparison (Table 16); the primary comparative claims — DE vs. GGPP vs. PRADA attack success — are made without any statistical significance testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported across multiple metrics: Success@K, ΔMRR, ΔnDCG, Δcos, and downstream percentage drops in EM/F1/ROUGE-L in Table 5.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 100 queries and 1,000 documents per dataset is not justified through power analysis or reasoning about statistical adequacy.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Main results tables (Tables 1-2) report no variance; standard deviations appear only for MLM NLL (Table 15), not for primary attack success metrics.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three baselines are included: GGPP (gradient-based white-box), PRADA (sparse black-box), and random suffix.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "GGPP (2024) and PRADA (2022) are the most relevant contemporary methods for the dense and sparse retriever attack settings respectively.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Extensive ablations: DE variants (seq_stop vs. fixed_stop vs. seq), suffix length effects (Figure 4, Appendix D), loss function comparison (Appendix E), prefix vs. suffix positioning (Table 3), and candidate pool size effects (Table 6).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics: Success@K (K=1,10,20), Avg Tok, Avg Iter, ΔMRR, ΔnDCG, Δcos, EM, F1, ROUGE-L, BERTScore, AUROC, AUPRC, MLM NLL.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation is not applicable; attack effectiveness is measured through automated retrieval and QA metrics.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is an adversarial optimization task, not a supervised prediction task; the held-out test set concept does not apply.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per dataset (SciFact, FiQA, FEVER, MS MARCO) and per retrieval threshold (K=1, 10, 20) across all main tables.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 explicitly tabulates queries where both prefix and suffix attacks fail; Section 4.5 stratifies outcomes into Top-1 success, Top-10-only, and Fail with corresponding quality metrics.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results include: cosine loss underperforming hinge loss (Table 12/Appendix E), monotonic suffix length schedule not improving results (Appendix D), and low Succ@1 rates (~10-20%) on most datasets.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "BERT-base-uncased is specified for retrieval, but the LLM used for answer generation in the downstream RAG pipeline (Section 4.5, Table 5) is never named or versioned — a critical omission.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "No system prompts or query templates for the downstream RAG generator LLM are provided; appendix tables show adversarial suffix output examples but not the generation prompts used.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "DE hyperparameters (F, CR, N, patience T) are described as typical ranges (e.g., F ∈ [0.5, 1.0], CR ∈ [0.1, 0.9]) rather than the exact values used in the reported experiments.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; this is a direct adversarial optimization attack on retrieval systems.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Data preprocessing is documented: random sampling of 1,000 documents and 100 queries from official BEIR corpus/query splits, BERT-base-uncased CLS embedding extraction (768-dim), cosine similarity retrieval.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Per-query attack outcomes and generated adversarial suffixes are not released as structured data; only the code repository is linked and we cannot verify its contents from the paper.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described: standard BEIR benchmarks, random sampling of 1,000-document subsets and 100 queries from official splits, target documents chosen as non-relevant passages or topically confusable distractors.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data comes from standard NLP benchmarks.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline is documented: query/target selection → BERT embedding → DE optimization loop (Algorithm 1) → retrieval evaluation → downstream QA generation and scoring.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This paper evaluates an adversarial attack on retrieval ranking, not LLM benchmark knowledge recall; standard contamination concerns do not apply.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable; benchmarks are used as retrieval corpora to be manipulated, not as knowledge tests for a generative model.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable for the same reason as above.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Per-query iteration counts (Table 2) and pool construction/query optimization times in seconds (Table 14) are reported, providing practical cost information.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Total computational budget (GPU/CPU hours, hardware specifications) is not stated; only per-query timing data are provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DE-based black-box attack achieves competitive or higher success rates than GGPP (gradient white-box) at Top-10 and Top-20 thresholds on dense retrievers",
       "evidence": "Table 2: DE_seq_stop matches or exceeds GGPP at Succ@10/Succ@20 on SciFact (0.573 vs 0.458) and FiQA (0.520 vs 0.480), though GGPP dominates Succ@1 on MS MARCO (0.830 vs 0.570)",
       "supported": "moderate"
     },
     {
       "claim": "Effective adversarial suffixes require only 2-3 tokens on average",
       "evidence": "DE_seq_stop achieves average suffix lengths of 1.32 (MS MARCO) to 2.76 (FEVER) tokens while maintaining high Top-10/Top-20 success rates (Table 2)",
       "supported": "strong"
     },
     {
       "claim": "Early stopping reduces query cost by approximately 40% without reducing attack success",
       "evidence": "Figure 2 shows DE_seq_stop reaches 97% Top-10 success at 2 tokens while DE_seq needs 3-4; Section 3.3.3 states the hybrid strategy 'cuts average query cost by ~40%'",
       "supported": "strong"
     },
     {
       "claim": "DE-generated suffixes evade BERT-based and RoBERTa-based adversarial detection",
       "evidence": "Table 4: RoBERTa detector achieves AUROC 0.2023 and AUPRC 0.4665 at 0.5% FPR target; Table 13: CLS attack probability is near-identical for original vs. attacked queries (~0.40)",
       "supported": "strong"
     },
     {
       "claim": "Readability-aware MLM pooling strategy significantly reduces suffix perplexity without degrading attack success",
       "evidence": "Table 16: Welch's t-test yields p < 1e-9 for NLL reduction across all three datasets; Table 6 shows stable Success@1 across pool sizes from 500 to 30,522",
       "supported": "strong"
     },
     {
       "claim": "Adversarial retrieval manipulation causes substantial downstream answer quality degradation in real QA pipelines",
       "evidence": "Table 5 shows 83.5% EM drop on SQuAD when target reaches rank 1 and 14.8% average EM drop across NQ-Open; tested on only 500 queries per dataset with unspecified generator LLM",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DeRAG demonstrates that gradient-free Differential Evolution can generate adversarial query suffixes of 2-3 tokens that effectively manipulate RAG retrieval rankings, matching gradient-based white-box attacks (GGPP) at broader retrieval thresholds (Top-10, Top-20) while requiring no model internals. The attack evades RoBERTa-based detectors with near-chance accuracy (AUROC 0.2023) and causes measurable downstream answer quality degradation (14-27% average EM drop on QA benchmarks, up to 83.5% when the adversarial document reaches rank 1). A readability-aware suffix construction strategy using MLM token pooling statistically significantly reduces suffix perplexity (Welch's t, p < 1e-9) without degrading attack success. However, all experiments use BERT-base-uncased on artificially small 1,000-document corpus subsets, limiting generalizability claims.",
   "red_flags": [
     {
       "flag": "Unrealistically small corpus",
       "detail": "Experiments use only 1,000-document subsets of BEIR datasets; production RAG systems operate over millions of documents where attack rank targets and success rates would differ substantially."
     },
     {
       "flag": "No CIs or significance tests on primary results",
       "detail": "Attack success rates (Success@K, ΔMRR, ΔnDCG) in Tables 1-2 are point estimates from 100 queries per dataset with no confidence intervals or statistical significance tests for the main comparative claims."
     },
     {
       "flag": "Single retriever model tested",
       "detail": "Dense retrieval experiments use only BERT-base-uncased (2018); modern instruction-tuned embedding models widely used in production (E5, GTE, OpenAI text-embedding-3) are untested."
     },
     {
       "flag": "Generator LLM unspecified for downstream evaluation",
       "detail": "Section 4.5 evaluates downstream RAG answer quality (Table 5) but never names or versions the LLM used for generation, making these results unreproducible."
     },
     {
       "flag": "Exact DE hyperparameters not reported",
       "detail": "The paper provides typical ranges for DE parameters (F ∈ [0.5, 1.0], CR ∈ [0.1, 0.9], N, T) but not the exact values used in the reported experiments."
     },
     {
       "flag": "Title overstates breadth of coverage",
       "detail": "Title claims attacks on 'Multiple Retrieval-Augmented Generation Applications' but only one dense retriever (BERT-base-uncased) and one sparse retriever (BM25) on small corpus subsets are tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Prompt Perturbation in Retrieval-Augmented Generation based Large Language Models (GGPP)",
       "relevance": "Primary dense-retriever baseline; gradient-based white-box attack on RAG that DeRAG is designed to compete with without gradient access"
     },
     {
       "title": "PRADA: Practical Black-box Adversarial Attacks against Neural Ranking Models",
       "relevance": "Primary sparse-retriever baseline; the most comparable black-box adversarial ranking attack method"
     },
     {
       "title": "BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models",
       "relevance": "Evaluation framework used for all retrieval experiments across SciFact, FiQA, FEVER, MS MARCO"
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "relevance": "Original RAG paper; defines the RAG paradigm whose retrieval stage DeRAG attacks"
     },
     {
       "title": "BadRAG: Identifying Vulnerabilities in Retrieval Augmented Generation of Large Language Models",
       "relevance": "Related poisoning-based backdoor attack on RAG corpora; complementary threat model"
     },
     {
       "title": "CtrlRAG: Black-box Adversarial Attacks Based on Masked Language Models in Retrieval-Augmented Language Generation",
       "relevance": "Related black-box RAG attack using MLM; close competitor using a different gradient-free approach"
     },
     {
       "title": "Differential evolution – a simple and efficient heuristic for global optimization over continuous spaces",
       "relevance": "Foundational algorithm (Storn & Price 1997) underlying the DeRAG optimization method"
     },
     {
       "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding",
       "relevance": "Core retrieval encoder used in all dense retrieval experiments"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Demonstrates a deployable black-box attack against production RAG systems with public code; directly actionable for practitioners assessing RAG security."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that ≤5 tokens suffice for effective retrieval manipulation is noteworthy, but RAG vulnerability to adversarial attacks is an expected result in this field."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Shows RAG systems can be manipulated to surface misinformation via small, detector-evading token appends — a concrete and practical AI safety concern for deployed systems."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The attack-vs-defense framing is inherently adversarial but the paper is technical rather than polemical; no controversial claims about deployed systems."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub using public BEIR benchmarks and BERT-base-uncased, making reproduction accessible to practitioners without specialized resources."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from National ChengChi University (Taiwan), a respected institution but not a major AI lab with brand recognition in the LLM community."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44120359",
         "title": "Diffusion vs. Autoregressive Language Models: A Text Embedding Perspective",
         "points": 19,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=44120359"
       },
       {
         "hn_id": "36931866",
         "title": "Universal and Transferable Adversarial Attacks on LLM",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36931866"
       },
       {
         "hn_id": "36903968",
         "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36903968"
       }
     ],
     "top_points": 19,
     "total_points": 23,
     "total_comments": 1
   }
 }