ai-research-survey

scan-v4.json (33320B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "DeRAG: Black-box Adversarial Attacks on Multiple Retrieval-Augmented Generation Applications via Prompt Injection",
     "authors": [
       "Jerry Wang",
       "Fang Yu"
     ],
     "year": 2025,
     "venue": "KDD Workshop on Prompt Optimization",
     "arxiv_id": "2507.15042",
     "doi": "10.48550/arXiv.2507.15042"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims competitive success rates vs GGPP (supported by Table 2), few tokens (supported by avg token columns), readability improvement validated by Welch's t-test (supported by Table 16), and detection evasion (supported by Table 4). The claims are reasonably hedged ('competitive and in some cases higher').",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (e.g., DE-optimized suffixes cause retrieval manipulation, early stopping reduces iterations) are supported by controlled single-variable ablations — comparing methods on the same data with only the attack strategy varying. The ablation studies (Section 4.3.2-4.3.3) isolate individual components.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims attacks on 'Multiple Retrieval-Augmented Generation Applications' but experiments use only BERT-base-uncased as the dense retriever and BM25 as the sparse retriever, with 1,000-document subsets. The conclusion states 'DeRAG reveals critical RAG vulnerabilities' without bounding this to the specific retrievers and small corpus sizes tested. Real RAG systems use more sophisticated retrievers and much larger corpora.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For instance, the high success rate on MS MARCO might be explained by corpus-specific properties rather than DE's general effectiveness. The paper does not consider whether results would hold with different retriever architectures, larger corpora, or whether BERT-base-uncased is unusually vulnerable.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between the retrieval manipulation proxy (Success@K, ΔMRR) and the downstream impact on answer quality, running a separate evaluation on SQuAD and NQ-Open (Section 4.5, Table 5) to validate that retrieval rank manipulation actually degrades generated answers.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations, threats-to-validity, or discussion section. The paper goes directly from evaluation results (Section 4) to a brief conclusion (Section 5) with no dedicated limitations discussion.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address limitations such as the small corpus size (1,000 documents vs. real-world millions), the single retriever model, or the gap between experimental and production RAG settings.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show, such as applicability to production-scale corpora, modern instruction-tuned retrievers, or multi-stage RAG pipelines with rerankers.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. There is no acknowledgments section disclosing grants or sponsors.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: both authors are from the Department of Management Information Systems, National ChengChi University, Taipei, Taiwan. No conflict with evaluated systems.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed. The absence of any funding disclosure makes this NO by default.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "RAG, Differential Evolution, adversarial prompt suffix, prompt injection, sparse/dense retrieval, and success@K are all defined or explained in Section 1–3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is explicitly stated: a gradient-free, black-box DE-based adversarial suffix optimization method (DeRAG) for manipulating RAG retrieval rankings without model internals access.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 covers related work on adversarial prompts (GGPP, TrojRAG, CtrlRAG), evolutionary optimization (DE, one-pixel attack), detection methods (APS, perplexity-based), and retrieval (BM25, DPR), with direct experimental comparison to GGPP and PRADA.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "A GitHub repository is provided: https://github.com/pen9rum/Rag_attack_DeRag (reference [34]), described as containing data and experiment results.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses publicly available BEIR benchmark datasets (MS MARCO, SciFact, FiQA, FEVER), and states 'Data and experiment results are available at' the GitHub repository (Section 4.1).",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are provided in the paper. Only the model (BERT-base-uncased, 110M parameters) is mentioned.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-style instructions for replicating experiments.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All main results in Tables 1-5 report only point estimates (e.g., Success@1 = 0.198) with no confidence intervals or error bars. The only ± notation appears in Table 15 for MLM NLL, not for the main attack results.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Welch's t-test is used only for the readability analysis (Table 16, Appendix H). The primary comparative claims — that DE outperforms GGPP and PRADA — are based solely on comparing point estimates across Tables 1 and 2 without any significance tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports multiple effect size metrics: ΔMRR, ΔnDCG@20, and Δcos (Tables 1, 2, 7), providing context for the magnitude of retrieval disruption. Downstream quality degradation is reported as percentage changes from baseline (Table 5).",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses 100 queries and 1,000-document subsets per dataset without any justification for why these sizes were chosen. No power analysis or discussion of whether 100 queries is sufficient for reliable estimates.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No standard deviations, variance, or multi-run spread measures are reported for the main attack results. Tables 1-5 all present single-run point estimates. DE is stochastic but no seed sensitivity is explored.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The paper compares against GGPP (white-box gradient-guided attack, reference [23]), PRADA (black-box sparse retrieval attack, reference [16]), and a random suffix baseline (Tables 1-2).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "GGPP (Hu et al., 2024) and PRADA (Wu & Zhang, 2022) are recent and represent the state of the art for white-box and black-box retrieval attacks respectively.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Multiple ablations are conducted: DE_seq_stop vs DE_seq vs DE_fixed_stop (Section 4.3.2, Figure 2), prefix vs suffix attacks (Table 3), suffix length variation (Appendix D, Figure 4), hinge loss vs cosine loss (Appendix E, Table 12), and pool size variation (Table 6).",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses Success@K (K=1,10,20), ΔMRR, ΔnDCG@20, Δcos, average tokens, and average iterations for attack evaluation. Downstream evaluation adds EM, F1, ROUGE-L, and BERTScore (Table 5).",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation is included. The readability of adversarial suffixes is assessed only via automated MLM NLL proxy (Appendix H), not human judgment. Detection evasion is tested only against automated detectors.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "No separation between development and test sets. The same 100 queries per dataset are used for both optimization and evaluation. There is no held-out test set mentioned.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by dataset (SciFact, FiQA, FEVER, MS MARCO), by method variant, by retrieval threshold K (1, 10, 20), and by attack outcome stratum (Top-1, Top-10 only, Fail) in Table 5.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports query counts where both prefix and suffix fail. Table 5 stratifies results by 'Fail' outcome. The paper notes diminishing returns beyond 5 tokens (Appendix D) and discusses when cosine loss fails vs hinge loss (Appendix E).",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that cosine loss is consistently worse than hinge loss (Appendix E, Table 12), that a monotonic suffix length schedule did not improve results (Section 4.3.3, Appendix D), and that suffixes beyond 5 tokens yield diminishing or negative marginal returns (Figure 4b).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "The main retriever is specified as 'BERT-base-uncased encoder' with 'over 110 million parameters' and '768-dimensional CLS embeddings' (Section 4.1). For detection, 'RoBERTa detector' is named (Section 4.4.1). These are specific, uniquely identifiable models.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "The downstream QA evaluation (Section 4.5) feeds retrieved chunks to an LLM generator, but the generation prompt/template is not provided. The adversarial suffixes themselves are shown (Appendix C), but the prompt format for the downstream answer generation is missing.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "DE hyperparameters are reported: scale factor F (0.5-1.0), crossover rate CR (0.1-0.9), suffix length budget n_max (1-10), patience T, and population size N (Section 3.3). Specific values for the example: N=3, F=0.5, CR=0.5. The downstream evaluation uses 5-token suffix with up to 120 iterations (Section 4.5).",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The method is a direct optimization loop (DE) over token sequences evaluated against a retriever.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 'we randomly sample a subset of 1,000 documents and 100 queries' but leaves ambiguity in target passage selection: 'either drawn uniformly at random from the non-relevant documents, or selected as a topically confusable distractor' (Section 4.1) — it is unclear which method was actually used in which experiments.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The underlying BEIR datasets are publicly available, and the paper states 'Data and experiment results are available at' the GitHub repository (Section 4.1, reference [34]).",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 describes the data source (BEIR framework), sampling procedure (1,000 documents and 100 queries per dataset), encoding method (BERT-base-uncased CLS embeddings), and retrieval mechanism (cosine similarity for dense, BM25 for sparse).",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. All data comes from standard public benchmarks (BEIR/MS MARCO/SciFact/FiQA/FEVER).",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The pipeline from raw BEIR data to experimental setup has gaps. The target passage selection method is ambiguously described ('either drawn uniformly at random... or selected as a topically confusable distractor'). The specific random seed for sampling is not stated, and the downstream QA evaluation pipeline details are sparse.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper evaluates an adversarial attack method against a retriever, not a pre-trained model's knowledge on a benchmark. BERT-base-uncased is used as a retrieval encoder, and the attack's validity does not depend on whether the model has seen the benchmark data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "The paper tests an attack strategy on retrieval behavior, not model knowledge. Train/test overlap in the BERT pre-training data is irrelevant to whether adversarial suffixes can manipulate retrieval rankings.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Contamination is not relevant because the paper evaluates the effectiveness of an adversarial attack method on retrieval systems, not the retriever's inherent knowledge of benchmark answers.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. All experiments are automated attacks on retrieval systems.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Pure computational experiment on retrieval benchmarks.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Average iteration counts are reported per method per dataset (Tables 1-2). Pool construction and query optimization times are reported in Appendix G (Table 14), with per-query times around 40-50 seconds. Section 4.3.1 discusses the iterations-tokens trade-off in detail.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget is stated. The paper does not report total GPU hours, hardware specifications, or aggregate compute cost across all experiments.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "DE is a stochastic algorithm but no multi-seed results are reported. All tables present single-run point estimates without any analysis of sensitivity to random initialization.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state how many times each experiment was run. Results appear to be from single runs, but this is never explicitly stated.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "DE hyperparameters (F, CR, N, patience) are reported but there is no description of how they were selected — no search budget, no grid/random search, no sensitivity analysis over hyperparameter configurations.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper presents results for DE_seq_stop, DE_fixed_stop, and DE_seq variants but does not justify how specific hyperparameter values were chosen or whether these are the best among tested configurations.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "The paper compares multiple methods across multiple datasets and thresholds without any correction for multiple comparisons. Only the readability analysis uses a significance test (Welch's t-test), with no family-wise correction.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors implement their own version of GGPP and PRADA baselines for comparison. There is no acknowledgment that their reimplementation may systematically underperform the original authors' implementations.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "GGPP iterations are listed as '—' in Table 2, making matched-compute comparison impossible. The paper discusses the iterations-tokens trade-off qualitatively (Section 4.3.1) but does not compare methods at equivalent compute budgets.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether Success@K on 1,000-document subsets with BERT-base-uncased actually measures vulnerability of real-world RAG systems, which use much larger corpora and more sophisticated retrievers.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved. The evaluation is a direct optimization against a retriever, not a scaffold-dependent agent evaluation.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether BERT-base-uncased's pre-training data includes any of the BEIR benchmark documents. BERT (2018) may have been trained on data overlapping with some benchmarks.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the attack evaluation setup leaks information that would not be available in a real attack scenario (e.g., knowing the exact corpus and encoder used).",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the sampled 100 queries and 1,000 documents are independent or share structural similarities that could inflate success rates.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference, or decontamination analysis.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DE-based black-box attacks (DeRAG) achieve competitive or higher success rates vs. white-box GGPP on dense retrievers using ≤5 tokens",
       "evidence": "Table 2: DE_seq_stop achieves 0.739 Succ@20 vs GGPP 0.565 on SciFact; DE_seq_stop matches or exceeds GGPP on 3/4 datasets at Succ@10 and Succ@20",
       "supported": "strong"
     },
     {
       "claim": "DE_seq_stop outperforms PRADA at Succ@1 on sparse retrievers while using comparable token budgets",
       "evidence": "Table 1: DE_seq_stop Succ@1 ranges 0.25–0.71 vs PRADA 0.01–0.07 across all four datasets, though PRADA achieves higher Succ@10/20 on most datasets",
       "supported": "moderate"
     },
     {
       "claim": "Early stopping reduces query cost by ~40% while maintaining attack success",
       "evidence": "Section 4.3.2 and Figure 2: DE_seq_stop reaches 50% Succ@1 with 2 tokens vs DE_seq requiring 5 tokens; Table 2 shows iteration savings",
       "supported": "moderate"
     },
     {
       "claim": "DE-generated suffixes evade BERT-based detection at near-chance accuracy (AUROC 0.2023)",
       "evidence": "Table 4: AUROC 0.2023, AUPRC 0.4665 at target FPR 0.5%; at 2% FPR threshold, recall is only 0.62%",
       "supported": "strong"
     },
     {
       "claim": "Readability-aware MLM pooling strategy significantly reduces suffix perplexity without degrading attack success",
       "evidence": "Tables 15–16: Welch's t-test confirms significant NLL reduction (p < 10^-9 across all datasets) for pool_size=5000 vs full pool; Table 6 shows stable Succ@1 across pool sizes",
       "supported": "strong"
     },
     {
       "claim": "Adversarial chunk insertion causes 14–27% downstream QA quality degradation even at Top-10 occupancy",
       "evidence": "Table 5: SQuAD average EM drops 26.7%; NQ-Open average EM drops 14.8%; Top-10-only success still causes 49.7% EM drop on SQuAD",
       "supported": "moderate"
     },
     {
       "claim": "Combining prefix and suffix attacks is complementary and improves overall attack success",
       "evidence": "Table 3: On FiQA, 17/100 queries succeed with both prefix and suffix where neither alone succeeds, and 45/100 succeed with either strategy",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "DeRAG demonstrates that gradient-free Differential Evolution optimization can generate adversarial token suffixes (≤5 tokens) that effectively manipulate RAG retrieval rankings without requiring access to model internals, matching or exceeding white-box GGPP attacks on dense retrievers. The approach evades BERT-based adversarial prompt detection with near-chance AUROC (0.2023), suggesting token-level perturbations are difficult to distinguish from benign queries. Downstream QA quality degrades 14–27% on average when adversarial chunks enter the retrieval window, with the worst degradation occurring even when the target reaches only Top-10 (not rank 1). A readability-aware MLM pooling strategy significantly reduces suffix perplexity (Welch's t-test p < 10^-9) while maintaining attack effectiveness.",
   "red_flags": [
     {
       "flag": "Tiny evaluation corpus",
       "detail": "Main experiments use only 1000-document subsets and 100 queries per dataset; real RAG deployments have millions of documents, and attack success may not transfer to large-scale corpora where adversarial suffixes compete against far more documents."
     },
     {
       "flag": "Single retriever architecture",
       "detail": "All dense retriever experiments use BERT-base-uncased (2018); no modern dense retrievers (DPR, Contriever, E5, Arctic-Embed) are tested, despite being mentioned in the introduction as state-of-the-art."
     },
     {
       "flag": "Downstream QA generator unspecified",
       "detail": "Table 5 reports EM/F1/ROUGE-L/BERTScore for a RAG pipeline, but the LLM generator model, version, prompt template, and generation parameters are never stated, making these results irreproducible."
     },
     {
       "flag": "No variance on main metrics",
       "detail": "Attack success rates in Tables 1–4 are point estimates over 100 queries with no confidence intervals or standard deviations; claimed differences of a few percentage points may not be statistically significant."
     },
     {
       "flag": "CtrlRAG omitted from comparison",
       "detail": "CtrlRAG (Sui 2025), a directly comparable black-box MLM-based RAG attack from the same year, is cited in the introduction but not included as a baseline."
     },
     {
       "flag": "No limitations section",
       "detail": "Despite significant scope restrictions (small corpus, one retriever, no defense analysis, workshop venue), the paper contains no limitations or threats-to-validity discussion."
     }
   ],
   "cited_papers": [
     {
       "title": "Prompt Perturbation in Retrieval-Augmented Generation based Large Language Models (GGPP)",
       "relevance": "Primary white-box baseline for dense retriever attack comparison throughout the paper"
     },
     {
       "title": "PRADA: Practical Black-box Adversarial Attacks against Neural Ranking Models",
       "relevance": "Primary black-box baseline for sparse retriever attack comparison"
     },
     {
       "title": "BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models",
       "relevance": "Evaluation framework used to standardize results across four QA datasets"
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "relevance": "Foundational RAG paper establishing the pipeline architecture being attacked"
     },
     {
       "title": "CtrlRAG: Black-box Adversarial Attacks Based on Masked Language Models in Retrieval-Augmented Language Generation",
       "relevance": "Contemporary competing black-box RAG attack method using MLM; cited but not compared"
     },
     {
       "title": "BadRAG: Identifying Vulnerabilities in Retrieval Augmented Generation of Large Language Models",
       "relevance": "Related poisoning-based backdoor attack on RAG systems"
     },
     {
       "title": "One Pixel Attack for Fooling Deep Neural Networks",
       "relevance": "Motivating prior work showing DE can craft adversarial examples in non-gradient settings"
     },
     {
       "title": "Black-box Adversarial Sample Generation Based on Differential Evolution",
       "relevance": "Prior work applying DE to adversarial sample generation; methodological foundation"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security researchers and RAG system developers could use this to test retrieval robustness; code is released but requires domain expertise to apply."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "RAG retrieval vulnerability to adversarial inputs is already known; the contribution is a specific black-box method rather than a surprising finding."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that RAG systems can be attacked without any access to model internals and that attacks evade detection, raising practical security concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to specific companies or products, straightforward technical contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data are released on GitHub; a researcher could reproduce the attacks, but there is no pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from National ChengChi University with no industry affiliation or brand recognition in the AI community."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44120359",
         "title": "Diffusion vs. Autoregressive Language Models: A Text Embedding Perspective",
         "points": 19,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=44120359"
       },
       {
         "hn_id": "36931866",
         "title": "Universal and Transferable Adversarial Attacks on LLM",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36931866"
       },
       {
         "hn_id": "36903968",
         "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36903968"
       }
     ],
     "top_points": 19,
     "total_points": 23,
     "total_comments": 1
   }
 }