ai-research-survey

scan.json (30919B)

---

 {
   "paper": {
     "title": "Exploring the Security Threats of Knowledge Base Poisoning in Retrieval-Augmented Code Generation",
     "authors": [
       "Bo Lin",
       "Shangwen Wang",
       "Liqian Chen",
       "Xiaoguang Mao"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.03233",
     "doi": "10.48550/arXiv.2502.03233"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Knowledge base poisoning in RACG systems poses a significant security threat: even a single poisoned code example can compromise up to 48% of generated code (CodeLlama with JINA retriever). Dense retrievers (JINA) are far more susceptible than sparse retrievers (BM25) due to better semantic matching. Code-specialized LLMs are more prone to generating vulnerable code than general-purpose models. Vulnerability propagation risk increases sharply when example-query similarity exceeds 60%, and CWE-352 (CSRF) consistently shows the highest vulnerability rates (~0.8) among top CWE types.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. The paper references third-party implementations (BM25 from GitHub, JINA from Huggingface) but does not release its own experimental code or scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The base dataset ReposVul is publicly available, but the authors' constructed dataset—including LLM-generated queries, poisoned knowledge bases, and generated code outputs—is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "§4.6 mentions 'a single A100-40G GPU server using the Ollama framework' with some model parameters, but no requirements.txt, Dockerfile, or detailed dependency list is provided. Not sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or runnable scripts are provided. A researcher would need to reverse-engineer the full pipeline from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4–9 are point estimates (e.g., VR = 0.48). No confidence intervals, error bars, or uncertainty measures are reported anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., '6.5% more vulnerabilities from one-shot to three-shot', 'CodeLlama exhibits the highest susceptibility') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout. For example, §5.1.1 states 'CodeLlama's VR increases from 0.29 to 0.48' (a 19pp increase), and §5.1.2 reports '6.5% (0.46→0.49)' providing both absolute and relative context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 12,053 instances from ReposVul. No justification for why this size is sufficient, and no power analysis is discussed. The manual inspection sample sizes (81–95) are justified via confidence level calculation (§6.2)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. §4.6 sets temperature to 0 'to reduce non-determinism' but does not report variance across runs or verify determinism."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "All experiments include a zero-poisoning baseline (poisoning number 0 in Table 4, proportion 0 in Table 5), allowing comparison of poisoned vs. unpoisoned performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The four LLMs (GPT-4o, Llama-3-8B, CodeLlama-13B, DeepSeek-Coder-V2-16B) are contemporary as of the study period, with models selected from the LLM Safety Leaderboard as of October 2024 (§4.4.1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies poisoning quantity (0–9 samples, 0–100% proportion), number of shots (1 vs 3), retriever type (JINA vs BM25), programming language, similarity range, and CWE type—each serving as an ablation of a different factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Vulnerability Rate (VR), CrystalBLEU similarity, and Vulnerability Rate in Retrieved Code (VRRC), defined in §4.5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "§6.2 describes manual inspection where 'Two authors independently evaluated the samples through manual review' on 360 generated code samples (95+81+93+91) to validate the LLM judge's accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation of development and test sets is described. The same dataset appears to be used for all experiments without a held-out split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: by programming language (Table 7), CWE type (Tables 9, 12), similarity range (Table 8), retriever, scenario, and LLM."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where poisoning is less effective: BM25 shows smaller VR increases than JINA (§5.1.1), CWE-434 has lowest VR (§5.2.3), and Scenario II is much harder for attackers (§5.1.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative/null results: poisoning has 'negligible' impact on code functionality (similarity metric barely changes), BM25 retriever shows minimal vulnerability increase, and low-similarity examples have 'relatively minor impact on vulnerability likelihood' (§5.2.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claim of '48% of the generated code' being vulnerable from a single poisoned sample is supported by Table 4 (CodeLlama + JINA: 0.29→0.48). The 6.5% increase from one-shot to three-shot matches Table 6 aggregated results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The study uses controlled experimental manipulation: systematically varying the number of poisoned examples while holding other factors constant. The causal claim that poisoning 'compromises' security is supported by this controlled design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 explicitly bounds generalization: the four languages 'may not fully represent real-world development scenarios' and account for only '42.7% of the total activity.' Findings are presented per-model and per-retriever rather than as universal claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6.3 provides an alternative explanation for JINA vs BM25 differences (retrieval effectiveness, validated with MRR/SR@k metrics). §6.5 discusses query generation accuracy (86%) as a potential confound. §5.1.1 discusses inherent LLM vulnerability generation even without poisoning."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "VR is measured by an LLM judge with 77–81% accuracy (§6.2), meaning ~20% of labels are incorrect. The paper does not discuss the gap between LLM-judge-detected vulnerability and actual exploitable security risk. The proxy (LLM judge classification) is treated as equivalent to the outcome (real-world security threat)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified as 'GPT-4o', 'Llama-3-8B', 'CodeLLAMA-13B', 'DeepSeek-Coder-V2-16B'. Parameter counts are given for open-source models, but no snapshot dates or API versions are specified. 'GPT-4o' is a marketing name without a snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix provides prompt templates for query generation (Prompt 1), vulnerability extraction (Prompt 2), and security assessment (Prompt 3), but all contain placeholders ({LANGUAGE}, {FUNCTION}, {DIFF}, etc.). The actual code generation prompt used to instruct LLMs in the RACG pipeline is not provided at all."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§4.6 reports: 'temperature of 0, top-p value of 0.95, a max_new_tokens setting of 4096, and a context window of 8192, keeping other parameters at default values.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard RAG pipeline (retriever + LLM generation) without agents, tools, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§4.1 documents filtering: functions shorter than three lines and names containing 'test' were removed. §4.2 describes knowledge base construction and poisoning process in detail. Table 2 provides final statistics (12,053 instances, 236 CWEs)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 'Threats to Validity' is a dedicated subsection with substantive discussion of query generation accuracy, programming language coverage, and their implications."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 discusses study-specific threats: DeepSeek-V2.5 may produce inaccurate queries (mitigated by manual review showing 86% accuracy), and the four programming languages account for only 42.7% of GitHub activity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While §6.5 notes that 'the selected languages may not fully represent real-world development scenarios,' the paper does not explicitly state what the results do NOT show—e.g., no statement about applicability to different retriever types, real-world deployment settings, or non-function-level poisoning."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated code, judge outputs, poisoned knowledge bases) is made available. Only aggregated results in tables are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§4.1 describes dataset selection in detail: 12 candidate datasets evaluated against 4 criteria (Table 1), ReposVul selected as it satisfies all requirements. Filtering criteria and resulting statistics (Table 2) are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from the ReposVul dataset, a standard publicly available vulnerability dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: ReposVul → filtering (§4.1) → query generation (§4.1) → knowledge base construction (§4.2) → poisoning (§4.2.1/§4.2.2) → code generation → result validation (§4.3). Each stage has explicit criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text. Whether the work is funded or unfunded is unknown."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with the National University of Defense Technology. They evaluate third-party LLMs (GPT-4o, Llama-3, CodeLlama, DeepSeek-Coder), not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding from outcomes cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four LLMs used. This matters because ReposVul vulnerabilities may be in GPT-4o's or Llama-3's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ReposVul code examples appeared in any LLM's training data. The models may have memorized the secure or vulnerable code patterns, confounding the results."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ReposVul is built from public repositories available before model training cutoffs. No discussion of whether models have seen these code snippets during training, which could affect both VR baseline and poisoning effectiveness."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Manual validation of the LLM judge by the authors is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes code generation and vulnerability propagation, not human behavior."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example latency are reported despite using the GPT-4o API and running 16 sub-scenarios across 12,053 instances."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('A100-40G GPU server') but no total compute budget (GPU hours, API spend, wall-clock time for experiments) is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 'to reduce non-determinism' (§4.6) but no multi-seed or multi-run experiments are conducted. Determinism is assumed, not verified."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results appear to be from a single run per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (temperature=0, top-p=0.95, etc.) appear to be fixed choices with no search or justification for these values beyond 'keeping other parameters at default values.'"
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results across all configurations rather than selecting a best one. For RQ2 analysis, moderate poisoning quantities are chosen with justification: 'LLM-generated code exhibits similar patterns across all metrics, regardless of poisoning quantity' (§5.2)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 4 LLMs × 2 retrievers × 2 scenarios × multiple poisoning levels without any multiple comparison correction or even significance testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors evaluate third-party LLMs (GPT-4o, Llama-3, CodeLlama, DeepSeek-Coder) rather than their own system. The poisoning methodology is the contribution, and they report results across all models without cherry-picking favorable configurations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (8B vs GPT-4o) without discussing compute differences. No performance-per-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The VR metric relies on an LLM judge with 77–81% accuracy. The paper validates the judge (§6.2) but does not discuss whether detecting vulnerability patterns via LLM actually measures real-world security risk—e.g., whether flagged vulnerabilities are exploitable."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The RAG pipeline is simple retrieval + generation without scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. ReposVul contains vulnerabilities from public repositories that predate the LLMs' training data, meaning models may have already seen these code patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing vulnerable code as in-context examples leaks information differently than real-world RAG settings (e.g., whether format/structure of injected examples is distinguishable)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and test examples share structural similarities (same repositories, same authors, duplicate patterns) that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, decontamination) are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Even a single poisoned code example can compromise up to 48% of generated code when programming intent is exposed.",
       "evidence": "Table 4 shows CodeLlama with JINA retriever: VR increases from 0.29 (baseline) to 0.48 with 1 poisoned sample in Scenario I (§5.1.1).",
       "supported": "strong"
     },
     {
       "claim": "Without access to user intent, injecting vulnerability code equivalent to 20% of the knowledge base can lead to approximately 36% of generated code being vulnerable.",
       "evidence": "Table 5 shows CodeLlama with JINA at 0.2 poisoning proportion: VR = 0.36 in Scenario II (§5.1.1).",
       "supported": "strong"
     },
     {
       "claim": "More few-shot examples increase vulnerability rates: LLMs generated 6.5% more vulnerabilities from one-shot to three-shot with JINA retriever.",
       "evidence": "Table 6 shows aggregated VR ('All' column) increasing from 0.46 to 0.49 (one-shot to three-shot) in JINA-I setting (§5.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "Code LLMs are more prone to generating vulnerable code than general-purpose LLMs.",
       "evidence": "Table 4 shows CodeLlama consistently has the highest VR across all configurations, followed by DS-Coder. Llama-3 (general-purpose) shows lower VR (§5.1.1).",
       "supported": "moderate"
     },
     {
       "claim": "C++ consistently exhibits the highest vulnerability rate across most sub-scenarios.",
       "evidence": "Table 7 shows C++ averaging 0.47 (JINA-I), 0.42 (BM25-I), 0.46 (JINA-II), 0.44 (BM25-II) compared to lower rates for Java and Python (§5.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "Example-query similarity above 60% significantly increases vulnerability risk.",
       "evidence": "Table 8 shows VR jumps from 0.35 at [40,60) to 0.42 at [60,80) and 0.53 at [80,100] in Scenario I (§5.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "CWE-352 (Cross-Site Request Forgery) consistently shows the highest vulnerability rates (~0.8) among MITRE Top-10.",
       "evidence": "Table 9 shows CWE-352 averaging 0.79 in Scenario I and 0.78 in Scenario II, far above other CWE types (§5.2.3).",
       "supported": "strong"
     },
     {
       "claim": "Dense retrievers (JINA) introduce significantly more vulnerabilities than sparse retrievers (BM25).",
       "evidence": "Tables 4-6 consistently show higher VR with JINA vs BM25. Table 11 confirms JINA's superior retrieval capability (MRR 0.85 vs 0.20) which amplifies vulnerability propagation (§6.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge with significant error rate",
       "detail": "The primary evaluation metric (VR) relies on an LLM judge (DeepSeek-V2.5) with only 77–81% accuracy across languages (§6.2). This means approximately 1 in 5 vulnerability labels may be incorrect, but no uncertainty propagation from this error rate to the final results is performed."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Numerous comparative claims are made (e.g., '6.5% more vulnerabilities', 'CodeLlama exhibits the highest susceptibility') based solely on comparing point estimates without any significance tests, confidence intervals, or error bars."
     },
     {
       "flag": "Training data contamination unaddressed",
       "detail": "ReposVul contains code from public repositories that likely appeared in the training data of GPT-4o, Llama-3, and CodeLlama. The baseline VR (0.18–0.29 without poisoning) could partly reflect memorized vulnerability patterns rather than in-context influence."
     },
     {
       "flag": "Single-run results without variance reporting",
       "detail": "All results appear to be from single experimental runs. Temperature=0 reduces but does not eliminate non-determinism, and LLM outputs can vary across API calls even at temperature 0. No variance or stability analysis is provided."
     },
     {
       "flag": "LLM-generated queries introduce noise",
       "detail": "§6.5 reports that LLM-generated queries achieve only 86% accuracy based on manual review of 100 samples per language. This means ~14% of queries may not accurately represent the code's functionality, yet this error is not propagated into the analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Foundational study assessing security of LLM-generated code (Copilot), directly relevant to understanding AI code generation security risks."
     },
     {
       "title": "How secure is code generated by ChatGPT?",
       "authors": ["Raphaël Khoury", "Anderson R Avila", "Jacob Brunelle", "Baba Mamadou Camara"],
       "year": 2023,
       "relevance": "Evaluates security of ChatGPT-generated code, establishing baseline vulnerability rates in LLM code generation."
     },
     {
       "title": "How secure is AI-generated code: a large-scale comparison of large language models",
       "authors": ["Norbert Tihanyi", "Tamas Bisztray", "Mohamed Amine Ferrag", "Ridhi Jain", "Lucas C Cordeiro"],
       "year": 2025,
       "relevance": "Large-scale comparison of LLM code security, providing context for vulnerability rates across different models."
     },
     {
       "title": "PoisonedRAG: Knowledge poisoning attacks to retrieval-augmented generation of large language models",
       "authors": ["Wei Zou", "Runpeng Geng", "Binghui Wang", "Jinyuan Jia"],
       "year": 2024,
       "arxiv_id": "2402.07867",
       "relevance": "Proposes knowledge base poisoning attacks on RAG systems for general LLM tasks; this paper extends the concept to code generation security."
     },
     {
       "title": "Poisoning web-scale training datasets is practical",
       "authors": ["Nicholas Carlini", "Matthew Jagielski", "Christopher A Choquette-Choo"],
       "year": 2024,
       "relevance": "Demonstrates practical feasibility of poisoning public data sources, supporting the threat model of RACG knowledge base poisoning."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Describes CodeLlama, a code-specialized LLM used as a primary subject in the study and shown to be most susceptible to poisoning."
     },
     {
       "title": "Retrieval augmented code generation and summarization",
       "authors": ["Md Rizwan Parvez", "Wasi Uddin Ahmad", "Saikat Chakraborty", "Baishakhi Ray", "Kai-Wei Chang"],
       "year": 2021,
       "arxiv_id": "2108.11601",
       "relevance": "Early work on retrieval-augmented code generation, establishing the RACG paradigm that this paper's security analysis targets."
     },
     {
       "title": "Using AI assistants in software development: A qualitative study on security practices and concerns",
       "authors": ["Jan H Klemmer", "Stefan Albert Horstmann", "Nikhil Patnaik"],
       "year": 2024,
       "relevance": "Qualitative study on developer security practices with AI assistants, complementary to this paper's quantitative vulnerability analysis."
     },
     {
       "title": "RMCBench: Benchmarking large language models' resistance to malicious code",
       "authors": ["Jiachi Chen", "Qingyuan Zhong", "Yanlin Wang"],
       "year": 2024,
       "relevance": "Benchmarks LLMs' ability to resist generating malicious code, directly relevant to understanding model susceptibility to poisoned examples."
     },
     {
       "title": "CodeRAG-Bench: Can retrieval augment code generation?",
       "authors": ["Zora Zhiruo Wang", "Akari Asai", "Xinyan Velocity Yu"],
       "year": 2024,
       "arxiv_id": "2406.14497",
       "relevance": "Benchmark for retrieval-augmented code generation, establishing evaluation methodology for RACG systems."
     },
     {
       "title": "ReposVul: A repository-level high-quality vulnerability dataset",
       "authors": ["Xinchen Wang", "Ruida Hu", "Cuiyun Gao"],
       "year": 2024,
       "relevance": "The primary dataset used in this study, providing paired vulnerable and secure code versions across multiple languages."
     },
     {
       "title": "Vul-RAG: Enhancing LLM-based vulnerability detection via knowledge-level RAG",
       "authors": ["Xueying Du", "Geng Zheng", "Kaixin Wang"],
       "year": 2024,
       "arxiv_id": "2406.11147",
       "relevance": "Uses RAG for vulnerability detection with an extraction-detection pipeline similar to this paper's LLM judge approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable insights for anyone building RACG systems (e.g., prefer second-most-similar retrieval, monitor for specific CWEs), but no tool or defense is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The general concept that poisoned inputs lead to poisoned outputs is intuitive; the specific magnitudes (48% from a single sample) are notable but not paradigm-shifting."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a concrete, novel attack vector on RACG code generation systems where a single malicious code example can compromise nearly half of generated code."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Raises concerns about RAG security but does not directly call out any product or company, and findings are presented in a measured academic tone."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tools are released. The attack and evaluation pipeline are described but not made available."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates GPT-4o (OpenAI) and other known models, but authors are from NUDT and the paper itself is not from a high-profile lab."
     }
   }
 }