ai-research-survey

scan-v5.json (26522B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Exploring the Security Threats of Knowledge Base Poisoning in Retrieval-Augmented Code Generation",
     "authors": [
       "Bo Lin",
       "Shangwen Wang",
       "Liqian Chen",
       "Xiaoguang Mao"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.03233",
     "doi": "10.48550/arXiv.2502.03233"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The 48% VR claim for a single poisoned sample with CodeLlama+JINA is directly supported in Table 4 (VR=0.48 at poisoning=1). The ~36% VR at 20% poisoning in Scenario II is confirmed in Table 5 (CodeLlama JINA VR=0.36 at proportion=0.2).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Controlled experiments systematically vary poisoning quantity from 0 to 9 samples and 0% to 100% proportion with unpoisoned baselines, providing adequate grounds for causal inference in this systems-evaluation context.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper draws broad conclusions about RACG security in general but tests only 4 LLMs, 4 programming languages, one vulnerability dataset (ReposVul), and two retrievers in a controlled lab setting. Conclusions like 'code LLMs are more prone to generate vulnerable code' extend beyond the tested scope without explicit bounding.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider that the high baseline VR (26-29% without any poisoning) may indicate the LLM judge is inflating results or that LLMs already struggle with secure code generation independently. The possibility that LLMs recognize and reproduce training-set vulnerable patterns (contamination) is not discussed.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly validates its LLM-as-judge proxy via manual inspection of ~360 samples, reporting 77-84% accuracy, and acknowledges this as an approximation before using it as the primary evaluation metric.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 'Threats to Validity' is a dedicated section, not merely a sentence in the conclusion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The section discusses specific threats: LLM-generated query accuracy (86% verified via manual review of 100 queries per language) and the limited language coverage (4 languages = 42.7% of GitHub pull request activity in Q1 2024).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The threats section acknowledges limitations but does not explicitly state what the results do NOT show (e.g., no statement that results don't generalize to non-JINA/BM25 retrievers, or don't address real-world attack feasibility beyond the lab setup).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper, including acknowledgments.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with National University of Defense Technology, clearly stated under each author name.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interests declaration is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "RACG is formally defined with a workflow diagram (Figure 1), Vulnerability Rate (VR), VRRC, and the two attack scenarios are formally defined with mathematical notation in Section 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are listed: first comprehensive study of RACG security risks, large-scale experimentation across 16 sub-scenarios, and practical insights on influencing factors.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly compares RACG poisoning with RAG poisoning (PoisonedRAG), and Sections 2.1-2.4 situate the work relative to LLMs, RAG, RACG, and existing attack types, showing how this work differs from prior code security studies.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No link to the experimental codebase is provided. The paper only references external tools (a public BM25 GitHub repo, HuggingFace for JINA embeddings) but does not release its own experimental framework.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The primary dataset ReposVul (Wang et al., ICSE 2024) is a publicly available repository-level vulnerability dataset, not a custom artifact.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions 'single A100-40G GPU server using the Ollama framework' but provides no requirements.txt, Dockerfile, or complete dependency specification with version numbers.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided. The pipeline is described conceptually across multiple sections but not with actionable commands or scripts to follow.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any metric (VR, Similarity, VRRC). All reported results are single-point estimates.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims are made throughout (JINA vs BM25, code LLMs vs general LLMs, one-shot vs three-shot) without any statistical significance tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute effect sizes are reported throughout (e.g., CodeLlama VR increases from 0.29 to 0.48 with one poisoned sample, a 0.19 absolute increase; 6.5% VR rise from one-shot to three-shot). These provide meaningful scale context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 12,053 instances from ReposVul are used without formal power analysis or justification for why this size is sufficient for the comparative claims made across 16 sub-scenarios.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or spread is reported across experimental runs. Temperature=0 reduces but does not eliminate non-determinism, and residual variance is not quantified.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Unpoisoned knowledge base (poisoning=0) is consistently included as baseline in Tables 4, 5, and 6 across all LLMs and retrievers.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Models include GPT-4o and state-of-the-art open-source models (Llama-3-8B, DeepSeek-Coder-V2-16B) selected from the LLM Safety Leaderboard as of October 2024.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "RQ2 systematically ablates poisoning quantity, number of few-shot examples, programming language, example-query similarity range, and CWE vulnerability type as independent sub-questions.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Three metrics are used: Vulnerability Rate (VR), Similarity (CrystalBLEU), and Vulnerability Rate in Retrieved Code (VRRC), capturing different aspects of the attack impact.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Two authors independently reviewed ~360 generated code samples (95+81+93+91 across four languages) from GPT-4o outputs to validate the LLM judge, constituting human evaluation of system outputs.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is not a traditional prediction task; the study evaluates vulnerability propagation under controlled poisoning conditions rather than generalization to unseen test data.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by programming language (Table 7), CWE vulnerability type (Tables 9 and 12 for Top-25), example-query similarity range (Table 8), and consistently by retriever and individual LLM throughout.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper notes CWE-434 has the lowest VR and BM25 has lower susceptibility, but does not present specific failure cases or examples of queries where poisoning did not propagate.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that BM25 retriever is substantially less susceptible to poisoning (VRRC 0.06 vs 0.41 for JINA with 5 samples), and that Scenario II (hidden intent) requires orders of magnitude more poisoning to achieve equivalent VRRC.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "GPT-4o is cited without a snapshot date or version string; open-source models have parameter counts but no commit hashes or Hugging Face revision identifiers.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendices A, B, and C provide the complete prompt templates for query generation, vulnerability pattern extraction, and security assessment respectively.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.6 reports temperature=0, top-p=0.95, max_new_tokens=4096, and context window=8192.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The full RACG pipeline is described in detail: retriever mechanics (BM25 token-frequency vs JINA cosine similarity), knowledge base construction, poisoning injection methodology (clustering-based for Scenario II), and the two-step LLM judge evaluation pipeline.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Filtering criteria are documented (remove functions <3 lines of implementation, remove names containing 'test'), and the query generation procedure for functions lacking comments is described with the specific LLM (DeepSeek-V2.5) used.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The generated code samples, LLM judge outputs, and intermediate experimental data are not released in any repository.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 systematically evaluates 12 candidate vulnerability datasets against four criteria, documenting the selection rationale for ReposVul. Post-filtering statistics are provided in Table 2.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Standard publicly available benchmark dataset (ReposVul) was used; no participant recruitment was involved.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from ReposVul dataset → filtering → query generation → knowledge base construction → poisoning injection → retrieval → LLM generation → LLM judge evaluation is documented across Sections 4.1-4.3.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff is stated for any of the four LLMs tested. This is critical since ReposVul is built from GitHub repositories that likely overlap with LLM training corpora.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss potential overlap between LLM training data (GitHub code) and the ReposVul test dataset (also GitHub-sourced). The high baseline VR (26-29% unpoisoned) may partly reflect LLMs already having learned vulnerable patterns from training.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "ReposVul contains real-world GitHub code that was likely in the pretraining corpora of all tested LLMs (CodeLlama trained on 500B code tokens). Whether LLMs recognize specific vulnerability patterns from training vs. from the retrieved examples is not addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; IRB not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No API costs for GPT-4o (used for ~12,053 generations plus judge calls) or GPU hours for the three local models are reported.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Only the hardware (single A100-40G GPU) is mentioned; total compute hours for 16 sub-scenarios × 12,053 instances of generation plus evaluation is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "A single poisoned code sample with the JINA retriever can render approximately 48% of CodeLlama-generated code vulnerable",
       "evidence": "Table 4 shows CodeLlama VR increases from 0.29 (no poisoning) to 0.48 (1 poisoned sample) with JINA retriever in Scenario I",
       "supported": "strong"
     },
     {
       "claim": "Code-specialized LLMs (CodeLlama) are more susceptible to knowledge base poisoning than general-purpose LLMs",
       "evidence": "Across all poisoning quantities and retrievers, CodeLlama consistently shows the highest VR (e.g., 0.53 at JINA, 9 samples) vs Llama-3 (0.37); attributed to code-focused training on larger datasets including vulnerable patterns",
       "supported": "moderate"
     },
     {
       "claim": "Dense retrievers (JINA) propagate vulnerabilities far more effectively than sparse retrievers (BM25)",
       "evidence": "Table 4 shows JINA achieving VRRC=0.41 vs BM25=0.06 with 5 poisoned samples; confirmed by Table 11 showing JINA MRR=0.85 vs BM25 MRR=0.20",
       "supported": "strong"
     },
     {
       "claim": "Increasing few-shot examples from one-shot to three-shot raises vulnerability rate by ~6.5% with JINA retriever",
       "evidence": "Table 6 shows aggregated VR increasing from 0.46 to 0.49 (6.5%) in Scenario I with JINA across all LLMs; VRRC rises from 0.41 to 0.44",
       "supported": "moderate"
     },
     {
       "claim": "Example-query similarity above 60% significantly increases vulnerability risk; similarity below 60% has minor impact",
       "evidence": "Table 8 shows VR rising steeply from 0.35 ([40,60) range) to 0.53 ([80,100] range) in Scenario I, while lower similarity ranges show only modest increases",
       "supported": "moderate"
     },
     {
       "claim": "CWE-352 (Cross-Site Request Forgery) consistently shows the highest vulnerability propagation rate (~0.79) among MITRE Top-10",
       "evidence": "Table 9 reports CWE-352 average VR of 0.79 in Scenario I and 0.78 in Scenario II across all four LLMs",
       "supported": "strong"
     },
     {
       "claim": "Knowledge base poisoning does not significantly degrade functional performance (code similarity)",
       "evidence": "Tables 4-5 show Similarity (CrystalBLEU) changes are minimal across poisoning levels (e.g., DS-Coder JINA: 0.76 baseline vs 0.78 at 9 poisoned samples)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "Knowledge base poisoning in RACG systems is a realistic, low-effort attack: a single poisoned sample injected into a 12,053-item knowledge base (0.008% poisoning rate) can render 48% of CodeLlama-generated code vulnerable when using a dense retriever. The attack is stealthy because it does not degrade functional performance (CrystalBLEU scores remain stable). Dense retrievers like JINA amplify the attack significantly (VRRC=0.41) compared to sparse retrievers like BM25 (VRRC=0.06) due to superior semantic retrieval. In the blind attack scenario (hidden programmer intent), achieving comparable impact requires injecting ~9,642 samples—orders of magnitude more effort, making it far more detectable.",
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All comparative claims (JINA vs BM25, code LLMs vs general LLMs, one-shot vs three-shot) are made without confidence intervals, significance tests, or variance across runs. Single-point estimates are presented as definitive findings."
     },
     {
       "flag": "Training data contamination unaddressed",
       "detail": "ReposVul is sourced from GitHub repositories; all tested LLMs (especially CodeLlama, trained on 500B code tokens from GitHub) likely encountered these vulnerable patterns during pretraining. The high baseline VR (26-29% with no poisoning) may reflect this contamination rather than inherent LLM vulnerability, but this confound is never discussed."
     },
     {
       "flag": "LLM judge reliability ceiling",
       "detail": "The vulnerability judge achieves only 77-84% accuracy. At 48% reported VR, a 20% false-positive rate would shift the true VR substantially. Error propagation from judge uncertainty is not quantified in reported results."
     },
     {
       "flag": "GPT-4o version unspecified",
       "detail": "GPT-4o is referenced only by marketing name without a snapshot date or API version string, making exact replication of results impossible for the closed-source model."
     },
     {
       "flag": "No replication package",
       "detail": "No experimental code, generated outputs, or intermediate data are released. Replication requires re-implementing the full pipeline (clustering-based poisoning, LLM judge, RACG scaffolding) from scratch."
     }
   ],
   "cited_papers": [
     {
       "title": "PoisonedRAG: Knowledge Poisoning Attacks to Retrieval-Augmented Generation of Large Language Models",
       "relevance": "Most directly related prior work on RAG knowledge base poisoning; this paper extends the attack surface to code generation security specifically"
     },
     {
       "title": "ReposVul: A Repository-Level High-Quality Vulnerability Dataset",
       "relevance": "Primary dataset used in all experiments; foundation of the knowledge base construction and poisoning scenarios"
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "relevance": "Foundational prior work on security of LLM-generated code without RAG; establishes the baseline security problem this work extends"
     },
     {
       "title": "How Secure is AI-Generated Code: A Large-Scale Comparison of Large Language Models",
       "relevance": "Recent large-scale baseline establishing LLM-generated code security rates across LLMs; direct comparison point for this study's unpoisoned baseline"
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "relevance": "Foundational RACG work that established the paradigm this paper's threat model targets"
     },
     {
       "title": "Vul-RAG: Enhancing LLM-based Vulnerability Detection via Knowledge-Level RAG",
       "relevance": "Shares the two-step vulnerability extraction-detection pipeline adopted in this paper's LLM judge design"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Justification for the LLM-as-judge evaluation methodology used as the primary vulnerability detection approach"
     },
     {
       "title": "Poisoning Web-Scale Training Datasets is Practical",
       "relevance": "Provides the realistic threat model foundation for knowledge base poisoning via public repository injection"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly attacks the security of production RACG systems (GitHub Copilot, Cursor, etc.) that millions of developers use, with a low-effort attack requiring only 1 poisoned sample."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that a 0.008% knowledge base poisoning rate can compromise 48% of generated code is a striking quantitative result; the stealthiness (no functional degradation) is a non-obvious finding."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a realistic, scalable attack vector against widely deployed AI coding tools with potential for supply-chain security compromise via publicly accessible code repositories."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Security attack research targeting popular AI coding tools has inherent controversy; the framing as 'first comprehensive study' creates urgency around a widely trusted technology."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Reproducing the attack requires setting up 4 LLMs, custom retrieval infrastructure, and the full experimental pipeline; no demo or code is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "National University of Defense Technology is recognized but not a top-tier AI lab; no famous product or model family is introduced."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }