ai-research-survey

scan-v4.json (32475B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Exploring the Security Threats of Knowledge Base Poisoning in Retrieval-Augmented Code Generation",
     "authors": [
       "Bo Lin",
       "Shangwen Wang",
       "Liqian Chen",
       "Xiaoguang Mao"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.03233",
     "doi": "10.48550/arXiv.2502.03233"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claim of '48% of the generated code' being vulnerable from a single poisoned sample is supported by Table 4 (CodeLlama + JINA: 0.29→0.48). The 6.5% increase from one-shot to three-shot matches Table 6 aggregated results.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The study uses controlled experimental manipulation: systematically varying the number of poisoned examples while holding other factors constant. The causal claim that poisoning 'compromises' security is supported by this controlled design.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 explicitly bounds generalization: the four languages 'may not fully represent real-world development scenarios' and account for only '42.7% of the total activity.' Findings are presented per-model and per-retriever rather than as universal claims.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6.3 provides an alternative explanation for JINA vs BM25 differences (retrieval effectiveness, validated with MRR/SR@k metrics). §6.5 discusses query generation accuracy (86%) as a potential confound. §5.1.1 discusses inherent LLM vulnerability generation even without poisoning.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "VR is measured by an LLM judge with 77–81% accuracy (§6.2), meaning ~20% of labels are incorrect. The paper does not discuss the gap between LLM-judge-detected vulnerability and actual exploitable security risk. The proxy (LLM judge classification) is treated as equivalent to the outcome (real-world security threat).",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 'Threats to Validity' is a dedicated subsection with substantive discussion of query generation accuracy, programming language coverage, and their implications.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 discusses study-specific threats: DeepSeek-V2.5 may produce inaccurate queries (mitigated by manual review showing 86% accuracy), and the four programming languages account for only 42.7% of GitHub activity.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While §6.5 notes that 'the selected languages may not fully represent real-world development scenarios,' the paper does not explicitly state what the results do NOT show—e.g., no statement about applicability to different retriever types, real-world deployment settings, or non-function-level poisoning.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text. Whether the work is funded or unfunded is unknown.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with the National University of Defense Technology. They evaluate third-party LLMs (GPT-4o, Llama-3, CodeLlama, DeepSeek-Coder), not their own products.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding from outcomes cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "RACG, the threat model (attacker goals, capabilities, two scenarios), and all three metrics (VR, Similarity via CrystalBLEU, VRRC) are defined precisely in Sections 3 and 4.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are bullet-pointed in Section 1: first comprehensive RACG security study, large-scale 16 sub-scenario experimentation, and practical factor-analysis insights.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 covers LLMs, RAG, RACG, and existing attacks; Section 6.4 explicitly distinguishes this work from PoisonedRAG by contrasting functional vs. security poisoning goals.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No repository URL or code archive is provided. The paper references third-party implementations (BM25 from GitHub, JINA from Huggingface) but does not release its own experimental code or scripts.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The base dataset ReposVul is publicly available, but the authors' constructed dataset—including LLM-generated queries, poisoned knowledge bases, and generated code outputs—is not released.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "§4.6 mentions 'a single A100-40G GPU server using the Ollama framework' with some model parameters, but no requirements.txt, Dockerfile, or detailed dependency list is provided. Not sufficient to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No reproduction instructions, README, or runnable scripts are provided. A researcher would need to reverse-engineer the full pipeline from the paper text.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 4–9 are point estimates (e.g., VR = 0.48). No confidence intervals, error bars, or uncertainty measures are reported anywhere in the paper.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes numerous comparative claims (e.g., '6.5% more vulnerabilities from one-shot to three-shot', 'CodeLlama exhibits the highest susceptibility') without any statistical significance tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported with baseline context throughout. For example, §5.1.1 states 'CodeLlama's VR increases from 0.29 to 0.48' (a 19pp increase), and §5.1.2 reports '6.5% (0.46→0.49)' providing both absolute and relative context.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The dataset contains 12,053 instances from ReposVul. No justification for why this size is sufficient, and no power analysis is discussed. The manual inspection sample sizes (81–95) are justified via confidence level calculation (§6.2).",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or spread measures are reported. §4.6 sets temperature to 0 'to reduce non-determinism' but does not report variance across runs or verify determinism.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "All experiments include a zero-poisoning baseline (poisoning number 0 in Table 4, proportion 0 in Table 5), allowing comparison of poisoned vs. unpoisoned performance.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The four LLMs (GPT-4o, Llama-3-8B, CodeLlama-13B, DeepSeek-Coder-V2-16B) are contemporary as of the study period, with models selected from the LLM Safety Leaderboard as of October 2024 (§4.4.1).",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The study systematically varies poisoning quantity (0–9 samples, 0–100% proportion), number of shots (1 vs 3), retriever type (JINA vs BM25), programming language, similarity range, and CWE type—each serving as an ablation of a different factor.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Three metrics are used: Vulnerability Rate (VR), CrystalBLEU similarity, and Vulnerability Rate in Retrieved Code (VRRC), defined in §4.5.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "§6.2 describes manual inspection where 'Two authors independently evaluated the samples through manual review' on 360 generated code samples (95+81+93+91) to validate the LLM judge's accuracy.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "No explicit separation of development and test sets is described. The same dataset appears to be used for all experiments without a held-out split.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Extensive breakdowns are provided: by programming language (Table 7), CWE type (Tables 9, 12), similarity range (Table 8), retriever, scenario, and LLM.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper discusses where poisoning is less effective: BM25 shows smaller VR increases than JINA (§5.1.1), CWE-434 has lowest VR (§5.2.3), and Scenario II is much harder for attackers (§5.1.1).",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Several negative/null results: poisoning has 'negligible' impact on code functionality (similarity metric barely changes), BM25 retriever shows minimal vulnerability increase, and low-similarity examples have 'relatively minor impact on vulnerability likelihood' (§5.2.2).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Models are identified as 'GPT-4o', 'Llama-3-8B', 'CodeLLAMA-13B', 'DeepSeek-Coder-V2-16B'. Parameter counts are given for open-source models, but no snapshot dates or API versions are specified. 'GPT-4o' is a marketing name without a snapshot date.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Appendix provides prompt templates for query generation (Prompt 1), vulnerability extraction (Prompt 2), and security assessment (Prompt 3), but all contain placeholders ({LANGUAGE}, {FUNCTION}, {DIFF}, etc.). The actual code generation prompt used to instruct LLMs in the RACG pipeline is not provided at all.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "§4.6 reports: 'temperature of 0, top-p value of 0.95, a max_new_tokens setting of 4096, and a context window of 8192, keeping other parameters at default values.'",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The system is a standard RAG pipeline (retriever + LLM generation) without agents, tools, or feedback loops.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "§4.1 documents filtering: functions shorter than three lines and names containing 'test' were removed. §4.2 describes knowledge base construction and poisoning process in detail. Table 2 provides final statistics (12,053 instances, 236 CWEs).",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw data (generated code, judge outputs, poisoned knowledge bases) is made available. Only aggregated results in tables are presented.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "§4.1 describes dataset selection in detail: 12 candidate datasets evaluated against 4 criteria (Table 1), ReposVul selected as it satisfies all requirements. Filtering criteria and resulting statistics (Table 2) are documented.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data comes from the ReposVul dataset, a standard publicly available vulnerability dataset.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline is documented: ReposVul → filtering (§4.1) → query generation (§4.1) → knowledge base construction (§4.2) → poisoning (§4.2.1/§4.2.2) → code generation → result validation (§4.3). Each stage has explicit criteria.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the four LLMs used. This matters because ReposVul vulnerabilities may be in GPT-4o's or Llama-3's training data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether ReposVul code examples appeared in any LLM's training data. The models may have memorized the secure or vulnerable code patterns, confounding the results.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "ReposVul is built from public repositories available before model training cutoffs. No discussion of whether models have seen these code snippets during training, which could affect both VR baseline and poisoning effectiveness.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study. Manual validation of the LLM judge by the authors is not a human subjects study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study analyzes code generation and vulnerability propagation, not human behavior.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference costs, API costs, or per-example latency are reported despite using the GPT-4o API and running 16 sub-scenarios across 12,053 instances.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is mentioned ('A100-40G GPU server') but no total compute budget (GPU hours, API spend, wall-clock time for experiments) is reported.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "Temperature is set to 0 'to reduce non-determinism' (§4.6) but no multi-seed or multi-run experiments are conducted. Determinism is assumed, not verified.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is never explicitly stated. Results appear to be from a single run per configuration.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "Hyperparameters (temperature=0, top-p=0.95, etc.) appear to be fixed choices with no search or justification for these values beyond 'keeping other parameters at default values.'",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports results across all configurations rather than selecting a best one. For RQ2 analysis, moderate poisoning quantities are chosen with justification: 'LLM-generated code exhibits similar patterns across all metrics, regardless of poisoning quantity' (§5.2).",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes many comparisons across 4 LLMs × 2 retrievers × 2 scenarios × multiple poisoning levels without any multiple comparison correction or even significance testing.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The authors evaluate third-party LLMs (GPT-4o, Llama-3, CodeLlama, DeepSeek-Coder) rather than their own system. The poisoning methodology is the contribution, and they report results across all models without cherry-picking favorable configurations.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "Models of vastly different sizes are compared (8B vs GPT-4o) without discussing compute differences. No performance-per-compute analysis is provided.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The VR metric relies on an LLM judge with 77–81% accuracy. The paper validates the judge (§6.2) but does not discuss whether detecting vulnerability patterns via LLM actually measures real-world security risk—e.g., whether flagged vulnerabilities are exploitable.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The RAG pipeline is simple retrieval + generation without scaffolding.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of temporal leakage. ReposVul contains vulnerabilities from public repositories that predate the LLMs' training data, meaning models may have already seen these code patterns.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether providing vulnerable code as in-context examples leaks information differently than real-world RAG settings (e.g., whether format/structure of injected examples is distinguishable).",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether training data and test examples share structural similarities (same repositories, same authors, duplicate patterns) that could inflate results.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention methods (canary strings, membership inference, decontamination) are applied.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "A single poisoned sample in the knowledge base can cause ~48% of CodeLlama-generated code to be vulnerable when using the JINA retriever (Scenario I).",
       "evidence": "Table 4: CodeLlama/JINA VR rises from 0.29 (baseline) to 0.48 at poisoning=1, representing a 19pp increase from 0.008% knowledge base contamination.",
       "supported": "strong"
     },
     {
       "claim": "Dense retrievers (JINA) amplify poisoning risk far more than sparse retrievers (BM25) due to superior retrieval of semantically relevant—and thus targeted—vulnerable examples.",
       "evidence": "Table 4: JINA VRRC=0.41 vs BM25 VRRC=0.06 at poisoning=5; validated by retriever effectiveness metrics in Table 11 (JINA MRR=0.85 vs BM25 MRR=0.20).",
       "supported": "strong"
     },
     {
       "claim": "Providing more few-shot examples in RACG increases vulnerability risk by approximately 6.5% (one-shot to three-shot with JINA retriever in Scenario I).",
       "evidence": "Table 6: Aggregated VR for JINA-I increases from 0.46 to 0.49 (3pp), with VRRC rising from 0.41 to 0.44; the 6.5% figure refers to the absolute VR change for JINA-I.",
       "supported": "moderate"
     },
     {
       "claim": "Code-specialized LLMs (CodeLlama) are more susceptible to vulnerability propagation than general-purpose LLMs of similar scale.",
       "evidence": "Throughout Tables 4–6, CodeLlama shows the highest VR across all configurations; however, CodeLlama (13B) is larger than Llama-3 (8B), confounding the architecture vs. size explanation.",
       "supported": "moderate"
     },
     {
       "claim": "Example-query similarity above 60% sharply increases vulnerability risk, while similarity below 60% has minimal impact.",
       "evidence": "Table 8: In Scenario I, aggregated VR increases from 0.35 ([40,60)) to 0.42 ([60,80)) to 0.53 ([80,100]), with VRRC showing the same threshold effect.",
       "supported": "strong"
     },
     {
       "claim": "CWE-352 (Cross-Site Request Forgery) exhibits the consistently highest vulnerability propagation rate (~0.79 in Scenario I, ~0.78 in Scenario II) among MITRE Top-10 weaknesses.",
       "evidence": "Table 9 confirms CWE-352 averages across all LLMs are 0.79 and 0.78 in Scenarios I and II respectively, substantially above other CWEs.",
       "supported": "strong"
     },
     {
       "claim": "The LLM-as-judge achieves 77–81% accuracy for vulnerability detection, making it an adequate substitute for manual review.",
       "evidence": "Table 10 reports accuracy of 0.76–0.84 (manual) and 0.79–0.82 (automated) across four languages; the paper claims this is 'commendable' without discussing how measurement error propagates to VR estimates.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "Knowledge base poisoning in RACG systems poses severe security risks: a single malicious code sample (0.008% of a 12,053-item knowledge base) can cause 48% of CodeLlama-generated code to contain vulnerabilities when using a dense semantic retriever. The attack is substantially more difficult without knowing developer queries (Scenario II), requiring 100% knowledge base poisoning to match Scenario I's effectiveness at 0.075% contamination. Dense retrievers create a security paradox—higher retrieval quality increases both code generation quality and vulnerability propagation rate. Vulnerability risk scales with example-query similarity (sharp threshold at 60%) and with the number of few-shot examples provided. CWE-352 (CSRF) is the most contagious vulnerability type, while Java and Python show more natural resistance than C and C++ due to built-in safety abstractions.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (JINA vs BM25, one-shot vs three-shot, language differences, LLM differences) are presented as point estimates without any significance tests or confidence intervals, making it impossible to distinguish real effects from noise."
     },
     {
       "flag": "LLM judge measurement error unaccounted for",
       "detail": "The LLM judge has 77–81% accuracy, meaning up to 23% of vulnerability judgments are wrong. Main findings (e.g., '48% VR') are presented without correcting for or propagating this error, potentially inflating or deflating reported rates."
     },
     {
       "flag": "GPT-4o version not pinned",
       "detail": "GPT-4o is referenced without a snapshot date or API version, making results for this model non-reproducible as the model is continuously updated."
     },
     {
       "flag": "Training data contamination ignored",
       "detail": "ReposVul draws from public GitHub repositories with real CVEs that are almost certainly in LLM training data; the possibility that LLMs have memorized these vulnerability patterns is never discussed."
     },
     {
       "flag": "Code LLM vs general LLM confound",
       "detail": "CodeLlama (13B) is compared to Llama-3 (8B); the claimed explanation that code specialization causes higher VR is confounded by model size—larger models generally follow examples more closely."
     },
     {
       "flag": "No code or data released",
       "detail": "The paper is entirely non-reproducible: no experimental code, no processed dataset, no generated outputs, and no vulnerability judge implementation are released despite the appendix providing prompts."
     }
   ],
   "cited_papers": [
     {
       "title": "PoisonedRAG: Knowledge Poisoning Attacks to Retrieval-Augmented Generation of Large Language Models",
       "relevance": "Closest prior work; this paper distinguishes RACG security poisoning from RAG functional poisoning studied in PoisonedRAG"
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "relevance": "Seminal study on LLM-generated code security without RAG; provides baseline for understanding inherent LLM security limitations"
     },
     {
       "title": "How Secure is Code Generated by ChatGPT?",
       "relevance": "Comparative study on LLM code security; this paper extends those findings to the RACG threat model"
     },
     {
       "title": "How Secure is AI-Generated Code: A Large-Scale Comparison of Large Language Models",
       "relevance": "Large-scale empirical benchmark of LLM code security; establishes baseline vulnerability rates for comparison"
     },
     {
       "title": "ReposVul: A Repository-Level High-Quality Vulnerability Dataset",
       "relevance": "Primary dataset used in this study; understanding its construction is essential to evaluating experimental validity"
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "relevance": "One of the four LLMs evaluated; code-specialized model showing highest vulnerability susceptibility"
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "relevance": "Foundational RACG paper establishing the paradigm whose security this study examines"
     },
     {
       "title": "CodeRAG-Bench: Can Retrieval Augment Code Generation?",
       "relevance": "Benchmarks RACG effectiveness; this paper adds security as a missing evaluation dimension"
     },
     {
       "title": "Poisoning Web-Scale Training Datasets is Practical",
       "relevance": "Demonstrates feasibility of the threat model's assumption that attackers can poison public code repositories"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable insights for anyone building RACG systems (e.g., prefer second-most-similar retrieval, monitor for specific CWEs), but no tool or defense is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The general concept that poisoned inputs lead to poisoned outputs is intuitive; the specific magnitudes (48% from a single sample) are notable but not paradigm-shifting."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a concrete, novel attack vector on RACG code generation systems where a single malicious code example can compromise nearly half of generated code."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Raises concerns about RAG security but does not directly call out any product or company, and findings are presented in a measured academic tone."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tools are released. The attack and evaluation pipeline are described but not made available."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates GPT-4o (OpenAI) and other known models, but authors are from NUDT and the paper itself is not from a high-profile lab."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }