ai-research-survey

scan.json (32295B)

---

 {
   "paper": {
     "title": "Give LLMs a Security Course: Securing Retrieval-Augmented Code Generation via Knowledge Injection",
     "authors": ["Bo Lin", "Shangwen Wang", "Yihao Qin", "Liqian Chen", "Xiaoguang Mao"],
     "year": 2025,
     "venue": "Conference on Computer and Communications Security",
     "arxiv_id": "2504.16429",
     "doi": "10.1145/3719027.3765049"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeGuarder, a security-hardening framework for RACG systems, injects security knowledge (root causes, fixing patterns) derived from CVE databases into code generation prompts. On the CyberSecEval benchmark across 4 LLMs and 4 languages, it improves security rates by 20.12% in standard RACG, 31.53% and 21.91% under two poisoning scenarios, without degrading functional correctness (slightly improves CodeBLEU and pass@1). It also outperforms SafeCoder by 9.80% in non-retrieval settings and demonstrates cross-language generalization even without target-language-specific security knowledge.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. No mention of releasing CodeGuarder's implementation."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmark CyberSecEval is publicly available, and the security knowledge base source data ReposVul is also public. However, the constructed security knowledge base S itself is not explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.6 mentions 'A100 GPU server using the Ollama' and basic model configurations (temperature=0, max_new_tokens=4096, context window=8192), but no requirements.txt, Dockerfile, or detailed dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or experiment scripts are provided or referenced."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results in Tables 4-9 are reported as point estimates without confidence intervals or error bars. The only variance data (0.38% max deviation across 5 runs) appears in the threats to validity section for a single model."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims that CodeGuarder 'improves' or 'outperforms' baselines are based solely on comparing raw numbers without any hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout (e.g., '20.12% improvement' from 60.84 to 73.08 in Table 4). Relative and absolute improvements are provided for all comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of CyberSecEval (1,916 instances) as benchmark is justified by its comprehensiveness but no power analysis or statistical justification for sample size adequacy is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results are single-run. Section 7.5 reports 5 runs of DS-V3 with max 0.38% SR deviation, but this is only for one model under one scenario and appears only in the threats to validity discussion, not in main results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "RQ1-RQ2 compare with/without CodeGuarder. RQ3 compares against three state-of-the-art baselines: Sven, SafeCoder, and CoSec (Table 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are recent: Sven (CCS 2023), SafeCoder (2024), CoSec (ISSTA 2024). These represent the state of the art in secure code generation hardening."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 presents an ablation study with two variants: CodeGuarder-QD (without query decomposition) and CodeGuarder-KRF (without knowledge re-ranking and filtering). Results in Table 11 show QD contributes ~10.1% SR improvement."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Security Rate (SR) and CodeBLEU Similarity (Sim) are used throughout. Additionally, Pass@1 and Pass@5 on MBPP and HumanEval are reported in Table 9 for functional correctness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All security evaluation is automated via CyberSecEval's Insecure Code Detector. No human evaluation of code security or functionality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The security knowledge base S is constructed from ReposVul while evaluation is on CyberSecEval, ensuring separation. Section 5.2 explicitly notes: 'Using the same dataset for both the vulnerable code and the security knowledge base could lead to overlapping fix strategies.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (C, C++, Java, Python in Tables 4-6), by LLM (4 models), by CWE type (Table 10, MITRE Top-25), and by scenario (standard, poisoning I, poisoning II)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 discusses failure cases: CWE-79 (XSS) has only 4.76% prevention rate due to lack of JavaScript-specific knowledge. CWE-22 (path traversal) at 8.26% is also highlighted as challenging."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: CWE-79 prevention is near zero (4.76%), smaller models benefit less from CodeGuarder than larger models, and excessive knowledge injection degrades performance (Table 12 shows degradation at high k' and k values)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 20.12% improvement in standard RACG, 31.53% and 21.91% under poisoning scenarios are directly supported by average SR values in Tables 4, 5, and 6 respectively. The generalization claim of 75.54% SR across four languages matches Table 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim that CodeGuarder improves security is supported by controlled comparisons: same LLM, same benchmark, same scenario, with the only variable being CodeGuarder's security knowledge injection. The ablation study (Section 7.3) isolates component contributions through single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim CodeGuarder is 'a pivotal advancement towards building secure and trustworthy RACG systems' and 'the first security-hardening framework for RACG systems.' These broad claims extend beyond the tested setting of CyberSecEval with 4 LLMs. The title frames results as 'Securing Retrieval-Augmented Code Generation' generally, not bounded to the specific benchmark and models tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section addresses measurement validity and non-determinism but does not consider alternative explanations for the improvements, such as whether longer prompts (rather than security content specifically) might account for some gains, or whether the improvements reflect prompt engineering rather than security knowledge per se."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5 explicitly acknowledges that their security measurement 'may not perfectly capture the true security posture' and that 50 CWEs 'may not represent the complete universe of potential security flaws.' They also note Sim is a proxy for functionality and supplement with test-based evaluation (MBPP/HumanEval) in Section 7.1."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4o', 'DeepSeek-V3', 'CodeLlama-13B', and 'DeepSeek-Coder-V2-16B' without specific version strings or API snapshot dates. 'GPT-4o' without a snapshot date does not count as a specified version per the schema criteria."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Three prompt templates are provided in the Appendix: Prompt 1 (security knowledge extraction), Prompt 2 (query decomposition), and Prompt 3 (security-augmented code generation with a concrete worked example). While Prompts 1-2 use placeholders, Prompt 3 includes actual filled content showing the exact format sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 reports temperature=0, max_new_tokens=4096, context window=8192, and notes other parameters at defaults. Section 7.4 reports k'=2 and k=5 as retrieval hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-stage pipeline is described in detail: offline knowledge base construction (Section 4.1), query decomposition (Section 4.2.1), similarity-based retrieval (Section 4.2.2), re-ranking and filtering (Section 4.2.3), and security-augmented generation (Section 4.3). Figure 2 provides the workflow diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes construction of all three knowledge bases: S from ReposVul function-level vulnerability pairs, K from fixed code in ReposVul, V from CyberSecEval vulnerable code. Table 2 provides statistics. Section 4.1 details the LLM-based extraction pipeline for security knowledge."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5 'Threats to Validity' provides substantive discussion of two specific threats: validity of security measurement and reliability/LLM non-determinism."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5 discusses study-specific threats: (1) CyberSecEval's Insecure Code Detector covers only 50 CWEs and may miss vulnerabilities, (2) LLM non-determinism quantified via 5 independent DS-V3 runs showing 0.38% max deviation. Both are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no statements like 'our results do not demonstrate protection against novel attack types' or 'we do not claim security against runtime vulnerabilities.' The threats section discusses measurement limitations but not explicit scope boundaries on claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated code, per-instance security verdicts, retrieval results) is made available. Only aggregated results in tables are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes the construction of all three knowledge bases with sources, processing steps, and statistics (Table 2). ReposVul provides 12,053 function-level vulnerability pairs; CyberSecEval provides 1,916 instances across 50 CWE types."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (CyberSecEval) and datasets (ReposVul)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: CVE instances → LLM-based extraction of functionality/root cause/fixing pattern → security knowledge base S (Section 4.1). The knowledge base statistics are provided in Table 2 (e.g., 8,861 C security knowledge entries vs 12,053 source vulnerabilities, with explanation that some vulnerabilities have multiple root causes)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgments section is present but contains no funding information. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with College of Computer Science, National University of Defense Technology, Changsha, China, and the State Key Laboratory of Complex & Critical Software Environment."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence. The authors are from a national defense university, which could have institutional interests in security tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four LLMs evaluated (GPT-4o, DeepSeek-V3, CodeLlama-13B, DeepSeek-Coder-V2-16B). This is relevant since CyberSecEval benchmarks could be in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CyberSecEval test instances appeared in the training data of the evaluated LLMs. CyberSecEval was published in 2023; all models were trained after this date."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CyberSecEval was published in 2023 (arXiv:2312.04724). Models like GPT-4o and DeepSeek-V3 were trained after this date and may have seen the benchmark. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API costs reported. CodeGuarder adds multiple LLM calls (query decomposition, knowledge retrieval, re-ranking) on top of standard RACG, but the cost overhead is never quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.6 mentions 'A100 GPU server' but provides no total GPU hours, API spend, or training time for reproducing SafeCoder/Sven baselines or running the full evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.5 reports 5 runs for DS-V3 only under the standard scenario, showing 0.38% max deviation. This covers only one of four models and one of three scenarios, and is relegated to threats to validity rather than main results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Main results appear to be single-run. The 5-run analysis is mentioned only for DS-V3 in the threats to validity section, not as the basis for main experimental results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 and Table 12 transparently show the hyperparameter grid search over k' ∈ {1,2,3} and k ∈ {3,5,7,9} for two models (DS-V3 and CodeLlama-13B), with all configurations' SR values reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 explains the selection of k'=2, k=5: 'aiming for robust performance across different model scales while mitigating degradation.' All tested configurations are shown in Table 12."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 4 LLMs, 4+ languages, and 3 scenarios without any statistical tests or multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own system and compare against baselines. For SafeCoder they use the authors' trained model, and for Sven/CoSec they follow official implementations. However, they do not explicitly acknowledge or discuss the bias of evaluating their own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "CodeGuarder adds overhead of query decomposition, retrieval, and re-ranking/filtering on top of standard RACG. This additional compute cost is never compared against the performance gains. Baselines (Sven, SafeCoder, CoSec) have different computational profiles not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 justifies CyberSecEval's construct validity: 1,916 instances across 50 CWE types, 96% detection precision. Section 7.5 acknowledges '50 CWEs... may not represent the complete universe of potential security flaws.' The paper contrasts CyberSecEval with alternatives (LLMSecEval: only 150 instances, 18 CWEs)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All model comparisons use the same RACG pipeline and CodeGuarder framework, controlling for scaffolding differences. The with/without CodeGuarder comparison isolates the security knowledge injection as the only variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CyberSecEval was published in 2023 and all evaluated models were trained or updated after this date. The possibility that models saw CyberSecEval instances during training is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The security knowledge injected into prompts is derived from vulnerability databases that may overlap with model training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly separates data sources: security knowledge from ReposVul, vulnerable code from CyberSecEval, stating: 'Using the same dataset for both the vulnerable code and the security knowledge base could lead to overlapping fix strategies, making poisoning attempts easier to detect.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, membership inference, n-gram overlap analysis) are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeGuarder improves security rate by 20.12% on average in standard RACG scenarios across 4 LLMs and 4 languages.",
       "evidence": "Table 4 shows average SR increases from 60.84 to 73.08 across GPT-4o, DS-V3, CodeLlama, and DS-Coder on C, C++, Java, and Python (Section 6.2).",
       "supported": "strong"
     },
     {
       "claim": "CodeGuarder improves security rate by 31.53% under targeted poisoning (Scenario I) where attacker has access to programming intents.",
       "evidence": "Table 5 shows average SR increases from 50.57 to 66.52 across all LLMs and languages (Section 6.3.1).",
       "supported": "strong"
     },
     {
       "claim": "CodeGuarder improves security rate by 21.91% under intent-agnostic poisoning (Scenario II).",
       "evidence": "Table 6 shows average SR increases from 60.05 to 73.20 across all LLMs and languages (Section 6.3.2).",
       "supported": "strong"
     },
     {
       "claim": "CodeGuarder does not compromise functional correctness of generated code.",
       "evidence": "Tables 4-6 show Sim metric slightly improves with CodeGuarder. Table 9 shows slight improvements on MBPP and HumanEval (e.g., GPT-4o Pass@1 on MBPP: 72.8 → 73.2) (Section 7.1).",
       "supported": "moderate"
     },
     {
       "claim": "CodeGuarder outperforms state-of-the-art SafeCoder by 9.80% in security rate in non-retrieval scenarios.",
       "evidence": "Table 7 shows CodeGuarder achieves 79.40% average SR vs SafeCoder's 72.31% on Mistral-7B across C, C++, Python (Section 6.4.1).",
       "supported": "moderate"
     },
     {
       "claim": "CodeGuarder generalizes to languages without target-specific security knowledge, improving security by 15.69% in standard scenarios.",
       "evidence": "Table 8 shows average SR improvements from 55.36% to 64.04% across C#, JavaScript, PHP, and Rust when no language-specific knowledge is available (Section 6.4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Query decomposition is the most impactful component, contributing ~10.1% SR improvement.",
       "evidence": "Table 11 ablation study: removing QD drops SR from 76.36% to 68.61% in standard scenario; removing only KRF drops to 74.13% (Section 7.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No code or artifact release",
       "detail": "Despite proposing a complete framework (CodeGuarder) with multiple components (knowledge base, retriever, re-ranker), no code repository, constructed knowledge base, or experimental artifacts are released. This prevents independent verification and reproduction."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims are based on raw number comparisons without any statistical tests, confidence intervals, or error bars on main results. The 5-run analysis covers only one model under one scenario."
     },
     {
       "flag": "Same research group evaluates their own attack and defense",
       "detail": "Reference [24] (Lin et al. 2025, 'Exploring the Security Threats of Knowledge Base Poisoning in RACG') shares authors with this paper. The same group defined the poisoning attack scenarios and now proposes the defense, potentially biasing both the threat model and evaluation design."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "CyberSecEval was published in 2023. GPT-4o and DeepSeek-V3, trained after 2023, may have seen benchmark instances during training. Baseline security rates could be inflated or deflated by contamination, affecting the measured improvement from CodeGuarder."
     },
     {
       "flag": "Baseline comparison limited to small models only",
       "detail": "The comparison with prior methods (Sven, SafeCoder, CoSec) in Table 7 is limited to Mistral-7B and CodeLlama-7B, not the larger models (GPT-4o, DS-V3) used in main experiments. This is justified by training costs but limits the strength of the state-of-the-art comparison."
     },
     {
       "flag": "Cost overhead not quantified",
       "detail": "CodeGuarder adds query decomposition (LLM call), retrieval, re-ranking, and prompt augmentation on top of standard RACG. The computational overhead is never measured, making it impossible to assess practical deployability."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Foundational study evaluating security of LLM-generated code, finding ~40% of Copilot code contains vulnerabilities."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "Proposes SVEN, a prefix-tuning approach for steering LLM code generation toward security properties."
     },
     {
       "title": "Instruction tuning for secure code generation",
       "authors": ["Jingxuan He", "Mark Vero", "Gabriela Krasnopolska", "Martin Vechev"],
       "year": 2024,
       "arxiv_id": "2402.09497",
       "relevance": "SafeCoder uses instruction tuning to train LLMs for secure code generation, the primary baseline for CodeGuarder."
     },
     {
       "title": "CoSec: On-the-Fly Security Hardening of Code LLMs via Supervised Co-decoding",
       "authors": ["Dong Li", "Meng Yan", "Yaosheng Zhang"],
       "year": 2024,
       "relevance": "Proposes co-decoding for security hardening without weight modification, a key baseline for non-retrieval comparison."
     },
     {
       "title": "Purple LLaMA CyberSecEval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa", "Cyrus Nikolaidis"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Primary evaluation benchmark with 1,916 instances across 50 CWE types for assessing LLM code security."
     },
     {
       "title": "CyberSecEval 3: Advancing the evaluation of cybersecurity risks and capabilities in large language models",
       "authors": ["Shengye Wan", "Cyrus Nikolaidis", "Daniel Song"],
       "year": 2024,
       "arxiv_id": "2408.01605",
       "relevance": "Extended version of CyberSecEval benchmark used for security evaluation in this study."
     },
     {
       "title": "Exploring the Security Threats of Knowledge Base Poisoning in Retrieval-Augmented Code Generation",
       "authors": ["Bo Lin", "Shangwen Wang", "Liqian Chen", "Xiaoguang Mao"],
       "year": 2025,
       "arxiv_id": "2502.03233",
       "relevance": "Companion paper by the same group that defines the RACG poisoning attack scenarios used in this evaluation."
     },
     {
       "title": "How secure is AI-generated code: a large-scale comparison of large language models",
       "authors": ["Norbert Tihanyi", "Tamas Bisztray", "Mohamed Amine Ferrag"],
       "year": 2025,
       "relevance": "Large-scale analysis of LLM code security across 13 models providing the vulnerability distribution used for CodeGuarder's re-ranking weights."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used to evaluate CodeGuarder's impact on functional correctness."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used to evaluate CodeGuarder's impact on functional correctness."
     },
     {
       "title": "Is your AI-generated code really safe? Evaluating large language models on secure code generation with CodeSecEval",
       "authors": ["Jiexin Wang", "Xitong Luo", "Liuwen Cao"],
       "year": 2024,
       "arxiv_id": "2407.02395",
       "relevance": "Evaluates security of LLM-generated code, corroborating persistent security vulnerabilities in generated code."
     },
     {
       "title": "Using AI Assistants in Software Development: A Qualitative Study on Security Practices and Concerns",
       "authors": ["Jan H Klemmer", "Stefan Albert Horstmann", "Nikhil Patnaik"],
       "year": 2024,
       "relevance": "Qualitative study on security practices with AI coding assistants, highlighting real-world security concerns in AI-assisted development."
     },
     {
       "title": "ReposVul: A Repository-Level High-Quality Vulnerability Dataset",
       "authors": ["Xinchen Wang", "Ruida Hu", "Cuiyun Gao"],
       "year": 2024,
       "relevance": "Provides the 12,053 function-level vulnerability pairs used to construct CodeGuarder's security knowledge base."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security hardening for RAG-based code generation is directly relevant to practitioners, but the framework is not released as a usable tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that injecting security knowledge helps is intuitive; the poisoning vulnerability of RACG was already established in the authors' prior work."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that poisoned knowledge bases can cause nearly half of generated code to be vulnerable, raising practical security concerns for RAG-based coding tools."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or challenges to established players; presents a constructive defense framework."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or installable tool is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates GPT-4o and DeepSeek-V3 which are recognizable, but the authors' institution (NUDT) is not widely known in the AI popular press."
     }
   }
 }