ai-research-survey

scan.json (28581B)

---

 {
   "paper": {
     "title": "Inducing Vulnerable Code Generation in LLM Coding Assistants",
     "authors": [
       "Binqi Zeng",
       "Quan Zhang",
       "Chijin Zhou",
       "Gwihwan Go",
       "Yu Jiang",
       "Heyuan Shi"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.15867",
     "doi": "10.48550/arXiv.2504.15867"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HACKODE demonstrates that LLM coding assistants can be induced to generate vulnerable code (buffer overflow, array violations, etc.) through adversarial attack sequences embedded in otherwise-correct referenced code examples, achieving 84.29% average attack success rate across four open-source LLMs (7b-15b). The attack transfers to unseen prompt templates and quantized models at 57.97% and 50.76% ASR respectively, and achieves 75.92% ASR on a real-world coding assistant application. A two-phase progressive generation approach (preliminary + enhancement) improves transferability by 25.49% over single-phase generation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository: https://github.com/HACKODE11/HACKODE. Section I contributions: 'The source code is included at the repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes constructing a dataset of 35 programming problems from StackOverflow but only explicitly states the 'source code' is released at the repository. The dataset itself is not explicitly stated as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (AMD EPYC 7763 CPUs, 8 NVIDIA V100 GPUs, Ubuntu 22.04 LTS) but no software environment details are provided — no requirements.txt, library versions, framework versions, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The reader would need to infer the procedure from the methodology description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables II-VII) report point estimate ASR percentages with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (HACKODE vs HACKODE−, comment vs variable renaming, across LLMs) based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage differences are reported with baseline context: e.g., 'average increase of 25.49%' (HACKODE 57.97% vs HACKODE− 32.48% in Table VI), and per-model ASR breakdowns in Tables II-VII allow readers to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises only 35 programming problems with no justification for this sample size, no power analysis, and no discussion of whether 35 is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. ASR numbers appear to be from single attack generation attempts per problem without reporting variability across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D includes ablation baselines: HACKODE− (without progressive generation, Table VI) and variable renaming as alternative injection method (Table VII)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "This paper introduces a novel attack type (inducing vulnerable code via external references) with no direct prior work to compare against. The ablation-based baselines (HACKODE−, variable renaming) are appropriate given the novelty of the threat model."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D presents two ablation studies: (1) progressive generation vs non-progressive (Table VI) and (2) comment insertion vs variable renaming (Table VII), isolating the contribution of each design choice."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: ASR (attack success rate), number of iterations, token lengths of attack sequences, responses, and assembled inputs (Table II)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated vulnerable code is performed. Attack success is determined automatically by checking whether the target vulnerability appears in the generated code. Human evaluation of vulnerability exploitability or code quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B evaluates transferability on 'new instructions, queries, and prompt templates' that 'were not used in the generation process of the attack sequences' — a proper held-out evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by vulnerability type (5 categories) and by LLM (4 models) in Tables II-VII, showing substantial variation (e.g., 25%-100% ASR across categories)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: Incorrect Variable on Mistral achieves only 25% ASR, GPTQ Llama2 drops to 29.63%, and general LLMs sometimes generate text instead of code. Section V-B analyzes Llama2's poor transferability."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: 25% ASR for Incorrect Variable on Mistral (Table II), 29.63% for GPTQ Llama2 (Table IV), and 0% ASR for variable renaming on Incorrect Variable and Infinite Loop (Table VII)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '84.29% success rate' and '75.92% ASR' on real-world application are directly supported by Tables II and V respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim (attack sequences cause LLMs to generate vulnerable code) is supported by the controlled experimental design: the same LLM with the same reference produces safe code without the attack sequence and vulnerable code with it. The ablation study (Table VI) further supports causality."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds claims: 'HACKODE currently focuses solely on open-source LLMs,' 'validated on LLMs with parameter sizes ranging from 7b to 15b,' and 'may not be highly effective against certain LLMs that exhibit significant performance degradation following quantization.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why the attack works beyond the proposed gradient-based mechanism. No consideration of confounds such as whether the models would generate vulnerable code at some baseline rate without the attack, or whether results are driven by model memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: whether the target vulnerability code appears in the LLM's generated output. ASR is a direct measure of attack success, not a proxy for a broader claim. The real-world experiment (Section V-C) validates that lab results translate to practical scenarios."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names and sizes are provided: Llama2-7b, Mistral-7b, CodeLlama-7b, and StarChat2-15b. For open-source models, these identify specific released model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 2 shows example instructions (IN1, IN2), prompt templates (PT1-PT3), and queries (Q1-Q3), but these are samples from a larger set. The paper states they 'draw on instructions from several LLM-powered applications' and 'use LLMs to generate new instructions' without providing the complete set. Full prompts cannot be reconstructed from the paper alone."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Attack hyperparameters are stated (maxStep=500, k=3). However, LLM generation hyperparameters (temperature, top-p) are only described as 'All other hyperparameters were kept at their default values' without stating the actual values."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper's own method (HACKODE) is a gradient-based attack optimization, not an agentic scaffolding system. The coding assistant being attacked uses a search-retrieve-generate pipeline, but the paper is evaluating an attack, not building scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes the data collection: 'utilized the StackOverflow API and the StackExchange library to collect the answers,' excluded problems solvable without external references, 'through manual validation' ensured code examples are present and referenceable. Target vulnerabilities for each problem are specified in Table I."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI Discussion includes a dedicated 'Limitations' paragraph discussing three specific limitations of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations paragraph identifies specific threats: (1) only open-source LLMs tested, not closed-source, (2) only 7b-15b parameter range tested, (3) GPTQ quantization degrades some models' performance, reducing attack effectiveness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly states what was not tested: closed-source LLMs, models smaller than 7b or larger than 15b, and acknowledges reduced effectiveness on heavily quantized models. Future work directions bound current claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Source code is released but raw experimental data (generated attack sequences, model outputs, full response logs) is not mentioned as available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes data collection: StackOverflow API and StackExchange library used, most up-to-date answers collected, excluded problems solvable without domain knowledge, manual validation performed, 35 problems in 4 languages (Python, Java, C++, PHP)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is StackOverflow problems (a standard public platform), not a recruited sample."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the general pipeline (collect from StackOverflow → filter → manual validation → construct vulnerabilities) but does not report how many problems were initially collected, how many were filtered at each stage, or specific filtering criteria beyond 'require domain knowledge.'"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Central South University (Zeng, Shi) and Tsinghua University (Zhang, Zhou, Go, Jiang). The authors are not affiliated with the evaluated LLMs' developers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. Absence of disclosure is not equivalent to absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests an adversarial attack against LLMs, not model capability on benchmarks. The attack deliberately uses problems the models cannot solve, so training data contamination of benchmark problems is not the concern being evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates attack effectiveness, not model knowledge. The attack works by manipulating inputs, not by exploiting memorized answers. Train/test overlap is structurally irrelevant to the attack evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper tests defenses/attacks rather than model knowledge. Contamination of the StackOverflow problems in training data would if anything make the attack harder (models might generate correct code on their own), which the paper already accounts for by selecting problems LLMs cannot solve."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, wall-clock time, or per-attack cost is reported. Iteration counts are given (avg 179.17) but not translated to time or compute cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed (AMD EPYC 7763, 8 V100 GPUs) but total compute time, GPU hours, or training budget for attack generation is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seed sensitivity analysis is reported. The attack generation involves random initialization and sampling, but results across different seeds are not shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs producing the main ASR results (Table II) is not explicitly stated. The transferability test uses 5 assembled inputs (Section V-B), and k=3 is stated for enhancement, but it's unclear how many independent attack generation attempts were made per problem."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters (maxStep=500, k=3) are stated but no justification for their selection, search budget, or sensitivity analysis is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Fixed hyperparameters are used without justification for their selection. No validation set-based selection or reporting of alternative configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own attack system (HACKODE) without acknowledging potential self-evaluation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While iteration counts are reported per attack (Table II), performance is not analyzed as a function of compute budget. No compute-matched comparisons between HACKODE and HACKODE−."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper constructs a benchmark of 35 StackOverflow problems but does not discuss whether this benchmark is representative of real-world attack scenarios. The real-world experiment (Section V-C) partially validates the benchmark but construct validity is not explicitly analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses scaffold variation by testing attack transferability across different prompt templates, instructions, and queries (Section V-B). The attack is designed to be robust to these scaffold differences, and Table III quantifies transferability."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper selects 'most up-to-date' StackOverflow problems that LLMs cannot solve, there is no explicit discussion of temporal leakage — whether the models' training data could have included these problems or their solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The attack sequence itself could potentially leak information about the target vulnerability through the reference text, but this is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 35 programming problems are independent or share structural similarities that could inflate attack success rates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The paper verifies that LLMs cannot solve the problems directly (which is related but not a leakage detection method per se)."
       }
     }
   },
   "claims": [
     {
       "claim": "HACKODE achieves an average 84.29% attack success rate across four LLMs (Llama2-7b 77.14%, Mistral-7b 80.00%, CodeLlama-7b 94.29%, StarChat2-15b 85.71%).",
       "evidence": "Table II shows per-vulnerability and per-model ASR results across 35 programming problems with 5 vulnerability types.",
       "supported": "moderate"
     },
     {
       "claim": "Attack sequences transfer to randomly assembled inputs with an average 57.97% ASR, and 83.58% of data passes at least one of five transfer tests.",
       "evidence": "Table III shows transfer rates on 5 new assembled inputs per problem using unseen instructions, templates, and queries.",
       "supported": "moderate"
     },
     {
       "claim": "Attack sequences transfer to quantized LLMs with 48.07% ASR (GPTQ) and 53.45% ASR (BitsAndBytes).",
       "evidence": "Table IV shows ASR on 4-bit quantized versions of the four target LLMs.",
       "supported": "moderate"
     },
     {
       "claim": "HACKODE achieves 75.92% ASR on a real-world coding assistant (ChatChat) powered by Mistral-7b (82.14%) and CodeLlama-7b (69.70%).",
       "evidence": "Table V shows per-vulnerability ASR on ChatChat application with StackOverflow API agent, tested on local pages to prevent dissemination.",
       "supported": "moderate"
     },
     {
       "claim": "The progressive generation approach improves ASR by 25.49% over non-progressive generation.",
       "evidence": "Table VI compares HACKODE (57.97% average) vs HACKODE− (32.48% average) on transferability tests.",
       "supported": "moderate"
     },
     {
       "claim": "Embedding attack sequences as code comments achieves 80.00% ASR vs 11.43% for variable renaming on Mistral-7b.",
       "evidence": "Table VII shows per-vulnerability comparison between comment insertion and variable renaming approaches.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small dataset",
       "detail": "Only 35 programming problems are used, with as few as 4 examples in some vulnerability categories (Incorrect Variable). This is too small for reliable ASR estimation, especially per-category breakdowns where a single problem flip changes the rate by 25%."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All ASR numbers are reported as point estimates without confidence intervals, error bars, or variance across runs. With n=35, the 95% CI for an 84% ASR is roughly ±12 percentage points."
     },
     {
       "flag": "Only small open-source models tested",
       "detail": "All models are 7b-15b parameter open-source models. No testing on commercial models (GPT-4, Claude) or larger open-source models (70b+), limiting generalizability claims about 'LLM Coding Assistants' broadly."
     },
     {
       "flag": "Simulated real-world experiment",
       "detail": "The 'real-world experiment' (Section V-C) uses local web pages instead of actual StackOverflow, and only one coding assistant application (ChatChat). The ecological validity of the attack spreading through actual forums is not tested."
     },
     {
       "flag": "No baseline vulnerability rate",
       "detail": "The paper does not measure how often the target LLMs generate vulnerable code without the attack sequence. Without a baseline vulnerability rate, the 84.29% ASR cannot be interpreted as the attack's marginal contribution."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for evaluating LLM code generation capabilities."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan"],
       "year": 2022,
       "relevance": "Directly relevant: evaluates security vulnerabilities in Copilot-generated code across top 25 CWE categories."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar"],
       "year": 2023,
       "relevance": "Studies how AI coding assistants impact security practices of users — directly relevant to LLM code security."
     },
     {
       "title": "Lost at C: A user study on the security implications of large language model code assistants",
       "authors": ["Gustavo Sandoval", "Hammond Pearce"],
       "year": 2023,
       "relevance": "User study examining security implications of LLM code assistants."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM used as evaluation target in this paper."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J Zico Kolter"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Key prior work on adversarial attacks against LLMs; HACKODE extends this to code generation domain."
     },
     {
       "title": "DeceptPrompt: Exploiting LLM-driven code generation via adversarial natural language instructions",
       "authors": ["Fangzhou Wu", "Xiaogeng Liu", "Chaowei Xiao"],
       "year": 2023,
       "arxiv_id": "2312.04730",
       "relevance": "Directly related attack on LLM code generation via adversarial prompts; HACKODE differs by targeting external references rather than user prompts."
     },
     {
       "title": "GitHub Copilot AI pair programmer: Asset or liability?",
       "authors": ["Arghavan Moradi Dakhel", "Vahid Majdinasab"],
       "year": 2023,
       "relevance": "Evaluates Copilot's code generation quality including correctness and usability concerns."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2024,
       "relevance": "Rigorous evaluation of LLM code correctness — relevant to understanding baseline code quality."
     },
     {
       "title": "AutoSafeCoder: A multi-agent framework for securing LLM code generation through static analysis and fuzz testing",
       "authors": ["Ana Nunez", "Nafis Tanveer Islam"],
       "year": 2024,
       "arxiv_id": "2409.10737",
       "relevance": "Proposes defenses for LLM code security through static analysis and fuzzing — complementary to HACKODE's attack perspective."
     },
     {
       "title": "SALLM: Security assessment of generated code",
       "authors": ["Mohammed Latif Siddiq", "Joanna Cecilia da Silva Santos"],
       "year": 2024,
       "relevance": "Benchmark for evaluating security of LLM-generated code."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Demonstrates LLM capability for competitive programming, relevant to code generation capabilities assessment."
     }
   ]
 }