ai-research-survey

scan.json (25772B)

---

 {
   "paper": {
     "title": "Guiding AI to Fix Its Own Flaws: An Empirical Study on LLM-Driven Secure Code Generation",
     "authors": ["Hao Yan", "Swapneel Suhas Vaidya", "Xiaokuan Zhang", "Ziyu Yao"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2506.23034"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs generate vulnerable code at rates of 9.8%-42.1% across diverse CWE types, with similar vulnerability distributions across models suggesting shared architectural or training-data limitations. Self-generated vulnerability hints reduce vulnerability rates only when they are relevant, precise, and contextualized — GPT-4o achieved 12.4% TarV-R reduction while weaker models showed minimal or negative effects. Post-hoc vulnerability repair with explained (GPT-4o-enriched) CodeQL feedback is more effective than raw direct feedback, but only for models with strong instruction-following capabilities.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymized repository link is provided: https://anonymous.4open.science/r/LLM-driven_Secure_Code_Generation-E89E."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available benchmarks: SecurityEval and SecCodePLT. Both are established public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymized repo may contain them but the paper itself does not describe reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., 'GPT-4o achieves a 12.4% reduction') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage differences with baseline context throughout (e.g., Table 4 subscript numbers showing changes vs. vanilla prompts, Table 8 showing reductions). This provides magnitude in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why these particular dataset sizes (1,071 and 121 questions) are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All numbers appear to be single-run results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The vanilla prompt serves as the baseline, with self-generated hints and feedback-based repair compared against it (Tables 4, 7, 8 all show changes vs. vanilla baseline in Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The study uses contemporary models including GPT-4o, Llama3.1/3.2, DeepSeek-Coder-V2, and references recent related work like Tony et al. (2024) and Wang et al. (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates the effect of hint quality: comparing hints with vs. without target vulnerability (Table 5), CWE definition vs. contextualized hints (Table 7), and direct vs. explained feedback (Table 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are defined and used throughout: Target Vulnerability Rate (TarV-R) and All Vulnerability Rate (AllV-R)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human validation was conducted: two evaluators assessed 80 hint-definition pairs for the LLM-as-judge evaluation (95.65% precision, 91.67% recall), and manual validation of 160 explained feedback samples."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks (SecurityEval and SecCodePLT) serve as held-out evaluation sets. No tuning was done on these datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides per-CWE vulnerability distribution across all models. Table 5 breaks down TarV-R by whether hints include the target vulnerability."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where hints increase vulnerability rates (Table 4, GPT-3.5 on SecurityEval), irrelevant hints misleading models, and models that fail to utilize feedback (Section 5.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: self-generated hints sometimes increase vulnerability rates, CodeLlama models show no improvement from feedback, and GPT-4o has higher vulnerability rates than GPT-3.5 despite being newer."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that LLMs are prone to generating insecure code, that advanced models benefit from hints and feedback, and that vulnerability hints and explained feedback help. All are supported by Tables 3, 4, 7, and 8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (hints reduce vulnerabilities, explained feedback improves repair) are supported by controlled comparisons: same model, same dataset, varying only the intervention (vanilla vs. hints vs. feedback type)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Secure Code Generation' broadly, but the study is limited to Python only. The threats-to-validity section acknowledges this but the title and framing do not bound the claim. The paper also only tests single-turn interactions."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: model instruction-following capability vs. hint quality, architectural vs. training data differences for vulnerability rate variations, and code-optimized vs. general models (Sections 5.1-5.3)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures CodeQL-detected vulnerability rates and frames results in terms of those specific measurements. It explicitly acknowledges CodeQL's limitations (static analysis only, cannot capture dynamic vulnerabilities) in the threats to validity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 specifies model versions: GPT-4o-0513, GPT-3.5-turbo-0125, CodeLlama-Instruct 7B/34B, Llama3.1-Instruct 8B, Llama3.2-Instruct 3B, StarCoder2-Instruct 15B, DeepSeek-Coder-V2-Lite-Instruct 16B."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 2-5 for all experimental conditions (vanilla, hint generation, hint-augmented generation, direct feedback repair, explained feedback generation, explained feedback repair). The fill values come from the public datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the models used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is prompt-based with single-turn or two-turn interactions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents that 6 CWEs (274 samples) were excluded from SecCodePLT because they are not covered by CodeQL, resulting in 21 CWEs with 1,071 samples. The datasets and their properties are described in Section 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion' contains a 'Threats to validity' subsection with substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific: CodeQL as static-only analysis, Python-only evaluation, two datasets may not cover all vulnerability types, and static single-turn setting vs. real-world multi-turn interactions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it focuses on Python, uses static analysis (CodeQL), operates in a single-turn setting, and is limited to two datasets. These scope boundaries are clearly stated in Section 6."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The anonymized repo may contain generated code, but the paper itself does not provide raw experimental outputs (generated code snippets, CodeQL results per sample)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the two benchmarks in detail, including their origins, sizes, CWE coverage, and how they were constructed (Table 1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: benchmark questions → LLM code generation → CodeQL analysis → vulnerability rate computation. The exclusion of 274 samples is explained. Section 3 details each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses Virginia Commonwealth Cyber Initiative (CCI) sponsorship, CAHMP GRA Fellowship, and NSF Award Number 2018631."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are from George Mason University. No evaluated product is affiliated with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (Virginia CCI, NSF, GMU) have no financial interest in the performance of any specific LLM evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 8 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SecurityEval or SecCodePLT problems appeared in the training data of any model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SecurityEval (2022) and SecCodePLT (2024) are publicly available. Models trained after their publication may have seen them. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study. The human validators for GPT-4o-as-judge are a validation check, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite using proprietary APIs (GPT-4o, GPT-3.5) across thousands of samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple seed runs are reported. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs per experiment is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Temperature and sampling parameters are not even reported."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not select among configurations — it evaluates fixed prompts across models."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the prompts and evaluated the results without acknowledging potential bias in prompt design favoring certain outcomes."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The explained feedback approach uses GPT-4o to generate enriched feedback for all models, adding significant compute, but this cost is never compared against the performance gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses construct validity by acknowledging CodeQL's limitations (static-only, Python-only) and noting the divergence between SecCodePLT and SecurityEval results indicates different vulnerability coverage matters."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; models are prompted directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SecurityEval was published in 2022, SecCodePLT in 2024. Models trained after these dates may have seen the problems. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether the task descriptions inadvertently hint at secure implementations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SecCodePLT is generated by mutating 5 seed problems per CWE (up to 70 variants). These are structurally dependent. This is not discussed as a potential confound."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs consistently generate vulnerable code at rates ranging from 9.8% to 42.1% across diverse vulnerability types.",
       "evidence": "Table 3 shows TarV-R 4.0%-27.3% and AllV-R 9.8%-42.1% across 8 models on two benchmarks (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "The top-10 vulnerability types across 8 models largely overlap, resulting in only 11 unique types, indicating shared weaknesses.",
       "evidence": "Figure 6 shows the vulnerability distribution across models on SecCodePLT with overlapping top vulnerability types (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o achieves a 12.4% TarV-R reduction on both datasets when using self-generated vulnerability hints.",
       "evidence": "Table 4 shows GPT-4o TarV-R dropping from 15.0% to 2.6% on SecCodePLT and from 25.6% to 13.2% on SecurityEval (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Irrelevant or imprecise self-generated hints can mislead models and increase vulnerability rates.",
       "evidence": "Table 5 shows that when hints do not include the target vulnerability, most models show increased TarV-R. GPT-3.5 AllV-R increased by 17.4% on SecurityEval (Table 4, Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Explained feedback significantly enhances vulnerability repair effectiveness compared to direct CodeQL feedback.",
       "evidence": "Table 8 shows explained feedback outperforms direct feedback for most models. GPT-4o achieves 14.3% AllV-R reduction on SecCodePLT with explained vs. 11.2% with direct feedback (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Newer models (GPT-4o, Llama3) exhibit higher vulnerability rates than their predecessors (GPT-3.5, CodeLlama).",
       "evidence": "Table 3: GPT-4o TarV-R 15.0% vs. GPT-3.5 7.7% on SecCodePLT; Llama3.1-8B AllV-R 40.5% vs. CodeLlama-7B 28.9% on SecurityEval (Section 5.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims are based on raw percentage differences without any significance tests. With single runs and no variance measures, it is impossible to know if observed differences are meaningful or noise."
     },
     {
       "flag": "Single-run results with no variance",
       "detail": "LLM outputs are stochastic, yet no multiple runs, seeds, or variance measures are reported. Results could change substantially with different random seeds or temperature settings."
     },
     {
       "flag": "Contamination not addressed",
       "detail": "SecurityEval (2022) has been publicly available for years. Models trained after 2022 may have seen these problems, inflating performance (or reducing vulnerability rates). SecCodePLT's mutated variants from 5 seeds create structural non-independence that is also not discussed."
     },
     {
       "flag": "Hyperparameters unreported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any model, yet these significantly affect code generation behavior and vulnerability rates."
     },
     {
       "flag": "GPT-4o used as both subject and evaluator",
       "detail": "GPT-4o is used to generate explained feedback for all models AND to judge hint preciseness, while also being one of the 8 evaluated models. This creates a potential confound where GPT-4o's repair benefits may partly stem from receiving feedback in its own 'language'."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? assessing the security of github copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Foundational study showing GitHub Copilot generated vulnerable code in 40% of cases across 18 CWE types."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2023,
       "relevance": "User study showing AI code assistants lead to more insecure code, directly relevant to LLM code security."
     },
     {
       "title": "SecurityEval Dataset: Mining Vulnerability Examples to Evaluate Machine Learning-Based Code Generation Techniques",
       "authors": ["Mohammed Latif Siddiq", "Joanna C. S. Santos"],
       "year": 2022,
       "doi": "10.1145/3549035.3561184",
       "relevance": "One of the two benchmarks used in this study; provides 121 coding problems across 69 CWE types."
     },
     {
       "title": "SecCodePLT: A Unified Platform for Evaluating the Security of Code GenAI",
       "authors": ["Yu Yang", "Yuzhou Nie", "Zhun Wang", "Yuheng Tang", "Wenbo Guo", "Bo Li", "Dawn Song"],
       "year": 2024,
       "arxiv_id": "2410.11096",
       "relevance": "The other benchmark used in this study; synthesized dataset with 1,345 coding problems covering 27 CWEs."
     },
     {
       "title": "Prompting techniques for secure code generation: A systematic investigation",
       "authors": ["Catherine Tony", "Nicolás E Díaz Ferreyra", "Markus Mutas", "Salem Dhiff", "Riccardo Scandariato"],
       "year": 2024,
       "arxiv_id": "2407.07064",
       "relevance": "Evaluated prompting techniques for secure code generation on GPT-series, finding RCI technique most effective."
     },
     {
       "title": "Examining zero-shot vulnerability repair with large language models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad", "Ramesh Karri", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Explored zero-shot vulnerability repair by embedding vulnerability information into prompts."
     },
     {
       "title": "Purple llama cyberseceval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Meta's comprehensive security evaluation benchmark for LLMs including insecure code generation assessment."
     },
     {
       "title": "How secure is code generated by chatgpt?",
       "authors": ["Raphaël Khoury", "Anderson R Avila", "Jacob Brunelle", "Baba Mamadou Camara"],
       "year": 2023,
       "relevance": "Showed GPT-3.5 produced 76% vulnerable code, relevant to LLM security evaluation."
     },
     {
       "title": "Is Your AI-Generated Code Really Safe? Evaluating Large Language Models on Secure Code Generation with CodeSecEval",
       "authors": ["Jiexin Wang"],
       "year": 2024,
       "arxiv_id": "2407.02395",
       "relevance": "Explored enhancing secure code generation by incorporating vulnerability information into prompts."
     },
     {
       "title": "Code security vulnerability repair using reinforcement learning with large language models",
       "authors": ["Nafis Tanveer Islam", "Mohammad Bahrami Karkevandi", "Peyman Najafirad"],
       "year": 2024,
       "arxiv_id": "2401.07031",
       "relevance": "RL-based framework for fine-tuning LLMs with dual functionality and security rewards."
     },
     {
       "title": "Automated software vulnerability patching using large language models",
       "authors": ["Yu Nong", "Haoran Yang", "Long Cheng", "Hongxin Hu", "Haipeng Cai"],
       "year": 2024,
       "arxiv_id": "2408.13597",
       "relevance": "LLM-based vulnerability patching using retrieved similar examples with ground-truth annotations."
     },
     {
       "title": "Is github's copilot as bad as humans at introducing vulnerabilities in code?",
       "authors": ["Owura Asare", "Meiyappan Nagappan", "N Asokan"],
       "year": 2023,
       "relevance": "Comparative study of Copilot vs. human-introduced code vulnerabilities."
     }
   ]
 }