ai-research-survey

scan.json (27860B)

---

 {
   "paper": {
     "title": "A systematic literature review on the impact of AI models on the security of code generation",
     "authors": [
       "Claudia Negri-Ribalta",
       "Rémi Geraud-Stewart",
       "Anastasia Sergeeva",
       "Gabriele Lenzini"
     ],
     "year": 2024,
     "venue": "Frontiers in Big Data",
     "doi": "10.3389/fdata.2024.1386720"
   },
   "scan_version": 3,
   "active_modules": [
     "survey_methodology"
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This SLR reviews 19 papers on AI code generation security, finding broad agreement that AI models produce code with known vulnerabilities (CWE Top-25), with AI-generated Python code being more secure than C or Verilog. The security of generated code varies by model, programming language, vulnerability type, and prompt design. Conflicting evidence exists on whether AI-generated code is worse than human-generated code, with some studies finding no significant difference. Mitigation strategies include better training datasets, improved prompting, post-processing, and end-user education, though none are conclusively effective.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code or scripts are released. The Zenodo archive (https://zenodo.org/records/11092334) contains the paper sample dataset, not analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The full sample dataset is available on Zenodo: 'We have provided the whole sample at: https://doi.org/10.5281/zenodo.10666386 for replication and transparency' (Section 6). Each paper has details on why it was included/excluded."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The survey involves no computational analysis requiring reproducible software environments, but analysis scripts could have been released with environment specs."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No explicit step-by-step reproduction instructions or README are provided. The methodology section (Section 3) describes the SLR process narratively with search strings and criteria, but there is no dedicated reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative systematic literature review with no statistical aggregation or meta-analysis computations."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The survey performs qualitative synthesis of findings from reviewed papers. No statistical tests are conducted by the authors."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes computed by the authors. The survey reports effect sizes from reviewed papers but does not produce its own."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "The sample size (N=19) is determined by the systematic search methodology, not predetermined. No power analysis is applicable to an SLR."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or statistical computations are performed by the authors."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare its findings against prior surveys or systematic reviews on the same topic. Background (Section 2) discusses related work but does not systematically compare findings with prior review efforts."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines apply to this systematic literature review."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components exists; this is a systematic literature review."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted requiring multiple metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs exist to be evaluated by humans."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring train/test splits are conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Section 5.3: Python, C, Java, Verilog), AI model family (Section 5.1, Figure 3), vulnerability type (CWE categories), and mitigation strategy (Section 5.4, Table 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where comparisons between papers are difficult or impossible due to different methodologies, vulnerabilities studied, and experimental setups (Section 5.1). Section 6 discusses threats and limitations of the review itself."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports conflicting evidence: Asare et al. (2023) and Sandoval et al. (2023) find AI-generated code is no worse than human-generated code, contradicting the general trend. The paper also reports that mitigation strategies have limited or unproven effectiveness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the work provides 'a comprehensive and systematic overview of the impact of AI in secure coding.' While the review is systematic, claiming comprehensiveness with only 19 papers covering a narrow set of AI models and programming languages is an overclaim. Core findings about vulnerabilities and mitigation strategies are supported by Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes the causal claim that AI models 'introduce' vulnerabilities into generated code. This is supported by synthesized evidence from 19 papers, several of which are experimental studies (Pearce et al. 2022, Perry et al. 2023, Sandoval et al. 2023) that provide causal evidence through controlled experiments. The survey appropriately hedges by noting conflicting results."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'a comprehensive and systematic overview of the impact of AI in secure coding,' but the review covers only 19 papers with a limited set of AI models (predominantly OpenAI) and programming languages. The conclusion appropriately hedges ('it is still premature to conclude'), but the abstract and title frame broader coverage than delivered."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations for why AI models generate insecure code: training data quality (Section 5.4.1), prompt design and temperature settings (Section 5.3.1), model architecture differences (Section 5.1), and the hypothesis that less training data for languages like Verilog leads to worse security (Section 5.3.4)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges measurement differences across studies: 'it remains difficult to compare the results' (Section 5.1) because papers researched 'different vulnerabilities' using different metrics. Footnote 9 notes that functional correctness and security are distinct concerns. The paper is aware that different CWE metrics measure different aspects of security."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not use AI models directly."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used; this is a systematic literature review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models are run; this is a systematic literature review."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a systematic literature review."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper selection pipeline is well documented with counts at each stage: 95 initial → 21 after deduplication and inclusion/exclusion → 43 after snowballing → 35 after criteria → 23 after full-text review → 19 after quality check (Figure 1, Sections 3-4). Filtering criteria are stated in Table 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to validity and future work' provides substantial discussion of reliability and validity issues, running approximately two full pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses threats specific to this study: sample representativeness depends on date of data collection, definition of 'code generation' affects results, high proportion of snowballed papers, researcher background affecting classification, and incomparable vulnerability taxonomies across papers making comparison 'at best, complicated and, at worst, a threat to our conclusions.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines scope boundaries: AI code generation means 'artifacts that suggest or produce code,' excluding tools that only verify/check code (Section 4.1). Table 3 lists exclusion criteria. Section 6 states what results do not show: 'the final sample of this research may increase and change depending on the day the data was gathered.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full sample with inclusion/exclusion decisions is available on Zenodo: 'Each paper has details on why it was included/excluded, at which phase, and with details and/or comments to help readers understand and replicate our research' (Section 6)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: databases used (IEEE Xplore, ACM, Scopus), specific search strings per database (Table 2), time period ('last week of November 2023'), and PICO framework (Section 3.1). Keywords and synonyms are provided in Table 1."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Paper 'recruitment' methods are thoroughly described: database searches with explicit strings (Table 2), forward snowballing via Google Scholar (Section 3.3), and the process for evaluating snowballed papers based on title, abstract, and keywords following Wohlin (2014)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from search to final analysis is documented in Figure 1 and Section 4.1: 95 initial articles → deduplication → inclusion/exclusion → 21 starting set → snowballing → 43 articles → inclusion/exclusion → 35 → full-text review → 23 → quality check → 19 final. Criteria at each stage are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: 'This research was funded in whole, or in part, by the Luxembourg National Research Fund (FNR), grant: NCER22/IS/16570468/NCER-FT.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Luxembourg (Security and Trust, and FHSE), and École Normale Supérieure, Paris. No authors are affiliated with AI companies whose products are being reviewed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Luxembourg National Research Fund (FNR) is a national funding agency with no commercial stake in whether AI models generate secure or insecure code."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "A competing interests statement is provided: 'The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review. The authors review published papers, not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review. Paper inclusion/exclusion criteria are covered under data_integrity."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions in this systematic literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions in this systematic literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review with no computational method whose cost could be reported."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review with no computational budget."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The survey follows Kitchenham and Charters (2007) SLR guidelines, uses the PICO framework (Section 3.1), provides reproducible search strings per database (Table 2), and includes a selection flow diagram (Figure 1). Additional guidelines from Wieringa et al. (2006), Wohlin (2014), and Petersen et al. (2015) are also followed."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "A quality assessment with an 8-question checklist (Table 4) and 4-point scoring system (Table 5) is applied to all papers. Quality scores for each paper are reported transparently in Table 8. Papers scoring below 50% (4 points) are excluded. Quality checks are done by at least two authors."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias—whether its source papers skew toward positive findings about AI-generated code vulnerabilities. Section 6 discusses sample representativeness and timing issues but not systematic positive-result bias in the reviewed literature."
       }
     }
   },
   "claims": [
     {
       "claim": "There is high-level agreement that AI models do not produce safe code and do introduce vulnerabilities, despite mitigations.",
       "evidence": "Synthesized from 19 reviewed papers, with multiple studies (Pearce et al. 2022, He and Vechev 2023, Tony et al. 2023, Perry et al. 2023) consistently finding vulnerabilities in AI-generated code. Section 5 and Conclusion.",
       "supported": "moderate"
     },
     {
       "claim": "AI-generated Python code has fewer vulnerabilities than AI-generated C code.",
       "evidence": "Pearce et al. (2022) found 39% vulnerability rate for Python vs 50% for C. He and Vechev (2023) found ~42% for Python. Pearce et al. (2023) also found C performed worse. Section 5.3.",
       "supported": "moderate"
     },
     {
       "claim": "Users who use AI assistants produce more insecure code and trust AI-generated code more than their own.",
       "evidence": "Perry et al. (2023) found experimental group produced less secure code: 21% secure in AI-assisted vs 43% in control group for encryption tasks. However, Sandoval et al. (2023) and Asare et al. (2023) found no significant difference. Section 5.1 and 5.3.",
       "supported": "weak"
     },
     {
       "claim": "AI models have fundamental limitations preventing them from creating complete malware from scratch, though they can generate malware snippets.",
       "evidence": "Botacin (2023) found ChatGPT cannot create malware from scratch but can create snippets. Pa Pa et al. (2023) found models could create up to 400 lines of malware with jailbreaks. Liguori et al. (2023) noted human review remains necessary. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "The conclusions diverge on whether AI-generated code is worse than human-generated code.",
       "evidence": "Perry et al. (2023) found AI-assisted users wrote more insecure code. Asare et al. (2023) and Sandoval et al. (2023) found AI models make similar or fewer mistakes than humans. The paper acknowledges 'there is no clear favor for one hypothesis over the other.' Conclusion.",
       "supported": "strong"
     },
     {
       "claim": "Prompting and context play a crucial role in the security of AI-generated code.",
       "evidence": "Pearce et al. (2022) showed a variation from 0% to 94% vulnerability rate depending on prompt content for SQL injection. Nair et al. (2023) explored prompt strategies for secure output. Perry et al. (2023) observed relationship between model parameters and code quality. Section 5.3.1.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample for broad claims",
       "detail": "Only 19 papers form the final sample, yet the abstract claims to provide 'a comprehensive and systematic overview of the impact of AI in secure coding.' The coverage is narrow—predominantly OpenAI models, Python and C—relative to the broad framing."
     },
     {
       "flag": "Disproportionate snowballing",
       "detail": "22 of 43 candidate papers (51%) came from snowballing rather than the database search (21 papers), suggesting the original search strategy may have been incomplete. The authors acknowledge this as unusual and discuss it in Section 6."
     },
     {
       "flag": "No publication bias assessment",
       "detail": "Despite being a systematic review, the paper does not discuss whether the reviewed literature skews toward papers that find vulnerabilities in AI-generated code (positive-result bias). Papers finding 'AI code is fine' may be underrepresented."
     },
     {
       "flag": "Qualitative synthesis without quantitative aggregation",
       "detail": "The survey qualitatively summarizes findings but does not statistically aggregate results across studies. Different methodologies, vulnerability taxonomies, and experimental setups across the 19 papers make comparison difficult, as the authors themselves acknowledge."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Foundational study finding 40% of Copilot's code suggestions contained vulnerabilities across CWE Top-25, directly relevant to AI code generation security."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar", "D. Boneh"],
       "year": 2023,
       "relevance": "User study finding AI-assisted developers produce more insecure code and trust AI-generated code more, key evidence on human-AI interaction in code security."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["J. He", "M. Vechev"],
       "year": 2023,
       "relevance": "Proposes SVEN for controlled secure/insecure code generation, demonstrating both security improvement (to >85%) and degradation capabilities for LLMs."
     },
     {
       "title": "Examining zero-shot vulnerability repair with large language models",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Studies LLM capability for zero-shot vulnerability repair across multiple models and CWEs, finding limited but model-dependent repair success."
     },
     {
       "title": "Lost at C: a user study on the security implications of large language model code assistants",
       "authors": ["G. Sandoval", "H. Pearce", "T. Nys", "R. Karri", "S. Garg", "B. Dolan-Gavitt"],
       "year": 2023,
       "relevance": "User study finding no conclusive evidence that LLM assistants increase CWE incidence compared to unaided developers in C programming."
     },
     {
       "title": "Is GitHub's Copilot as bad as humans at introducing vulnerabilities in code?",
       "authors": ["O. Asare", "M. Nagappan", "N. Asokan"],
       "year": 2023,
       "doi": "10.48550/arXiv.2204.04741",
       "relevance": "Compares Copilot's vulnerability introduction to human developers, finding Copilot can avoid detected vulnerabilities in a substantial number of scenarios."
     },
     {
       "title": "How effective are neural networks for fixing security vulnerabilities",
       "authors": ["Y. Wu", "N. Jiang", "H. V. Pham", "T. Lutellier", "J. Davis", "L. Tan"],
       "year": 2023,
       "relevance": "Proposes VJBench benchmark and evaluates LLMs on Java vulnerability repair, finding models fix very few vulnerabilities (Codex at 20.4%)."
     },
     {
       "title": "Large language models and simple, stupid bugs",
       "authors": ["K. Jesse", "T. Ahmed", "P. T. Devanbu", "E. Morgan"],
       "year": 2023,
       "relevance": "Studies simple stupid bug generation by code LLMs, finding models propose twice as many bugs as correct code in Java."
     },
     {
       "title": "CodexLeaks: privacy leaks from code generation language models in GitHub's Copilot",
       "authors": ["L. Niu", "S. Mirza", "Z. Maradni", "C. Pöpper"],
       "year": 2023,
       "relevance": "Demonstrates personal data leakage through Copilot code suggestions, a novel security exploit of AI code generation models."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "doi": "10.48550/arXiv.2107.03374",
       "relevance": "Foundational Codex paper establishing evaluation of LLMs for code generation, widely cited in AI coding capability research."
     },
     {
       "title": "CodeAttack: code-based adversarial attacks for pre-trained programming language models",
       "authors": ["A. Jha", "C. K. Reddy"],
       "year": 2023,
       "relevance": "Proposes black-box adversarial attack on code generation models by targeting vulnerable tokens, demonstrating security risks of code LLMs."
     },
     {
       "title": "GPThreats-3: is automatic malware generation a threat?",
       "authors": ["M. Botacin"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's capability for malware generation, finding it can create snippets but not complete malware, relevant to AI safety in code generation."
     },
     {
       "title": "GitHub considered harmful? Analyzing open-source projects for the automatic generation of cryptographic API call sequences",
       "authors": ["C. Tony", "N. E. D. Ferreyra", "R. Scandariato"],
       "year": 2022,
       "relevance": "Studies security risks of AI-generated cryptographic API calls, finding significantly lower accuracy on security-sensitive tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Practitioners can learn about security risks of AI code generation, but the paper offers no tool, technique, or actionable mitigation they can deploy."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the widely-held expectation that AI-generated code has security issues; the finding of conflicting evidence on human vs AI code quality is notable but not contrarian."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises security concerns about AI code generation including malware creation, personal data leaks, and vulnerability introduction, topics that resonate with security-conscious audiences."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or drama; this is a neutral academic literature review without provocative claims."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No demo, tool, or code artifact is produced; the output is a literature review with a Zenodo dataset."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Reviews papers about well-known tools (Copilot, ChatGPT, Codex) but comes from University of Luxembourg, not a major AI lab."
     }
   }
 }