ai-research-survey

scan.json (24740B)

---

 {
   "paper": {
     "title": "AI Code Generators for Security: Friend or Foe?",
     "authors": ["Roberto Natella", "Pietro Liguori", "Cristina Improta", "Bojan Cukic", "Domenico Cotroneo"],
     "year": 2024,
     "venue": "IEEE Security & Privacy",
     "arxiv_id": "2402.01219",
     "doi": "10.1109/MSEC.2024.3355713"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released at https://github.com/dessertlab/violent-python, as stated in the paper with a footnote URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The violent-python dataset is publicly available at the GitHub repository linked in the paper (https://github.com/dessertlab/violent-python)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a machine with a Debian-based distribution, with 8 vCPU, 16 GB RAM, and two NVIDIA T4 GPUs' but does not provide a requirements.txt, Dockerfile, or detailed library versions needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper describes the experimental setup in general terms but does not include commands, scripts, or a README with instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single point estimates (e.g., '77.89%', '65.38%', '60.76%') with no confidence intervals or error bars in any figure or table."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeBERT outperforms competitors and that fine-tuning improves performance, but no statistical significance tests are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute edit distance percentages with baselines for comparison (e.g., '19.05% vs 77.89%' for zero-shot vs fine-tuned, and '77.89%' vs '65.38%' vs '60.76%' across generators), and contrasts with general-purpose code generation at '86.7%'. This provides enough context to understand effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 1,372 samples split 80/10/10, but there is no justification for why this size is sufficient. The paper notes the size is 'in line with other state-of-the-art corpora' but provides no power analysis or formal justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any experimental results. All numbers are single-run point estimates with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares three code generators (CodeBERT, GitHub Copilot, Amazon CodeWhisperer) and also compares fine-tuned vs zero-shot performance. Additionally, it references general-purpose code generation performance (86.7%) as an external baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GitHub Copilot and Amazon CodeWhisperer were contemporary commercial AI code generators at the time of writing (2024). CodeBERT, while from 2020, is used as a fine-tunable open model and justified as 'representative of the state-of-the-art.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison of zero-shot learning vs fine-tuning (Figure 1) serves as an ablation showing the contribution of fine-tuning. The three granularity levels (line, block, function) also ablate the effect of description granularity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: edit distance (ED). The paper explicitly justifies not using compilation accuracy but does not employ any additional semantic similarity metrics. The paper references their prior work on metric correlation but only reports ED in this study."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges that 'manual code review where a human evaluator checks if the code generated by the models is semantically correct' is the 'golden standard' but states it is 'often unfeasible.' No human evaluation was performed. Table 3 provides cherry-picked qualitative examples but this is not systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We split the dataset into sets for training (the samples for fine-tuning the model), validation (to tune the hyperparameters of the models), and test (for the evaluation), using a random selection with the common 80%-10%-10% ratio.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by code granularity level (individual lines, multi-line blocks, entire functions) and Table 1 shows the dataset distribution across four security categories (Penetration Testing, Forensic Analysis, Network Traffic Analysis, OSINT and Social Engineering)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 includes qualitative examples showing where CodeBERT and CodeWhisperer fail (e.g., CodeBERT generating 'something close to the expected output, yet incomplete' and CodeWhisperer generating 'a verbose snippet that is syntactically correct but that diverges from the natural language description')."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that fine-tuning does not boost CodeBERT performance for blocks and functions compared to Copilot, and that CodeWhisperer shows lower performance. The paper also notes that without fine-tuning, performance is much lower."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'LLMs can be close, but not fully match the accuracy of LLMs at generating general-purpose programs' (supported by comparing 77.89% to 86.7%), that fine-grained descriptions give best results (supported by Figure 1-2), and that fine-tuning CodeBERT helps (supported by Figure 1). These are all supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that fine-tuning improves performance, demonstrated through a controlled comparison of zero-shot vs fine-tuned CodeBERT on the same test set (Figure 1). The ablation across granularity levels also uses controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'AI Code Generators for Security: Friend or Foe?' is broad, but the study only tests Python code from a single book ('Violent Python'), using a single similarity metric (ED), on three specific tools. The conclusion generalizes to 'cybersecurity professionals must embrace AI code generators' without bounding to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, it does not consider whether CodeBERT's advantage could be due to the ED metric favoring shorter outputs, whether the dataset composition biases results, or whether CodeWhisperer's lower performance could be due to API configuration differences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GitHub Copilot', 'Amazon CodeWhisperer', and 'CodeBERT' without specifying exact versions, snapshot dates, or API versions. No model version identifiers are provided for any of the three systems."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides concrete examples of the natural language intents used as prompts at all three granularity levels (line, block, function). Since the dataset is released, the full set of prompts is available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any of the three models. Temperature, top-p, max tokens, learning rate, number of epochs for fine-tuning, and other training parameters are all absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions running CodeBERT 'along with data processing operations, both before translation to prepare the input data, and after translation to improve the quality and the readability of the code in output' but does not describe these processing operations in detail."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that the dataset was built from the 'Violent Python' book and that descriptions were 'manually described' in three granularity levels, but does not document specific preprocessing steps, annotation guidelines, or how the 80/10/10 split was performed beyond 'random selection.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations, threats to validity, or similar section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of scope boundaries, such as limitation to Python, to a single book source, or to specific model versions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The violent-python dataset is publicly available at https://github.com/dessertlab/violent-python, allowing independent verification of the data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes that the dataset was built from the book 'Violent Python' by T.J. O'Connor, covers four security areas, and that descriptions were 'manually described' in three alternative granularities based on 'the contents of the chapter around each script, and on comments in the code where available.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study evaluates AI code generators on a benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document the full pipeline from the book to the final dataset. It mentions manual annotation but does not describe how many annotators were involved, what quality control was applied, or how the final 1,372 samples were selected from the book's content."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work has been partially supported by MUR PRIN 2022, project FLEGREA, CUP E53D23007950001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Naples Federico II and University of North Carolina at Charlotte. None of the authors appear affiliated with the companies whose products are evaluated (GitHub/Microsoft, Amazon)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is MUR PRIN 2022 (Italian Ministry of University and Research), which has no financial stake in the performance of any of the evaluated AI code generators."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the three models (CodeBERT, GitHub Copilot, Amazon CodeWhisperer). This is relevant because the Violent Python book (2012) could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Copilot or CodeWhisperer may have seen the Violent Python book's code during pre-training, which is publicly available and from 2012. This is a significant contamination concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Violent Python book was published in 2012 and its code examples are publicly available online. Both Copilot and CodeWhisperer were trained on public code repositories, creating a significant contamination risk that is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported for any of the three code generators."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is mentioned (8 vCPU, 16 GB RAM, two NVIDIA T4 GPUs) but no total compute budget, training time, or GPU hours are stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuning CodeBERT on offensive code significantly outperforms zero-shot learning, especially for individual lines (19.05% vs 77.89% ED).",
       "evidence": "Figure 1 shows ED scores for zero-shot vs fine-tuned CodeBERT across three granularity levels: lines (19.05% vs 77.89%), blocks (18.23% vs 43.45%), functions (22.31% vs 37.14%).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned CodeBERT provides the best performance for single-line generation (77.89%) compared to Copilot (65.38%) and CodeWhisperer (60.76%).",
       "evidence": "Figure 2 shows ED scores for all three generators on the same test set for single lines.",
       "supported": "moderate"
     },
     {
       "claim": "CodeBERT and Copilot have similar performance for blocks and functions, while CodeWhisperer shows lower performance.",
       "evidence": "Figure 2: blocks (43.45% vs 45.42% vs 38.76%), functions (37.14% vs 35.27% vs 31.31%).",
       "supported": "moderate"
     },
     {
       "claim": "Current LLMs can achieve performance close to general-purpose code generation for security-oriented individual line generation.",
       "evidence": "The paper compares 77.89% ED for fine-tuned CodeBERT on individual lines to 86.7% for general-purpose Python code generation (cited from reference [6]).",
       "supported": "weak"
     },
     {
       "claim": "CodeWhisperer's lower performance is attributed to its focus on Amazon-specific platforms and APIs.",
       "evidence": "The paper states 'CodeWhisperer, differently from a general purpose tool such as Copilot, caters first and foremost to development use cases associated with Amazon platforms.' This is speculation without supporting evidence.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Fine-tuning CodeBERT on security-oriented Python code from the 'Violent Python' book produces the best edit distance scores for individual line generation (77.89%), outperforming commercial tools GitHub Copilot (65.38%) and Amazon CodeWhisperer (60.76%). For more complex multi-line blocks and functions, the fine-tuning advantage diminishes, with Copilot matching CodeBERT. Performance decreases as code complexity increases (lines > blocks > functions) across all generators. The results suggest LLMs can approach general-purpose code generation accuracy (86.7%) for security code at fine granularity, but need fine-tuning for the security domain.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk",
       "detail": "The dataset is derived from the publicly available 2012 book 'Violent Python.' Both Copilot and CodeWhisperer were trained on public code repositories and could have seen this book's code examples during training. This would inflate their scores and is not discussed."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only edit distance (ED) is used to evaluate code quality. The paper itself acknowledges that manual review is the 'golden standard' but does not perform it. ED measures textual similarity, not semantic correctness — code that works differently but achieves the same goal would score poorly."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are point estimates without confidence intervals, error bars, or significance tests. The comparative claims (e.g., CodeBERT outperforms Copilot) have no statistical backing."
     },
     {
       "flag": "No model version specification",
       "detail": "None of the three AI code generators have specified versions. Copilot and CodeWhisperer are rapidly evolving commercial services; results could differ significantly across versions, making the results non-reproducible."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats to validity section, despite several significant methodological limitations (single metric, potential contamination, single data source, no version control on commercial APIs)."
     },
     {
       "flag": "Unfair comparison",
       "detail": "CodeBERT is fine-tuned on the training split from the same dataset used for evaluation, while Copilot and CodeWhisperer cannot be fine-tuned. This gives CodeBERT a structural advantage that makes direct comparison problematic."
     }
   ],
   "cited_papers": [
     {
       "title": "Can we generate shellcodes via natural language? An empirical study",
       "authors": ["P. Liguori", "E. Al-Hossami", "D. Cotroneo", "R. Natella", "B. Cukic", "S. Shaikh"],
       "year": 2022,
       "doi": "10.1007/s10515-022-00331-3",
       "relevance": "Directly studies AI code generation for security (shellcode), a core topic of LLM-based offensive code generation."
     },
     {
       "title": "EVIL: Exploiting Software via Natural Language",
       "authors": ["P. Liguori", "E. Al-Hossami", "V. Orbinato", "R. Natella", "S. Shaikh", "D. Cotroneo", "B. Cukic"],
       "year": 2021,
       "doi": "10.1109/ISSRE52982.2021.00042",
       "relevance": "Presents methods for generating exploits from natural language using neural machine translation, directly related to LLM-based code generation for security."
     },
     {
       "title": "LIMA: Less is More for Alignment",
       "authors": ["C. Zhou", "P. Liu", "P. Xu", "S. Iyer", "J. Sun"],
       "year": 2023,
       "arxiv_id": "2305.11206",
       "relevance": "Foundational LLM alignment work relevant to understanding fine-tuning with limited data for specialized tasks."
     },
     {
       "title": "Who Evaluates the Evaluators? On Automatic Metrics for Assessing AI-based Offensive Code Generators",
       "authors": ["P. Liguori", "C. Improta", "R. Natella", "B. Cukic", "D. Cotroneo"],
       "year": 2023,
       "doi": "10.1016/j.eswa.2023.120073",
       "relevance": "Evaluates similarity metrics for AI-generated security code, directly relevant to methodology of evaluating LLM code generation."
     },
     {
       "title": "IntelliCode Compose: Code Generation Using Transformer",
       "authors": ["A. Svyatkovskiy", "S. K. Deng", "S. Fu", "N. Sundaresan"],
       "year": 2020,
       "doi": "10.1145/3368089.3417058",
       "relevance": "Foundational work on transformer-based code generation, providing the 86.7% baseline referenced in this paper."
     },
     {
       "title": "ExploitGen: Template-Augmented Exploit Code Generation Based on CodeBERT",
       "authors": ["G. Yang", "Y. Zhou", "X. Chen", "X. Zhang", "T. Han", "T. Chen"],
       "year": 2023,
       "doi": "10.1016/j.jss.2022.111577",
       "relevance": "Uses CodeBERT for exploit generation with template augmentation, directly comparable methodology for security code generation."
     },
     {
       "title": "Harnessing GPT-4 for Generation of Cybersecurity GRC Policies: A Focus on Ransomware Attack Mitigation",
       "authors": ["T. R. McIntosh", "T. Liu", "T. Susnjak", "H. Alavizadeh", "A. Ng", "R. Nowrozy", "P. A. Watters"],
       "year": 2023,
       "doi": "10.1016/j.cose.2023.103424",
       "relevance": "Evaluates GPT-4 for generating cybersecurity policies, relevant to LLM applications in security."
     },
     {
       "title": "An Attacker's Dream? Exploring the Capabilities of ChatGPT for Developing Malware",
       "authors": ["Y. M. P. Pa", "S. Tanizaki", "T. Kou", "M. van Eeten", "K. Yoshioka", "T. Matsumoto"],
       "year": 2023,
       "doi": "10.1145/3607505.3607513",
       "relevance": "Studies ChatGPT's capability for malware development, directly relevant to LLM security misuse evaluation."
     },
     {
       "title": "From ChatGPT to ThreatGPT: Impact of Generative AI in Cybersecurity and Privacy",
       "authors": ["M. Gupta", "C. Akiri", "K. Aryal", "E. Parker", "L. Praharaj"],
       "year": 2023,
       "doi": "10.1109/ACCESS.2023.3300381",
       "relevance": "Survey on generative AI impact on cybersecurity including attack generation, directly relevant to the survey scope."
     },
     {
       "title": "GPThreats-3: Is Automatic Malware Generation a Threat?",
       "authors": ["M. Botacin"],
       "year": 2023,
       "doi": "10.1109/SPW59333.2023.00027",
       "relevance": "Evaluates AI code generators for malware generation, a closely related benchmark evaluation study."
     }
   ]
 }