ai-research-survey

scan.json (24572B)

---

 {
   "paper": {
     "title": "Codexity: Secure AI-assisted Code Generation",
     "authors": ["Sung Yong Kim", "Zhiyu Fan", "Yannic Noller", "Abhik Roychoudhury"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2405.03927",
     "doi": "10.48550/arXiv.2405.03927"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link (https://github.com/Codexity-APR/Codexity) and a Zenodo archive (https://doi.org/10.5281/zenodo.10572275) in the abstract section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo archive and GitHub repository are listed as 'Tool and Data' in the abstract, indicating the benchmark dataset of 751 vulnerable subjects is included."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (MacBook Pro 2019, Ubuntu 18.04 Server with RTX 4090) and model names but does not provide requirements.txt, Dockerfile, or detailed dependency/library version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The demo walkthrough (Appendix A) describes how to use the VSCode extension but not how to reproduce the experimental results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 2 report only point estimates (e.g., 15.9%, 39.3%) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Codexity prevents 60% of vulnerabilities compared to ChatGPT, but no statistical significance tests are performed to support comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage reductions with baseline context: 'reduction of 60% vulnerable programs' (from 75.9% to 15.9% for iteration repair), and 36.6% and 29.5% reductions for preshot variants. These provide enough context for effect size assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 90 prompts and 990 completions. No justification is given for why this sample size is adequate, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Table 2 shows single-run results. For preshot repair with temperature 0.2, no spread across multiple runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against ChatGPT (baseline without repair), FootPatch (a static-analysis-based APR tool), and GitHub Copilot in RQ2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GitHub Copilot and ChatGPT (gpt-3.5-turbo) are contemporary tools. FootPatch (2018) is older but is chosen for a specific reason: it also uses Infer for vulnerability detection, making it a relevant comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 examines the tradeoffs between the two repair strategies (iteration repair vs. preshot repair), which serves as an ablation of the framework's components. The preshot repair variants with different local models (StarCoder vs. SantaCoder) also function as ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: number/percentage of vulnerable codes generated, average generation time (seconds), and lines of code (LoC) in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is performed. All vulnerability detection relies on automated static analysis tools (Infer, CppCheck). Human evaluation of code quality or correctness beyond vulnerability would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The same 90 prompts used to construct the benchmark are used for all evaluations. There is no separation into development and test sets, and no held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a detailed breakdown of vulnerability categories (24 types including Null Dereference, Resource Leak, Buffer Overrun, etc.) with counts for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "In RQ2, the paper discusses FootPatch's failure to handle null dereferences due to incorrect variable name extraction. In RQ3, the paper acknowledges that preshot repair sometimes outputs comments instead of code (6.1% for StarCoder, 8.5% for SantaCoder), which are counted as vulnerable."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that preshot repair has a higher vulnerable code generation rate than iteration repair, and acknowledges that iteration repair introduces new vulnerabilities in some iterations, leading to additional repair rounds. The paper also notes preshot repair sometimes fails to produce code at all."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'Codexity can prevent 60% of the vulnerabilities being exposed to the software developer,' which is supported by Table 2 showing a reduction from 75.9% to 15.9% vulnerable code (iteration repair). The 751 vulnerable subjects from 90 prompts claim is also supported in Section 4.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims Codexity's repair strategies reduce vulnerabilities. This is demonstrated through controlled comparisons: same prompts processed with and without the repair framework, which constitutes adequate single-variable manipulation for this type of causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Secure AI-assisted Code Generation' implies broad applicability, but the evaluation is limited to C programs, a specific set of 90 prompts, and gpt-3.5-turbo. The paper mentions future work for Java/Python in the conclusion but does not bound its claims to C. The abstract also makes general claims about 'AI programming assistants' while only testing ChatGPT and two local models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the paper does not consider whether the improvements come from simply prompting ChatGPT a second time (regardless of vulnerability feedback), or whether the static analyzers might miss certain vulnerability types, creating a false sense of security."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the latest ChatGPT (gpt-3.5-turbo)' without specifying a snapshot date or exact version (e.g., gpt-3.5-turbo-0613). StarCoder and SantaCoder are identified by name and parameter count (15.5B, 1.1B) but without specific version hashes or dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the conversational prompt template used for vulnerability repair, including the system message structure with placeholders for vulnerable code and SAST feedback. The actual prompt structure is visible and reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are reported: 0 for iteration repair, 0.2 for StarCoder (following its authors' setup), 0 and 0.8 for benchmark construction. Maximum prediction tokens of 1024 and maximum iterations of 3 are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the Codexity workflow in detail in Section 3, including the iteration repair loop (Section 3.2) and preshot repair strategy (Section 3.3), with Figure 2 showing the workflow diagrams for both strategies."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Figure 3 describe the benchmark construction pipeline in detail: 403 posts collected from ShareGPT/StackOverflow using keywords, filtered to 124 vulnerable posts via static analysis, then further refined to 90 vulnerable prompts through a two-round detection strategy with ChatGPT at two temperature settings."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion briefly mentions future work directions (other languages, fine-tuning) but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. The conclusion mentions future work for Java/Python, implying the current scope is C only, but there is no explicit statement bounding the claims to C programs or specific vulnerability types detectable by CppCheck/Infer."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo archive (doi:10.5281/zenodo.10572275) and GitHub repository are provided, which should contain the benchmark data and tool for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection in detail: 403 posts from ShareGPT and the first 700 StackOverflow posts were collected using 'c' and 'int main' as keywords, then filtered through a two-round vulnerability detection strategy."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses publicly available code prompts from ShareGPT and StackOverflow, which are standard public data sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Section 4.1 document the full pipeline: 403 posts → static analysis → 124 vulnerable posts → ChatGPT completions at two temperatures (1,364 completions) → static analysis → 90 vulnerable prompts with 751 vulnerable completions. Counts are provided at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was partially supported by a Singapore Ministry of Education (MoE) Tier 3 grant \"Automated Program Repair\", MOE-MOET32021-0001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: National University of Singapore and Singapore University of Technology and Design. No affiliations with evaluated products (ChatGPT/OpenAI, GitHub Copilot/Microsoft)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Singapore Ministry of Education, which is a government funding agency with no financial stake in the specific outcomes of the evaluation of commercial LLM tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-3.5-turbo, StarCoder, and SantaCoder but does not state the training data cutoff dates for any of these models. Since the benchmark uses StackOverflow posts that could be in the training data, this is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from StackOverflow and ShareGPT posts, which are publicly available and likely in the training data of gpt-3.5-turbo. This potential overlap is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The StackOverflow posts used to construct the benchmark were publicly available before the training cutoff of gpt-3.5-turbo. ShareGPT data similarly predates the model. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study. The evaluation is entirely automated using benchmarks and static analysis tools."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports average generation time (e.g., 82.8s for iteration repair, 45.4s for StarCoder preshot) but does not report API costs, tokens consumed, or cost per example. RQ3 discusses cost tradeoffs qualitatively but provides no dollar amounts or token counts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (MacBook Pro 2019, Ubuntu 18.04 with RTX 4090) but total computational budget (GPU hours, total API spend, total experiment time) is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Codexity with iteration repair prevents 60% of vulnerabilities compared to ChatGPT baseline.",
       "evidence": "Table 2 shows ChatGPT generated 751/990 (75.9%) vulnerable codes, while IR-ChatGPT generated 157/990 (15.9%), a reduction of 60 percentage points (Section 4.2, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "Preshot repair with StarCoder reduces vulnerable code generation by 36.6% compared to ChatGPT.",
       "evidence": "Table 2 shows PR-StarCoder-15.5B generated 389/990 (39.3%) vulnerable codes vs. 751/990 (75.9%) for ChatGPT (Section 4.2, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "Preshot repair with SantaCoder reduces vulnerable code generation by 29.5% compared to ChatGPT.",
       "evidence": "Table 2 shows PR-SantaCoder-1.1B generated 459/990 (46.4%) vulnerable codes vs. 751/990 (75.9%) for ChatGPT (Section 4.2, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "FootPatch detected 20 vulnerabilities but patched none on the benchmark.",
       "evidence": "Section 4.2, RQ2 describes FootPatch finding 19 null dereferences and one memory leak but failing to patch any due to incorrect variable identification.",
       "supported": "strong"
     },
     {
       "claim": "GitHub Copilot generates vulnerable code for 84.44% of the 90 prompts.",
       "evidence": "Section 4.2, RQ2 states Infer and CppCheck found 76 (84.44%) of Copilot's programs to be vulnerable.",
       "supported": "moderate"
     },
     {
       "claim": "Codexity is the first security-focused code generation framework.",
       "evidence": "Stated in Section 1 introduction. The related work section (Section 5) surveys prior approaches but none integrate SAST tools directly into the code generation loop for IDE use.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Codexity integrates LLMs with static analysis tools (Infer, CppCheck) to reduce security vulnerabilities in AI-generated code. The iteration repair strategy reduces vulnerable code generation from 75.9% to 15.9% (60 percentage point reduction) on a benchmark of 990 code completion attempts derived from 90 vulnerable prompts. Preshot repair, which uses a cheaper local model to anticipate vulnerabilities before querying the commercial LLM, achieves 36.6% (StarCoder) and 29.5% (SantaCoder) reductions with faster generation times. The approach is limited to C code and relies on static analyzers' detection capabilities.",
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All results are single-run point estimates with no confidence intervals, significance tests, or variance reporting. The 60% claim rests entirely on comparing two raw percentages."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations or threats to validity, which is a significant omission for a tool paper making effectiveness claims."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The benchmark is constructed from StackOverflow and ShareGPT posts that are likely in gpt-3.5-turbo's training data. The model may already know these specific code patterns, affecting the validity of vulnerability generation rates."
     },
     {
       "flag": "Circular vulnerability definition",
       "detail": "Vulnerabilities are defined as whatever CppCheck and Infer detect. The paper evaluates success by whether these same tools report no vulnerability after repair. This means the evaluation only measures whether the LLM can produce code that passes the same static analyzers, not whether the code is actually secure."
     },
     {
       "flag": "Overbroad generalization",
       "detail": "The title and abstract imply general 'secure AI-assisted code generation' but the evaluation is limited to C programs, a specific set of CWE types detectable by two static analyzers, and a single commercial LLM (gpt-3.5-turbo)."
     },
     {
       "flag": "No correctness evaluation",
       "detail": "The paper evaluates only whether vulnerabilities are removed but does not check whether the repaired code is functionally correct. The repair process could introduce functional bugs while removing vulnerabilities."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "doi": "10.1109/SP46214.2022.9833571",
       "relevance": "Foundational work assessing security vulnerabilities in Copilot-generated code, directly motivating Codexity."
     },
     {
       "title": "How Secure is Code Generated by ChatGPT?",
       "authors": ["Raphaël Khoury", "Anderson R Avila", "Jacob Brunelle", "Baba Mamadou Camara"],
       "year": 2023,
       "arxiv_id": "2304.09655",
       "relevance": "Evaluates security of ChatGPT-generated code, providing evidence for the vulnerability problem Codexity addresses."
     },
     {
       "title": "Security Weaknesses of Copilot Generated Code in GitHub",
       "authors": ["Yujia Fu", "Peng Liang", "Amjed Tahir", "Zengyang Li", "Mojtaba Shahin", "Jiaxin Yu"],
       "year": 2023,
       "arxiv_id": "2310.02059",
       "relevance": "Studies security weaknesses in Copilot-generated code found in real GitHub repositories."
     },
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad", "Ramesh Karri", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "doi": "10.1109/SP46215.2023.10179324",
       "relevance": "Evaluates LLMs for vulnerability repair using CodeQL, directly related to using LLMs for security fixing."
     },
     {
       "title": "A New Era in Software Security: Towards Self-Healing Software via Large Language Models and Formal Verification",
       "authors": ["Yiannis Charalambous", "Norbert Tihanyi", "Ridhi Jain", "Youcheng Sun", "Mohamed Amine Ferrag", "Lucas C Cordeiro"],
       "year": 2023,
       "arxiv_id": "2305.14752",
       "relevance": "Explores LLMs combined with formal verification for automated vulnerability repair."
     },
     {
       "title": "Large Language Models for Code: Security Hardening and Adversarial Testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "Proposes learning strategies for improved secure code generation with LLMs."
     },
     {
       "title": "Demystifying GPT Self-Repair for Code Generation",
       "authors": ["Theo X Olausson", "Jeevana Priya Inala", "Chenglong Wang", "Jianfeng Gao", "Armando Solar-Lezama"],
       "year": 2023,
       "arxiv_id": "2306.09896",
       "relevance": "Studies LLM self-repair for code generation, relevant to iterative repair approaches."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "One of the code generation models used in Codexity's preshot repair strategy."
     },
     {
       "title": "SantaCoder: don't reach for the stars!",
       "authors": ["Loubna Ben Allal", "Raymond Li", "Denis Kocetkov"],
       "year": 2023,
       "arxiv_id": "2301.03988",
       "relevance": "Another code generation model used in Codexity's preshot repair strategy."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Discusses using multiple LLMs to balance cost and quality, relevant to Codexity's multi-model approach."
     },
     {
       "title": "Trust Enhancement Issues in Program Repair",
       "authors": ["Yannic Noller", "Ridwan Shariffdeen", "Xiang Gao", "Abhik Roychoudhury"],
       "year": 2022,
       "doi": "10.1145/3510003.3510040",
       "relevance": "Discusses trust issues in automated program repair, relevant to reliability of AI-assisted code fixes."
     }
   ]
 }