ai-research-survey

scan.json (28092B)

---

 {
   "paper": {
     "title": "Bugs in Large Language Models Generated Code: An Empirical Study",
     "authors": [
       "Florian Tambon",
       "Arghavan Moradi Dakhel",
       "Amin Nikanjam",
       "Foutse Khomh",
       "Michel C. Desmarais",
       "Giuliano Antoniol"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2403.08937"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package is referenced as [18] and available at https://github.com/FlowSs/BugsInLLMs. The paper states: 'We make the dataset used in this study publicly available online [18] for other researchers and practitioners to replicate or build upon our work.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset and all collected artifacts are shared via the replication package at [18]. The paper states: 'All collected artifacts and data generated during our study are accessible in our replication package [18].' Survey responses and labeling data are included."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, or dependency list with versions) is mentioned in the paper. The study involves manual labeling and a survey; the paper references specific versions of the CoderEval dataset but does not specify a computational environment for reproducing the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the replication package is provided, the paper does not include step-by-step reproduction instructions. The methodology is described in detail for the manual labeling and survey process, but there are no specific scripts or commands to replicate the analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports percentages and weighted averages for bug patterns and survey results but does not include confidence intervals or error bars on these figures. The sampling is stated at '95% confidence interval and an error rate of 5%' for sample size justification, but no CIs are reported on the actual results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Spearman's Rho correlation to test the relationship between survey-reported bug frequency and the distribution observed in the sample set (Section 3.2.3, Section 4.2.2). The correlation values (rho = 0.47 for Codex, 0.28 for PanGu, -0.18 for CodeGen) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Spearman's Rho values are reported with context (e.g., 'ρ = 0.47, medium correlation' for Codex), which provides effect size information. The percentage distributions of bug patterns across models also provide baseline context for interpreting magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The sample size is explicitly justified in Section 3.1.1: 'At a 95% confidence interval and an error rate of 5%, this necessitated the analysis of 323 samples. We sampled 333 code fragments.' The survey sample size (34 responses) is contextualized against the 382 invitations sent."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the weighted average Likert-scale scores or the bug pattern distributions. Only point estimates (percentages and weighted averages) are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares bug patterns across three LLMs (CodeGen, PanGu-Coder, and Codex), and relates findings to prior bug taxonomies from human-written code and prior studies (Fan et al., Liu et al., Jesse et al.). The survey validation also serves as a baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The three LLMs studied (CodeGen, PanGu-Coder, and Codex) are from 2022 and earlier. At the time of the 2024 publication, more capable models like GPT-4, Claude, and Code Llama were available and widely used. The paper acknowledges this as a limitation tied to the CoderEval dataset version used."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a taxonomy construction and qualitative analysis study, not a system with components to ablate. The study characterizes bug patterns rather than proposing a method with separable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study uses multiple metrics: bug pattern distribution percentages across models, inter-rater agreement (78.2%), Spearman correlation between survey and sample set frequencies, and four Likert-scale dimensions (frequency, diagnosing, complexity, fixing) for survey validation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The taxonomy was validated through an online survey with 34 practitioners and researchers (Section 3.2, Section 4.2). Participants assessed each bug pattern on frequency, diagnosing difficulty, complexity, and fixing difficulty using Likert scales."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning evaluation study; there is no train/test split. The study constructs a taxonomy through manual analysis, so a held-out test set is not applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-category breakdowns are provided in Table 2 (bug pattern distribution per LLM), Table 3 (bug pattern distribution per runnable level), Figure 3 (heatmap per task), and Table 4/Figure 4 (survey results per bug pattern)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure cases through illustrative code examples (Listings 1-10) for each bug pattern. Specific failure modes like prompt ambiguity leading to Misinterpretation and version mismatches are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the correlation between survey responses and sample distributions was low for some models (rho = -0.18 for CodeGen) and discusses discrepancies between survey results and sample-based findings (e.g., Prompt-biased Code being more prevalent in survey responses than in CoderEval)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 10 bug patterns from 333 bugs across 3 LLMs validated by 34 practitioners are all supported in the results. The claim that participants 'generally asserted the significance and prevalence' is supported by the survey results in Section 4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is mostly descriptive and avoids strong causal claims. Where causal-like language appears (e.g., 'limited information in the prompt can drastically lead the model astray'), it is presented as observations with illustrative examples rather than tested causal relationships. The taxonomy is presented as descriptive rather than explanatory."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to the three studied LLMs and Python code from CoderEval. Section 7 (Threats to Validity) explicitly notes limitations to Python, the three specific LLMs, and the CoderEval dataset. The external validity section notes: 'future works should consider expanding our study to cover a more diverse set of LLMs and functions.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for observed patterns. For instance, it considers whether prompt ambiguity rather than LLM weakness drives certain bugs, whether runnable levels bias results, and why survey frequencies differ from sample frequencies (e.g., CoderEval's limited docstrings vs. practitioners' richer prompts). Section 4.2.2 discusses how prompt type explains discrepancies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names the three LLMs (CodeGen, PanGu-Coder, Codex) and provides parameter count ranges but does not specify exact model versions, snapshot dates, or checkpoints used. For CodeGen, 350M to 16.1B parameter variants are mentioned but the specific variant used is not stated. For Codex, the version is not specified beyond 'Codex.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompts are the docstrings from CoderEval functions, which are shown in the code listings (Listings 1-10) throughout the paper. The paper states code was generated 'using the docstring' as prompt, and the exact CoderEval dataset version is linked (commit hash ec1177750cf10b5faa414a0e76d1430e75141a44)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, sampling strategy) are reported for the LLM code generation. The paper uses code generated within the CoderEval dataset but does not mention the generation settings used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLMs generate code directly from prompts (docstrings) in a single-turn generation setting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data filtering pipeline is well documented in Section 3.1.1: starting from 6,900 Python samples, filtering by runnable level to 1,997, then filtering by buggy flag, then sampling 333. Filtering criteria at each stage and the rationale are clearly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated and substantive discussion organized into construct, internal, external, and conclusion validity threats, spanning approximately 2 pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: the limitation to Python, the three specific LLMs, the CoderEval dataset's prompt style, the 8.9% response rate for the survey, the focus on runnable levels up to plib_runnable, and the risk of missing bug patterns due to sampling. These are concrete and study-specific."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: limited to Python only, three specific LLMs, CoderEval tasks, runnable levels up to plib_runnable. Section 7 notes: 'future works should consider expanding our study to include other programming languages' and 'to cover a more diverse set of LLMs.' The paper also notes that type-based bugs and memory management issues cannot be detected in Python."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at [18] (https://github.com/FlowSs/BugsInLLMs) includes the labeled buggy code samples, the labeling data, and anonymized survey responses, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 describes the data collection in detail: the CoderEval dataset source, the specific GitHub commit used, the filtering criteria (runnable levels, buggy flag), the sampling method (95% CI, 5% error rate), and balanced distribution across LLMs."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes participant recruitment: collecting emails from GitHub collaborators on LLM code repositories (200 emails via PyDriller), researcher emails from Google Scholar papers (182 emails from 56 papers), and Reddit posts on LocalLLaMA and MachineLearning channels. Total: 382 unique email addresses."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented from CoderEval (6,900 Python samples) → filtering by runnable level (1,997) → filtering by buggy flag → sampling (333). The labeling process is documented in Table 1 with initial agreements and conflicts per round. Survey analysis methodology is described in Section 3.2.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the paper's first page footnote: 'This work was supported by: Fonds de Recherche du Québec (FRQ), the Canadian Institute for Advanced Research (CIFAR) as well as the DEEL project CRDPJ 537462-18 funded by NSERC and CRIAQ, together with its industrial partners Thales Canada inc, Bell Textron Canada Limited, CAE inc and Bombardier inc.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Polytechnique Montréal, which is clearly stated. The authors are academics and do not work for any of the LLM companies whose products are evaluated (OpenAI for Codex, Huawei for PanGu-Coder, Salesforce for CodeGen)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (FRQ, CIFAR, NSERC, CRIAQ, Thales, Bell Textron, CAE, Bombardier) are not producers of the evaluated LLMs and have no direct financial interest in the bug taxonomy findings."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. There is no declaration of conflicts of interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs (CodeGen, PanGu-Coder, Codex) on the CoderEval benchmark but does not state the training data cutoff dates for any of the three models. This is relevant because CoderEval tasks are from GitHub and could potentially overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap between the CoderEval benchmark tasks and the LLMs' training data. The CoderEval tasks were extracted from GitHub repositories, which could have been in the training data of the models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the CoderEval functions could have appeared in the training data of the three LLMs. Since all three models were trained on GitHub code and CoderEval is sourced from GitHub, this is a relevant contamination risk that is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the survey study. There is no link to OSF, AsPredicted, or any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the survey involving 34 human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 reports participant demographics: job titles (12 PhD students, 4 researchers, 8 undergrad/grad students, 1 lecturer, 6 developers, 2 data scientists, 1 CTO), LLMs used, and programming languages used."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes inclusion criteria: participants were either GitHub users who collaborated on repositories containing LLM-generated code, or researchers who published on LLM code generation. The selection methodology is described in detail."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control groups. The survey is cross-sectional, collecting perceptions from all participants about the same bug patterns. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study collecting participants' perceptions of bug patterns. Blinding is not applicable to this design."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: 382 surveys sent, 34 responses received (8.9% response rate). Reddit distribution is also mentioned. The response rate is contextualized against similar SE surveys (8.4% to 13.3%)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This study analyzes existing LLM-generated code from the CoderEval dataset rather than proposing a new method with inference costs. The paper is a taxonomy construction study, not a system proposal."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the manual labeling effort: 'The entire process required approximately 108 person-hours' (Section 3.1.2). While this is human effort rather than compute cost, it quantifies the resource budget for the main analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "10 distinctive bug patterns were identified in LLM-generated code from a sample of 333 bugs across three LLMs (CodeGen, PanGu-Coder, Codex).",
       "evidence": "Section 4.1.1 presents the full taxonomy with percentages per category. Table 2 shows distribution across models. Figure 2 visualizes the taxonomy.",
       "supported": "strong"
     },
     {
       "claim": "Misinterpretation is the most common bug pattern overall (20.77%), while Missing Corner Case is the most common for the stronger model Codex (23.53%).",
       "evidence": "Table 2 in Section 4.1.2 provides exact percentages per model and overall. The difference between stronger (Codex) and weaker models (CodeGen, PanGu-Coder) is discussed.",
       "supported": "strong"
     },
     {
       "claim": "Several LLM bug patterns (Hallucinated Object, Wrong Attribute, Silly Mistake) are different from bugs typically found in human-written code.",
       "evidence": "Section 4.1.2 discusses how these patterns would rarely occur in human code, supported by the observation that IDEs and linters would catch many of these. Survey respondents rated these patterns as low complexity (unlikely for human developers).",
       "supported": "moderate"
     },
     {
       "claim": "The frequency of bug patterns reported by survey respondents correlates with the distribution observed for Codex (Spearman rho = 0.47).",
       "evidence": "Section 4.2.2 reports the Spearman correlation: rho = 0.47 (medium) for Codex, 0.28 for PanGu-Coder, -0.18 for CodeGen.",
       "supported": "moderate"
     },
     {
       "claim": "Missing Corner Case bugs are the hardest to diagnose according to survey respondents (score > 3 on 5-point Likert scale).",
       "evidence": "Figure 4 and Section 4.2.3 show Missing Corner Cases has a diagnosing score of 3.35, the highest among all categories. Participant quotes support this finding.",
       "supported": "strong"
     },
     {
       "claim": "Bug patterns are evenly distributed over different tasks and runnable levels in the CoderEval dataset.",
       "evidence": "Figure 3 (heatmap) and Table 3 (runnable levels) show generally even distribution with few exceptions. Some trends are noted (Missing Corner Case increases with runnable level).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "observational"
   ],
   "key_findings": "The study identifies 10 distinctive bug patterns in LLM-generated code through manual analysis of 333 buggy code samples from CodeGen, PanGu-Coder, and Codex on the CoderEval benchmark. Misinterpretation is the most common pattern overall (20.77%), while stronger models like Codex show more Missing Corner Case bugs (23.53%). Several bug patterns (Hallucinated Object, Wrong Attribute, Silly Mistake) are qualitatively different from typical human-made bugs. A validation survey with 34 practitioners confirms the relevance of the taxonomy, with respondents reporting similar bug frequencies especially for Codex-like models (Spearman rho = 0.47).",
   "red_flags": [
     {
       "flag": "Outdated models evaluated",
       "detail": "The three LLMs studied (CodeGen, PanGu-Coder, Codex) are from 2022 and earlier. By the 2024 publication date, much more capable models (GPT-4, Claude, Code Llama) were widely available. The paper acknowledges this but the taxonomy may not fully represent current LLM bug patterns."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The CoderEval tasks are sourced from GitHub repositories, and all three models were trained on GitHub code. The paper does not discuss whether the models may have seen these exact functions during training, which could affect which bugs they generate."
     },
     {
       "flag": "Small survey sample with low response rate",
       "detail": "The validation survey has only 34 respondents from 382 invitations (8.9% response rate). While contextualized against similar SE surveys, this limits the statistical power of the survey-based validation."
     },
     {
       "flag": "No IRB/ethics approval mentioned",
       "detail": "The survey involved 34 human participants but no IRB or ethics board approval is mentioned."
     },
     {
       "flag": "Missing hyperparameters for LLM generation",
       "detail": "The paper uses code generated by the CoderEval dataset but does not report the generation hyperparameters (temperature, top-p, sampling strategy), which can significantly affect the types of bugs generated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the Codex model and HumanEval benchmark, foundational to evaluating LLM code generation capabilities."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["H. Yu", "B. Shen", "D. Ran"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "The benchmark dataset used in this study, designed for evaluating LLMs on real-world programming tasks rather than competitive programming."
     },
     {
       "title": "Github copilot ai pair programmer: Asset or liability?",
       "authors": ["A. Moradi Dakhel", "V. Majdinasab", "A. Nikanjam"],
       "year": 2023,
       "doi": "10.1016/j.jss.2023.111734",
       "relevance": "Assesses code quality of Copilot-generated code and compares repair effort with human-written code."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Evaluates correctness of ChatGPT-generated code and shows existing test cases may miss LLM-specific bugs."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Z. Fan", "X. Gao", "M. Mirchev", "A. Roychoudhury", "S. H. Tan"],
       "year": 2023,
       "relevance": "Studies automated program repair for LLM-generated buggy code, finding Codex outperforms APR tools at fixing its own bugs."
     },
     {
       "title": "Lost in translation: A study of bugs introduced by large language models while translating code",
       "authors": ["R. Pan", "A. R. Ibrahimzada", "R. Krishna"],
       "year": 2023,
       "arxiv_id": "2308.03109",
       "relevance": "Develops a bug taxonomy for LLM-generated code in code translation tasks, complementary to this study's code generation focus."
     },
     {
       "title": "Refining chatgpt-generated code: Characterizing and mitigating code quality issues",
       "authors": ["Y. Liu", "T. Le-Cong", "R. Widyasari"],
       "year": 2023,
       "relevance": "Analyzes code quality issues in ChatGPT-generated code using static analysis tools on LeetCode problems."
     },
     {
       "title": "Large language models and simple, stupid bugs",
       "authors": ["K. Jesse", "T. Ahmed", "P. T. Devanbu", "E. Morgan"],
       "year": 2023,
       "relevance": "Examines LLM propensity to generate simple bugs (SStuBs) and the difficulty of distinguishing them from correct code."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["P. Vaithilingam", "T. Zhang", "E. L. Glassman"],
       "year": 2022,
       "relevance": "User study showing Copilot users had lower task success rates, relevant to understanding LLM bug impact on developer productivity."
     },
     {
       "title": "Taxonomy of real faults in deep learning systems",
       "authors": ["N. Humbatova", "G. Jahangirova", "G. Bavota"],
       "year": 2020,
       "relevance": "Establishes methodology for bug taxonomy construction in AI systems using open coding, which this paper adapts."
     },
     {
       "title": "Adaptive test generation using a large language model",
       "authors": ["M. Schäfer", "S. Nadi", "A. Eghbali", "F. Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "Explores LLM-based test generation, relevant to automated approaches for detecting bugs in LLM-generated code."
     },
     {
       "title": "CodaMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["C. Lemieux", "J. P. Inala", "S. K. Lahiri", "S. Sen"],
       "year": 2023,
       "relevance": "Combines LLMs with search-based testing to improve code coverage, relevant to testing LLM-generated code."
     }
   ]
 }