ai-research-survey

scan.json (26494B)

---

 {
   "paper": {
     "title": "CodeMirage: Hallucinations in Code Generated by Large Language Models",
     "authors": ["Vibhor Agarwal", "Yulong Pei", "Salwa Alamir", "Xiaomo Liu"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2408.08333"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states in a footnote on page 1: 'The dataset will be released upon acceptance.' No GitHub or Zenodo link is provided. A promise of future release counts as NO per schema rules."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same footnote: 'The dataset will be released upon acceptance.' The CodeMirage dataset is not currently available. The base datasets (HumanEval, MBPP) are public, but the novel artifact — the hallucinated code snippets — is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is provided. The paper mentions using CodeLLaMA-7B-Instruct and OpenAI's API but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes methodology at a high level but does not give enough detail to replicate without the dataset or code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports only point estimates for accuracy, macro-precision, macro-recall, and macro-F1. No confidence intervals, error bars, or ± notation is provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 'performs the best' and 'beats the fine-tuned CodeBERT model by 6.15 percentage macro-F1 score' but does not perform any significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides raw metric differences (e.g., '6.15 percentage macro-F1 score' improvement), no formal effect sizes (Cohen's d, odds ratios, etc.) are reported. The raw difference with baseline context is marginal — the paper does report 'from X to Y' implicitly via the table, but the 6.15pp difference is stated in isolation without systematic effect size reporting across conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the dataset size (1,137 snippets) or the 200-sample human annotation subset. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No standard deviation, variance across seeds, or multiple-run results are reported for any of the LLM experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple models: CodeLLaMA, GPT-3.5, GPT-4, and fine-tuned CodeBERT as baselines for code hallucination detection (Section 5.1, Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-4 (2023), CodeLLaMA (2023), and fine-tuned CodeBERT. For a 2024 paper, these are reasonably contemporary models representing both open-source and proprietary state-of-the-art."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims in contributions (Section 1) to 'conduct comprehensive experiments and ablation studies' but the results section (Section 5.3) only compares different models — it does not ablate components of the detection approach (e.g., removing the one-shot example, varying prompt structure). Comparing different models is not an ablation study."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports accuracy, macro-precision, macro-recall, and macro-F1 scores for all models."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes human annotation of 200 code snippets by 5 annotators to validate the dataset quality, with Cohen's kappa reported (0.76). This evaluates the quality of the LLM-generated hallucinated code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For CodeBERT, an 80:20 stratified train/test split is used (Section 5.1). The LLM models (GPT-3.5, GPT-4, CodeLLaMA) are evaluated with one-shot prompting on the full dataset without any tuning on it, so there is no train/test leakage for those."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 provides results broken down by base dataset (HumanEval vs. MBPP) but not by hallucination type. The paper does not report per-category (dead code, syntactic error, logical error, etc.) detection performance, which would be important for understanding where models struggle."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases or provide qualitative error analysis. It notes CodeLLaMA 'does not perform well' and GPT-3.5 has lower performance, but does not analyze why or show example failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CodeLLaMA achieves very low macro-F1 scores (0.0424 and 0.0271), and that GPT-3.5 also performs poorly. These are genuine negative results showing that not all LLMs are effective at this task."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are generally supported: it introduces a taxonomy, proposes the CodeMirage benchmark, and reports GPT-4 performs best on HumanEval and gives 'comparable results' to CodeBERT on MBPP. Table 5 confirms these claims. The 'comparable' hedging for MBPP is appropriate."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims through ablation language in contributions ('ablation studies to demonstrate the capabilities') and says that the gap between GPT-3.5 and GPT-4 is 'surprising,' implying model capability causes better detection. However, model comparisons alone without controlling for confounds (prompt sensitivity, token limits, etc.) do not adequately support causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Hallucinations in Code Generated by Large Language Models' broadly, but the study only covers Python code from HumanEval and MBPP, generated by GPT-3.5. The taxonomy claims to be 'comprehensive' but is validated only on Python. The paper does not bound its generalization to Python or to these specific benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, GPT-4's better performance could be partly due to the detection prompt being better suited to GPT-4, or the hallucinated code being generated by GPT-3.5 giving GPT-4 an advantage in detecting its patterns. No such confounds are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5', 'GPT-4', and 'CodeLLaMA-7B-Instruct' but does not specify exact model versions or snapshot dates for the OpenAI models (e.g., 'gpt-3.5-turbo-0613'). CodeLLaMA-7B-Instruct is specified by size and variant, which is more specific but still lacks a precise checkpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix A (generation prompts for each hallucination type) and Appendix B (detection prompt), including the layout in Tables 1 and 4. The actual example code within prompts is shown. The placeholder structure is clear (programming question inserted)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 reports temperature of 0.7 and max tokens of 256 for the detection experiments. However, hyperparameters for the generation phase (GPT-3.5 generating hallucinated code) are not reported — temperature, top-p, etc. for dataset construction are missing. CodeBERT fine-tuning hyperparameters (learning rate, epochs, etc.) are also not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is straightforward one-shot prompting without tool use, feedback loops, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes how hallucinated code was generated (Section 4.1) but does not describe any preprocessing, filtering, or quality control on the generated outputs beyond the human annotation validation on a 200-sample subset. It is unclear whether any generated outputs were discarded for quality reasons before the final 1,137 total."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper goes directly from results to 'Conclusions and Future Work' (Section 6) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no mention of potential biases from using GPT-3.5 to generate the hallucinated code, single-language limitation, or annotation limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not bound its claims to Python, to HumanEval/MBPP, or to GPT-3.5-generated code. The broad title suggests general applicability without such boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The CodeMirage dataset is not released ('will be released upon acceptance'). Raw annotations, LLM outputs, and individual predictions are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the dataset generation process in detail: base datasets (HumanEval, MBPP) are used, hallucination types are randomly assigned per problem, type-specific prompts are fed to GPT-3.5, and the output is collected. Section 4.2 describes the human annotation procedure."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 mentions '5 human annotators, well-versed in Python programming' but does not describe how they were recruited, their backgrounds, whether they were colleagues/students, or whether this introduces any bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The generation pipeline is described at a high level, but intermediate steps are unclear. For instance, it is not stated how many GPT-3.5 generations failed or were discarded, whether all 1,137 problems yielded exactly one hallucinated snippet each, or what the conversion rate was. The jump from 164+973=1,137 total problems to exactly 1,137 hallucinated snippets suggests one-to-one correspondence, but this is not explicitly confirmed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section listing funding sources. The disclaimer states it was prepared by JP Morgan AI Research but does not disclose specific funding or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Surrey and JP Morgan AI Research (London, UK and New York, USA). The first author's work was done during an internship at JP Morgan AI Research, as noted in the footnote."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work was done at JP Morgan AI Research, which is a corporate lab. JP Morgan uses LLMs for code generation in practice and has a stake in understanding and mitigating hallucinations. No discussion of whether this creates a conflict of interest. The paper does not evaluate JP Morgan products specifically, but the funder is not independent of the general research area."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement. The disclaimer section is a legal boilerplate about JP Morgan but does not address financial conflicts of interest related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-3.5, GPT-4, and CodeLLaMA on code from HumanEval and MBPP benchmarks but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) are well-known public benchmarks that could be in the training data of GPT-3.5, GPT-4, and CodeLLaMA. The paper does not discuss potential train/test overlap at all."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP were published in 2021, before the training cutoff of all models used. The fact that GPT-3.5 may have seen these problems could affect the quality and nature of the hallucinated code it generates, and GPT-4/CodeLLaMA may have seen the correct solutions, affecting detection performance. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human annotation task in Section 4.2 is dataset validation by expert annotators, not a human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are performing a professional coding task (labeling code snippets), not participating as research subjects. IRB approval is not typically required for expert annotation tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are described as 'well-versed in Python programming' but since this is expert annotation for dataset validation rather than a human subjects study, detailed demographics are not required."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "This is expert annotation, not a human subjects study. Annotator selection criteria are relevant but fall outside the human_studies category."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment was conducted. The annotation task does not involve treatment/control conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment was conducted requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment was conducted. The annotation task does not have meaningful attrition concerns."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report inference costs, tokens consumed, or wall-clock time for any of the experiments (neither dataset generation via GPT-3.5 nor hallucination detection via multiple LLMs)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours for CodeLLaMA/CodeBERT, API costs for GPT-3.5/GPT-4, or total training time for CodeBERT fine-tuning."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 performs the best on HumanEval dataset for code hallucination detection, beating fine-tuned CodeBERT by 6.15 percentage points in macro-F1.",
       "evidence": "Table 5 shows GPT-4 achieves macro-F1 of 0.5512 on HumanEval vs CodeBERT's 0.4897, a difference of 6.15pp (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 gives comparable results to fine-tuned CodeBERT on MBPP dataset.",
       "evidence": "Table 5 shows GPT-4 macro-F1 of 0.5195 on MBPP vs CodeBERT's 0.6344. GPT-4 has higher macro-Precision (0.6644 vs 0.6455) but lower macro-F1 (Section 5.3).",
       "supported": "weak"
     },
     {
       "claim": "The CodeMirage dataset's automatically assigned hallucination type labels are reliable, with 0.81 accuracy against human annotations.",
       "evidence": "Section 4.2 reports accuracy of 0.81 between automatic gold labels and majority human annotations on 200 sampled snippets, with Cohen's kappa of 0.76 among annotators.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first work studying hallucinations in LLM-generated code.",
       "evidence": "Section 1 states 'To the best of our knowledge, this is the first attempt at studying hallucinations in the code generated by LLMs.' Related work (Section 2) discusses adjacent but not identical problems.",
       "supported": "moderate"
     },
     {
       "claim": "There is a big performance gap between GPT-3.5 and GPT-4 for code hallucination detection.",
       "evidence": "Table 5 shows GPT-3.5 achieves macro-F1 of 0.2654 (HumanEval) and 0.2092 (MBPP) vs GPT-4's 0.5512 and 0.5195, roughly 2x difference (Section 5.3).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces CodeMirage, a benchmark of 1,137 GPT-3.5-generated hallucinated Python code snippets across 5 hallucination types (dead code, syntactic errors, logical errors, robustness issues, security vulnerabilities) built on HumanEval and MBPP. GPT-4 with one-shot prompting outperforms fine-tuned CodeBERT on HumanEval (macro-F1: 0.5512 vs 0.4897) but falls behind on MBPP (0.5195 vs 0.6344). CodeLLaMA-7B-Instruct performs very poorly on this task (macro-F1 < 0.05). Human annotators achieve Cohen's kappa of 0.76, and automatic labels align with human majority at 0.81 accuracy.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. This is a significant omission given the many potential confounds (single language, single generation model, contamination risk, etc.)."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "HumanEval and MBPP (both 2021) are almost certainly in the training data of GPT-3.5, GPT-4, and CodeLLaMA. This could affect both the quality of hallucinated code generation and the detection performance. The paper does not discuss this at all."
     },
     {
       "flag": "Broad title with narrow scope",
       "detail": "The title claims 'Hallucinations in Code Generated by Large Language Models' generally, but the study only covers Python, only two benchmarks (HumanEval, MBPP), and only GPT-3.5-generated hallucinated code. No bounds on generalization are stated."
     },
     {
       "flag": "Dataset not released",
       "detail": "The CodeMirage dataset is described but not available ('will be released upon acceptance'). This prevents independent verification or replication."
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "All results are single-run point estimates with no uncertainty quantification. Claims of one model outperforming another rest on raw number comparisons without statistical testing."
     },
     {
       "flag": "GPT-4 'comparable' to CodeBERT claim is generous",
       "detail": "On MBPP, GPT-4 macro-F1 (0.5195) trails CodeBERT (0.6344) by 11.5 percentage points. Calling this 'comparable' is generous framing. The paper highlights GPT-4's higher precision but the overall metric gap is substantial."
     },
     {
       "flag": "Generation model used for detection benchmark",
       "detail": "GPT-3.5 generates the hallucinated code and is then evaluated as a detector. This creates a potential confound: GPT-3.5's poor detection performance may partly reflect difficulty detecting its own idiosyncratic patterns, while GPT-4 may benefit from having a different architecture."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used as a base dataset in this paper, central to code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used as a base dataset in this paper for code generation evaluation."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as a baseline for hallucination detection experiments."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Pre-trained code model fine-tuned as a baseline for code hallucination detection."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Evaluates functional correctness of LLM-generated code with automated test generation, closely related to code quality assessment."
     },
     {
       "title": "LLM is like a box of chocolates: the non-determinism of ChatGPT in code generation",
       "authors": ["Shuyin Ouyang", "Jie M Zhang"],
       "year": 2023,
       "arxiv_id": "2308.02828",
       "relevance": "Studies non-determinism in LLM code generation, directly relevant to reliability and hallucination concerns."
     },
     {
       "title": "Purple llama CyberSecEval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Benchmark for cybersecurity evaluation of LLM coding assistants, related to security vulnerability hallucination type."
     },
     {
       "title": "Large language models of code fail at completing code with potential bugs",
       "authors": ["Tuan Dinh", "Jinman Zhao"],
       "year": 2023,
       "arxiv_id": "2306.03438",
       "relevance": "Studies buggy-code completion problem, directly relevant to understanding LLM code generation failures."
     },
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee"],
       "year": 2023,
       "relevance": "Foundational survey on LLM hallucinations that provides the text hallucination framework this paper extends to code."
     },
     {
       "title": "Siren's song in the AI ocean: A survey on hallucination in large language models",
       "authors": ["Yue Zhang", "Yafu Li"],
       "year": 2023,
       "arxiv_id": "2309.01219",
       "relevance": "Comprehensive survey on LLM hallucination taxonomy, directly relevant to this paper's extension to code hallucinations."
     },
     {
       "title": "Software vulnerability and functionality assessment using LLMs",
       "authors": ["Rasmus Ingemann Tuffveson Jensen", "Vali Tawosi", "Salwa Alamir"],
       "year": 2024,
       "arxiv_id": "2403.08429",
       "relevance": "Uses LLMs to assess software vulnerabilities, related to the security vulnerability hallucination detection task."
     },
     {
       "title": "AI-assisted coding: Experiments with GPT-4",
       "authors": ["Russell A. Poldrack", "Thomas Lu"],
       "year": 2023,
       "arxiv_id": "2304.13187",
       "relevance": "Empirical study of GPT-4 for coding tasks, directly relevant to understanding LLM code generation capabilities."
     }
   ]
 }