ai-research-survey

scan.json (23884B)

---

 {
   "paper": {
     "title": "A Systematic Literature Review of Code Hallucinations in LLMs: Characterization, Mitigation Methods, Challenges, and Future Directions for Reliable AI",
     "authors": [
       "Cuiyun Gao",
       "Guodong Fan",
       "Chun Yong Chong",
       "Shizhan Chen",
       "Chao Liu",
       "David Lo",
       "Zibin Zheng",
       "Qing Liao"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2511.00776"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, analysis scripts, or data extraction tools are mentioned or released. A survey can release its analysis scripts and extracted data tables but this one does not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The extracted study data (Table 2) and search results are described but not released as a downloadable dataset. No supplementary data files or repositories are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tooling specifications are provided. While this is a survey paper, any analysis scripts or tools used for data extraction and classification could have been documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the search, screening, or analysis process are provided beyond the methodology description in Section 3. The search strings and databases are stated but there is no replication package."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or perform statistical aggregation. No confidence intervals are applicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative statistical claims are made that would require significance testing. The paper is a narrative synthesis, not a meta-analysis."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental results are reported. The paper is a qualitative systematic review, not a meta-analysis with statistical aggregation."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with sample sizes are conducted. The number of reviewed papers (60) is a result of the search and screening process, not a statistical sample."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or quantitative aggregation is performed. This is a qualitative survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares itself against the prior survey by Lee et al. [67] in Section 1, listing four specific advantages. It also positions itself relative to NLP hallucination surveys."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is against Lee et al. [67] from 2025, which is the most recent and closely related survey on hallucinations in code generation LLMs."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system or method to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a system. It reviews metrics used by others (Section 7.4) but does not apply them."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. No system outputs are produced that would require human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are run; there is no test set to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-category breakdowns: publication venue types (Fig. 6a), contribution types (Fig. 6b), publication trends per year (Fig. 7), hallucination cause categories (Fig. 8), and mitigation method categories (Fig. 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Challenges and Opportunities) discusses limitations and failure modes of current approaches, including the difficulty of distinguishing hallucination from mistake (8.1.1), design pattern unawareness (8.1.4), and challenges with private data (8.1.6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings: 'no method excelling universally' for knowledge editing (Section 5.3.2), that 'most existing efforts...overlook the unique structural and semantic properties of code' (RQ3 summary), and that current benchmarks are limited to Python/Java (Section 7.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to (1) survey 60 papers, (2) define hallucination in code context, (3) review mitigation strategies, (4) review benchmarks, and (5) discuss challenges. All five are substantiated in Sections 3-8 respectively."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that synthesizes findings from reviewed papers. It does not make causal claims of its own about interventions or effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Threats to validity) explicitly bounds the scope: 'we limit the scope to publications from 2022 to 2025, which may result in the omission of some earlier foundational work' and acknowledges that the search covered only three databases."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As a pure survey/taxonomy paper with no empirical results of its own, there are no findings for which alternative explanations would apply. The paper presents no original empirical claims requiring alternative explanation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any LLMs in its methodology. It is a manual systematic literature review."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. The paper is a manual systematic review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or experiments are used. This is a systematic literature review."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full search and filtering pipeline: search strings are provided (Section 3.2), three databases are named, inclusion/exclusion criteria are listed (Section 3.3), inter-rater reliability is measured via Cohen's Kappa (0.59 pilot, 0.73 full), and Table 1 shows counts at each stage (67+46 total → 60+24 after dedup → 36+24 after review). Filtering criteria at each stage are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 is a dedicated 'Threats to validity' section with two subsections (9.1 Literature Selection Validity, 9.2 Data Analysis Validity) spanning approximately one page."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1 discusses specific threats: the temporal limit to 2022-2025 may miss earlier work, the search covered only three databases (ACM DL, IEEE Xplore, Web of Science), and terminology is not standardized. Section 9.2 notes that two independent annotators extracted data with cross-validation by a third reviewer, and acknowledges the risk of mis-classification due to inconsistent terminology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1 states: 'we limit the scope to publications from 2022 to 2025' and the search was restricted to three databases. Section 3.3 lists explicit exclusion criteria (grey literature excluded, extended journal versions preferred). The scope is bounded to code hallucination specifically."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The list of 60 reviewed papers is not released as a downloadable dataset. Individual references are cited in-text but no supplementary data file or spreadsheet with the extracted data (Table 2) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 provides the exact search strings used, names the three databases (ACM Digital Library, IEEEXplore, Web of Science), and states the search date (May 25, 2025). Section 3.3 lists inclusion/exclusion criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper reviews published literature, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows the full pipeline: total papers per database, total before deduplication (67 code + 46 NLP), after removing duplicates (61 + 46), and after review (36 + 24 = 60). Section 3.3 describes the two-author screening with Cohen's Kappa measurement and third-author tie-breaking."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology, Shandong Agriculture and Engineering University, Monash University Malaysia, Tianjin University, Chongqing University, Singapore Management University, and Sun Yat-sen University. None of the authors appear to be affiliated with companies whose products are being reviewed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed at all, so independence cannot be assessed. The absence of any funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. It does not evaluate any pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. No model evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. No model evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Paper inclusion/exclusion criteria are assessed under data_preprocessing_documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. It does not propose or run any method that incurs inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Code hallucination is a growing research area, with 20 publications in 2024 compared to 5 in 2023.",
       "evidence": "Figure 7(a) shows publication counts per year. Section 4.3 states: 'a notable rise in 2024 with 20 publications, compared to 5 in 2023.'",
       "supported": "moderate"
     },
     {
       "claim": "ArXiv preprints account for the largest portion (54%) of code hallucination publications, followed by conferences (30%) and journals (16%).",
       "evidence": "Figure 6(a) and Section 4.3: 'Preprints on arXiv account for the largest portion (54%), followed by conferences (30%) and journals (16%).'",
       "supported": "strong"
     },
     {
       "claim": "Most code hallucination mitigation approaches still adopt methods similar to NLP, overlooking code's unique structural and semantic properties.",
       "evidence": "RQ3 summary in Section 6: 'most of these approaches often overlook the unique structural and semantic properties of code' and 'Solutions specifically tailored to code hallucination...are still relatively scarce.'",
       "supported": "moderate"
     },
     {
       "claim": "Only one prior survey paper (1%) exists specifically on code hallucination, highlighting the lack of comprehensive overviews.",
       "evidence": "Figure 6(b) and Section 4.3: 'Only one survey paper (1%) exists, further highlighting the lack of comprehensive overviews.'",
       "supported": "moderate"
     },
     {
       "claim": "Current code hallucination benchmarks focus primarily on Python and Java, leaving low-resource or domain-specific languages underrepresented.",
       "evidence": "Section 7.2 and Table 7 show all listed benchmarks use Python or Python+Java. Section 7.2: 'these benchmarks focus primarily on a limited set of PLs, e.g., Python or Java, leaving low-resource or domain-specific languages underrepresented.'",
       "supported": "strong"
     },
     {
       "claim": "Inter-rater agreement for study selection was 'moderate' (Cohen's Kappa 0.59) in pilot and 'substantial' (0.73) for full assessment.",
       "evidence": "Section 3.3: 'The agreement rate in the pilot study is \"moderate\" (0.59)...The agreement rate in the full assessment is \"substantial\" (0.73).'",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This systematic literature review of 60 papers identifies three categories of code hallucination (knowledge/factuality, functional misalignment/faithfulness, and environmental compatibility) and traces their causes to model-level, prompt/input, training data, and project/environment factors. The review finds that most mitigation approaches still borrow from NLP without adequately addressing code-specific structural properties. Current benchmarks are narrowly focused on Python and Java, and the paper identifies key challenges including distinguishing hallucination from ordinary errors, the relationship between model confidence and hallucination, and the tension between generalization and factual accuracy.",
   "red_flags": [
     {
       "flag": "No quality assessment of reviewed studies",
       "detail": "The survey does not perform any structured quality assessment or risk-of-bias evaluation of the 60 reviewed papers. It summarizes and categorizes findings but does not evaluate whether the underlying studies are methodologically sound. This risks laundering the signal-to-noise ratio of its sources."
     },
     {
       "flag": "Moderate inter-rater agreement",
       "detail": "The Cohen's Kappa for full paper selection was 0.73 ('substantial'), but the pilot was only 0.59 ('moderate'). While this is reported transparently, the initial moderate agreement suggests the inclusion criteria were not sufficiently precise, introducing potential selection bias."
     },
     {
       "flag": "No replication package",
       "detail": "Despite following Kitchenham's SLR guidelines, the paper provides no downloadable dataset, extraction forms, or supplementary materials. The list of 60 papers must be reconstructed from in-text citations, making independent verification difficult."
     },
     {
       "flag": "No funding or competing interests disclosure",
       "detail": "The paper contains no acknowledgments section, no funding disclosure, and no competing interests statement. While the authors are all academic, the absence of any disclosure is a gap."
     }
   ],
   "cited_papers": [
     {
       "title": "Hallucination by Code Generation LLMs: Taxonomy, Benchmarks, Mitigation, and Challenges",
       "authors": ["Yunseo Lee", "John Youngeun Song", "Dongsun Kim", "Jindae Kim", "Mijung Kim", "Jaechang Nam"],
       "year": 2025,
       "arxiv_id": "2504.20799",
       "relevance": "Directly related prior survey on hallucinations in code generation LLMs, covering types, benchmarks, and mitigation strategies."
     },
     {
       "title": "Exploring and evaluating hallucinations in llm-powered code generation",
       "authors": ["Fang Liu", "Yang Liu", "Lin Shi", "Houkun Huang", "Ruifeng Wang", "Zhen Yang", "Li Zhang"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "Proposes the HALLUCODE benchmark for evaluating hallucination recognition in LLM-generated code."
     },
     {
       "title": "CodeHalu: Investigating code hallucinations in llms via execution-based verification",
       "authors": ["Yuchen Tian", "Weixiang Yan", "Qian Yang"],
       "year": 2025,
       "relevance": "Introduces the CodeHaluEval benchmark with 8,883 samples for systematic quantitative evaluation of code hallucinations."
     },
     {
       "title": "LLM hallucinations in practical code generation: Phenomena, mechanism, and mitigation",
       "authors": ["Ziyao Zhang", "Chong Wang", "Yanlin Wang"],
       "year": 2025,
       "relevance": "Empirical study of hallucination mechanisms and mitigation in repository-level code generation scenarios."
     },
     {
       "title": "Codemirage: Hallucinations in code generated by large language models",
       "authors": ["Vibhor Agarwal", "Yulong Pei", "Salwa Alamir", "Xiaomo Liu"],
       "year": 2024,
       "arxiv_id": "2408.08333",
       "relevance": "Introduces a code hallucination taxonomy and the CodeMirage benchmark dataset for evaluation."
     },
     {
       "title": "We Have a Package for You! A Comprehensive Analysis of Package Hallucinations by Code Generating LLMs",
       "authors": ["J. Spracklen", "R. Wijewickrama", "A. H. M. N. Sakib", "A. Maiti", "M. Jadliwala"],
       "year": 2024,
       "relevance": "Systematically evaluates package hallucinations in LLM-generated code across 576K samples and 16 LLMs, revealing supply chain threats."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["Lei Huang", "Weijiang Yu", "Weitao Ma"],
       "year": 2025,
       "relevance": "Comprehensive taxonomy and analysis of hallucinations in LLMs broadly, relevant as foundational NLP hallucination survey."
     },
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske"],
       "year": 2023,
       "relevance": "Foundational survey on hallucination in NLG covering definitions, evaluation metrics, and mitigation techniques."
     },
     {
       "title": "De-Hallucinator: Mitigating LLM Hallucinations in Code Generation Tasks via Iterative Grounding",
       "authors": ["Aryaz Eghbali", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2401.01701",
       "relevance": "Proposes iterative grounding with API references to mitigate hallucinations in LLM code generation."
     },
     {
       "title": "Towards Mitigating API Hallucination in Code Generated by LLMs with Hierarchical Dependency Aware",
       "authors": ["Yujia Chen", "Mingyu Chen", "Cuiyun Gao"],
       "year": 2025,
       "relevance": "Proposes MARIN framework for mitigating API hallucination through dependency-aware constrained decoding."
     },
     {
       "title": "Constrained decoding for secure code generation",
       "authors": ["Yanjun Fu", "Ethan Baker", "Yu Ding", "Yizheng Chen"],
       "year": 2024,
       "arxiv_id": "2405.00218",
       "relevance": "Proposes CODEGUARD+ benchmark and explores constrained decoding for secure code generation."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["S. G. Patil", "T. Zhang", "X. Wang", "J. E. Gonzalez"],
       "year": 2023,
       "relevance": "Fine-tuned LLaMA model for API call generation that outperforms GPT-4 and significantly reduces hallucinations."
     },
     {
       "title": "Bugs in large language models generated code: An empirical study",
       "authors": ["Florian Tambon", "Arghavan Moradi-Dakhel", "Amin Nikanjam", "Foutse Khomh"],
       "year": 2025,
       "relevance": "Empirical study identifying 10 distinctive bug patterns in LLM-generated code including Hallucinated Object pattern."
     },
     {
       "title": "Identifying and Mitigating API Misuse in Large Language Models",
       "authors": ["T. Y. Zhuo", "J. He", "J. Sun", "Z. Xing", "D. Lo", "J. Grundy", "X. Du"],
       "year": 2025,
       "relevance": "First comprehensive study of API misuse patterns in LLM-generated code with taxonomy of four misuse types."
     }
   ]
 }