ai-research-survey

scan.json (24035B)

---

 {
   "paper": {
     "title": "LLM Hallucinations in Practical Code Generation: Phenomena, Mechanism, and Mitigation",
     "authors": ["Ziyao Zhang", "Yanlin Wang", "Chong Wang", "Jiachi Chen", "Zibin Zheng"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.20550"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "LLM code hallucinations fall into three categories: Task Requirement Conflicts (43.5%), Factual Knowledge Conflicts (31.9%), and Project Context Conflicts (24.6%), with eight subcategories. Task Requirement Conflicts are the most prevalent across all six studied models. Four root causes are identified: training data quality, intention understanding capacity, knowledge acquisition capacity, and repository-level context awareness. A lightweight RAG-based mitigation consistently improves Pass@1 across all models, though improvements are modest (0.87-3.05 percentage points).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package URL is provided: https://github.com/DeepSoftwareAnalytics/LLMCodingHallucination, mentioned in the abstract and contributions section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available CoderEval benchmark and states the replication package includes 'code, data, and experimental results.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, dependency versions, etc.) are described in the paper. Model sizes are listed but no runtime environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The replication package is linked but no README or reproduction steps are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table I reports only point estimates for Pass@1 with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RAG-based mitigation improves all models but provides no significance tests for these differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports absolute percentage point improvements for each model (e.g., 'CodeLlama 2.17% → 5.22% (↑3.05%)'), providing baseline context for the effect size."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for using 230 coding tasks from CoderEval. No power analysis or discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported across the 10 generated code snippets per task. Only aggregate Pass@1 numbers shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The RAG-based mitigation is compared against a 'Raw Method' baseline (docstring + function signature only) in Table I."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is the raw (no-RAG) method. No comparison against other hallucination mitigation approaches or more sophisticated RAG methods. RepoCoder is referenced but not compared against directly."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study on the RAG-based mitigation (e.g., varying number of retrieved snippets, window size, similarity method)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is used to evaluate the mitigation method. No other metrics (Pass@k for k>1, hallucination rate reduction, etc.) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The hallucination taxonomy is constructed through manual open coding by authors and three volunteers with extensive Python experience (Section III-C). This constitutes human evaluation of LLM-generated code outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The CoderEval benchmark provides test cases for evaluating generated code correctness. The evaluation uses the standard benchmark test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows hallucination distribution across categories, and Figure 11 breaks down hallucination types per model. Detailed subcategory percentages are provided throughout Section IV-A."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure case analysis with code examples in Figures 3-10, showing specific hallucination instances across all categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All RAG results show improvement. No discussion of cases where RAG failed or made things worse, or configurations that didn't work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a hallucination taxonomy, distribution analysis, four root causes, and RAG mitigation effectiveness — all supported by Sections IV and V."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper identifies four 'root causes' of hallucinations (Section IV-C) but these are speculative analyses without controlled experiments. Claims like 'training data quality causes hallucinations' are causal but not experimentally validated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Practical Code Generation' broadly but results are limited to Python tasks from CoderEval (230 tasks). The paper discusses this in threats to validity but the title and framing overreach."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The root cause analysis in Section IV-C presents four factors as the explanation without considering alternatives. No discussion of whether the taxonomy itself or annotation process could explain the distribution patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Pass@1 to measure 'correctness' and manual hallucination counts to measure 'hallucination prevalence' but does not discuss whether these proxies capture the full scope of practical code generation quality they claim to study."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: GPT-3.5-Turbo, CodeGen-350M-Mono, PanGu-α-2.6B, DeepSeekCoder-6.7B, CodeLlama-7b-Python-hf, StarCoder2-7B (Section III-B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that docstrings and function signatures are provided as input, but does not show the actual prompt templates or formatting used when querying the LLMs."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0.6 with nuclear sampling, following CoderEval settings (Section III-A). RAG uses window size 20 lines, sliding step 2 lines, top-10 retrieval."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The RAG method is a simple retrieval-then-generate pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C describes the open coding process: 10% initial sample → preliminary taxonomy → iterative labeling of remaining 90%. Section V-B describes the RAG corpus construction with sliding window parameters."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' discusses external, internal, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: Python-only language limitation, 230-task dataset scale, absence of formal inter-rater reliability measure, model bias during annotation (mitigated by mixing results). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states the study focused on Python only and acknowledges 'constructing hallucination taxonomies for other programming languages and comparing them with our current taxonomy is a valuable future direction.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at the GitHub URL claims to include 'code, data, and experimental results,' which would include the annotation data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes the dataset (CoderEval, 230 Python tasks), the LLM generation process (10 snippets per task, temperature 0.6), and the annotation methodology (open coding with iterative refinement)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Three 'newly invited volunteers with extensive Python programming experience' performed annotation, but how they were recruited is not described. Only their experience levels are mentioned (two with 10+ years, one with 4 years)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C documents the full pipeline: initial 10% sample → preliminary taxonomy → full annotation by three volunteers → iterative taxonomy refinement. The RAG pipeline is also documented in Section V-B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sun Yat-sen University and Nanyang Technological University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are provided for any of the six models. CoderEval tasks could have been in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval tasks or their source repositories appeared in any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CoderEval is sourced from public GitHub repositories. No discussion of whether these repositories were in the training data of the evaluated models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a study sense. The annotators performed open coding on LLM outputs, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects research. Annotators are researchers performing analysis."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotators' experience is mentioned but this is not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency reported for any of the six models or the RAG mitigation approach."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget stated. Running 6 models × 230 tasks × 10 generations each is significant but unquantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results are reported as single aggregate numbers with no cross-seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 10 code snippets are generated per task using nuclear sampling with temperature 0.6 (Section III-A)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "RAG parameters (window size 20, step 2, top-10) are taken from RepoCoder without stating whether alternatives were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "RAG configuration adopted from RepoCoder without justification for why those specific parameters are optimal for this task."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own RAG mitigation without acknowledging potential bias in the evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RAG adds retrieval overhead to each generation but no compute comparison between raw and RAG methods is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses why CoderEval is more appropriate than HumanEval/MBPP for studying practical hallucinations, citing its repository-level context and non-standalone functions (Section I, II-A)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Models are queried directly with prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. CoderEval tasks come from public repos that could have been in training data of all models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The RAG evaluation excludes ground-truth code from the retrieval corpus: 'code lines containing or following the ground-truth code are excluded from the scanning process' (Section V-B)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval's source repositories overlap with model training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method applied (no canary strings, membership inference, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM hallucinations in code generation fall into three major categories with eight subcategories: Task Requirement Conflicts (43.53%), Factual Knowledge Conflicts (31.91%), and Project Context Conflicts (24.56%).",
       "evidence": "Manual open coding on 6 LLMs × 230 tasks × 10 snippets, iterative taxonomy construction described in Section III-C, distribution in Figure 2.",
       "supported": "moderate"
     },
     {
       "claim": "Task Requirement Conflicts are the most prevalent hallucination type across all six models.",
       "evidence": "Figure 11 shows distribution across models. All models show Task Requirement Conflicts as the largest category.",
       "supported": "strong"
     },
     {
       "claim": "RAG-based mitigation consistently improves Pass@1 for all six studied LLMs.",
       "evidence": "Table I shows improvements: CodeGen +1.31%, PanGu-α +1.70%, DeepSeekCoder +0.87%, CodeLlama +3.05%, StarCoder2 +2.57%, ChatGPT +2.21%.",
       "supported": "weak"
     },
     {
       "claim": "Four root causes contribute to hallucinations: training data quality, intention understanding capacity, knowledge acquisition capacity, and repository-level context awareness.",
       "evidence": "Qualitative analysis in Section IV-C with illustrative examples but no controlled experiments isolating each factor.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No inter-rater reliability measure",
       "detail": "The paper acknowledges in Section VII that there is 'no formal inter-rater reliability measure for annotating hallucinations.' For a taxonomy paper, inter-rater agreement (e.g., Cohen's kappa) is essential to validate the classification scheme."
     },
     {
       "flag": "Very low absolute performance",
       "detail": "Pass@1 scores range from 0.04% to 12.61%. At these levels, small absolute improvements (e.g., +0.87%) may not be meaningful and could be within noise margins, but no statistical tests are provided."
     },
     {
       "flag": "No contamination analysis",
       "detail": "CoderEval is sourced from public GitHub repositories. All evaluated models may have trained on these exact repositories, which would fundamentally affect the hallucination analysis."
     },
     {
       "flag": "Root cause analysis is speculative",
       "detail": "The four identified 'root causes' are presented as findings but are qualitative interpretations without controlled experiments to isolate causal factors."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, foundational code generation evaluation."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["H. Yu", "B. Shen"],
       "year": 2024,
       "relevance": "The primary benchmark used in this study; evaluates repository-level code generation."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["X. Du", "M. Liu"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Related benchmark for class-level code generation evaluation."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the six LLMs evaluated; key open-source code model."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["B. Rozière", "J. Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the six LLMs evaluated; Meta's code generation model family."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The next generation",
       "authors": ["A. Lozhkov", "R. Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "One of the evaluated models; open-source code LLM."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["F. Zhang", "B. Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Basis for the RAG-based mitigation approach used in this paper."
     },
     {
       "title": "Exploring and evaluating hallucinations in LLM-powered code generation",
       "authors": ["F. Liu", "Y. Liu"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "Prior work on code hallucination taxonomy for standalone functions; this paper extends to repository-level."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["L. Huang", "W. Yu"],
       "year": 2023,
       "arxiv_id": "2311.05232",
       "relevance": "Foundational survey on LLM hallucinations in the general domain."
     },
     {
       "title": "Quality and trust in LLM-generated code",
       "authors": ["C. Spiess", "D. Gros"],
       "year": 2024,
       "arxiv_id": "2402.02047",
       "relevance": "Studies trust and quality issues in LLM-generated code, directly relevant to reliability concerns."
     },
     {
       "title": "ClarifyGPT: Empowering LLM-based code generation with intention clarification",
       "authors": ["F. Mu", "L. Shi"],
       "year": 2023,
       "arxiv_id": "2310.10996",
       "relevance": "Alternative approach to mitigating code generation errors through requirement clarification."
     },
     {
       "title": "EvoCodeBench: An evolving code generation benchmark aligned with real-world code repositories",
       "authors": ["J. Li", "G. Li"],
       "year": 2024,
       "arxiv_id": "2404.00599",
       "relevance": "Related repository-level code generation benchmark with temporal evolution."
     }
   ]
 }