ai-research-survey

scan.json (23893B)

---

 {
   "paper": {
     "title": "Code Hallucination",
     "authors": [
       "Mirza Masfiqur Rahman",
       "Ashish Kundu",
       "Ramana Kompella",
       "Elisa Bertino"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.04831"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or code release of any kind is mentioned in the paper. The HallTrigger technique is described only in natural language."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of hallucinated code examples, prompts used, or model outputs is released. The paper describes case studies but does not provide structured data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, dependency list, or setup instructions are provided. The paper only mentions using ChatGPT (GPT-3.5, GPT-4), Gemini, and Copilot via their web/API interfaces."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The prompts used to trigger hallucinations are shown in figures but there is no systematic protocol for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. The paper presents qualitative case studies without any quantitative metrics or uncertainty measures."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no comparative quantitative claims that would require significance tests. Results are presented as qualitative case studies."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper does not quantify the rate or magnitude of hallucinations beyond qualitative observations (e.g., Table 1 uses symbols for hallucination/no hallucination without counts or rates)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of test cases, prompts, or trials per case study is not stated or justified. It is unclear how many prompts were tried for each case study or how often hallucinations occurred."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or spread measures are reported. The paper does not indicate how many times each experiment was repeated or what the variation in outcomes was."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparison is provided. The paper compares three models against each other (Table 1), but there is no baseline technique for detecting or triggering hallucinations to compare HallTrigger against."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions CodeHalu and HalluCode as related work but does not compare HallTrigger against them quantitatively or systematically."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "HallTrigger uses three principles (interactive prompting, meta-prompts, RLHF-based reward prompting) but no ablation study is performed to measure the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are used. The evaluation is entirely qualitative, with case studies showing individual examples of hallucinations. Table 1 uses symbols (hallucination/no hallucination) but no formal metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper implicitly involves manual inspection of model outputs (the authors determined whether outputs were hallucinated), but no formal human evaluation protocol is described. There are no inter-rater reliability measures or structured evaluation criteria."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study paper, not a benchmark evaluation. There is no train/test split structure."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a per-case-study breakdown showing which models exhibited hallucination for each of the 9 case types, separating code generation (G) from code analysis (A) tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about failure cases (hallucinations). Cases where models did NOT hallucinate are also noted (e.g., Copilot successfully avoided several hallucination types, marked with open circles in Table 1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cases where models did not exhibit hallucination (Table 1 shows multiple 'no hallucination' results for Copilot and Gemini), and notes that syntax errors were not found despite being plausible (Section 5.1, Case 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims HallTrigger is 'effective' and that 'pervasive LLM hallucination has sheer impact on software development,' but the evidence consists only of 9 qualitative case studies with no quantitative measure of effectiveness or impact. The word 'effective' is not grounded in any metric."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the three dynamic attributes (interactive prompts, meta-prompts, RLHF reward) cause hallucinations, but no controlled experiment isolates these factors. The causal mechanism is conjectured (Section 4: 'We conjecture that this largely instigates the creativity') rather than demonstrated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim hallucination is 'pervasive' and 'model-agnostic' based on testing only 3 models with 9 case studies. The title 'Code Hallucination' without qualification suggests generality beyond what was tested. Section 8 claims the approach 'demonstrates that code hallucination is prevalent for all black box large models' based on only 3 models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why models produced the observed outputs. For example, Case 3 (memorized LeetCode solutions) could be explained by training data contamination rather than hallucination per se, but this distinction is not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'ChatGPT (OpenAI GPT-3.5, 4)', 'Google Gemini', and 'Microsoft Copilot' without specific version numbers, API endpoints, or snapshot dates. No specific model version like 'gpt-4-0613' is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Several actual prompts are shown in the figures (Figures 1-7) and described in the case studies. For example, the exact prompt for Case 1 is shown in Figure 2, and the modified LeetCode prompt for Case 3 is quoted in the text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "For Copilot, the paper states 'we kept the setting to balanced' (Section 3), but no temperature, top-p, or other API parameters are reported for ChatGPT or Gemini. These settings significantly affect output."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "HallTrigger uses sequential multi-turn prompting, meta-prompts, and reward-based prompting (Section 4), but the exact scaffolding/workflow is described only at a high level. The sequence of prompts, decision logic for adapting prompts, and specific meta-prompt templates are not fully specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing is documented. It is unclear how prompts were designed, how many variations were tested, or how representative examples were selected from potentially many attempts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations' and discusses two specific limitations: the manual effort required by HallTrigger and the difficulty of remediation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses manual effort and remediation difficulty, but does not address threats to the validity of the findings themselves — e.g., selection bias in case studies, lack of quantification, potential non-reproducibility due to model updates, or the small number of models tested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound findings to specific models, languages (all examples are Python), or types of code tasks. The conclusion makes broad claims without scoping them."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (full conversation logs, complete model outputs, all attempted prompts) is made available. Only selected examples are shown in figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The data collection procedure is vaguely described. It is unclear how many prompts were tested, over what time period, how case studies were selected, or what the selection criteria were for the examples shown."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study involves the authors prompting LLMs directly."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The path from prompt design to example selection to Table 1 results is not described. It is unclear how the authors determined whether a model response constituted hallucination versus a correct-but-different solution."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or statement is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: three authors from Cisco Research and one from Purdue University. Since the paper evaluates third-party models (OpenAI, Google, Microsoft) rather than Cisco products, the affiliation does not create a direct conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. Three of four authors are from Cisco Research, which is a competitor to the companies whose products are being evaluated (OpenAI, Google, Microsoft), creating a potential bias toward finding flaws."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model capability on a benchmark. It tests whether models can be prompted to produce hallucinated code — this is about triggering failure modes, not measuring knowledge. Contamination is not relevant to the claims."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model performance on a benchmark. However, Case 3 does implicitly involve contamination (LeetCode problems in training data), which the paper actually discusses as a hallucination type rather than a contamination issue."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The study triggers hallucinations through crafted prompts, not through benchmark testing."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study. The authors manually interacted with LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; no IRB approval needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or time per experiment is reported, despite the method involving multiple rounds of prompting commercial APIs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or total cost is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "HallTrigger can effectively trigger arbitrary code hallucinations from black-box LLMs without accessing model parameters or architecture.",
       "evidence": "Section 4 describes the HallTrigger framework using three principles (interactive prompting, meta-prompts, RLHF-based reward). Sections 5.1 and 5.2 show 9 case studies. Table 1 shows hallucination results across 3 models. However, 'effectiveness' is not quantified — no success rate, no count of attempts vs. triggered hallucinations.",
       "supported": "weak"
     },
     {
       "claim": "Code hallucination is pervasive and model-agnostic across popular black-box LLMs.",
       "evidence": "Table 1 shows hallucinations were observed in all three tested models (GPT-4, Gemini, Copilot), but with varying frequency. However, only 3 models were tested with 9 case studies, sample sizes per case are not reported, and 'pervasive' is not quantified.",
       "supported": "weak"
     },
     {
       "claim": "LLMs exhibit memorization behavior that causes them to produce solutions matching training data even when requirements are modified.",
       "evidence": "Case 3 (Section 5.1) shows GPT-4 generating a solution closely matching the standard LeetCode 'Merge k Sorted Lists' solution despite the output requirement being changed to 'return 5 in a list'. Variable and class names match LeetCode skeleton code.",
       "supported": "moderate"
     },
     {
       "claim": "Models can be repetitively trapped in hallucination loops through sequential prompting.",
       "evidence": "Case 8 (Section 5.1) shows Gemini-Advanced repeatedly failing to correctly count lines in generated code, even after correction prompts. Figure 5 illustrates the model claiming 5 lines is 10 lines.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs fail to identify logical flaws in code that closely resembles known algorithms.",
       "evidence": "Case 9 (Section 5.2) shows models failing to detect modified/incorrect logic in merge sort and pair sum code segments, treating them as correct versions of the algorithms.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "The paper presents a taxonomy of 9 types of code hallucination in LLMs and introduces HallTrigger, a technique to deliberately trigger hallucinations using interactive prompting, meta-prompts, and reward-based feedback. Testing on ChatGPT (GPT-3.5/4), Gemini, and Microsoft Copilot, the authors demonstrate that all models are susceptible to various hallucination types including incorrect algorithm suggestions, code bloating, imaginary methods, runtime errors, and failure to detect logical flaws. The results are entirely qualitative with no quantitative metrics of hallucination rates or effectiveness.",
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "The entire evaluation consists of 9 qualitative case studies with cherry-picked examples. No hallucination rates, success rates for HallTrigger, or any quantitative metrics are reported. Table 1 uses symbols (hallucination/no hallucination) without counts or frequencies."
     },
     {
       "flag": "Unknown sample size and selection bias",
       "detail": "It is completely unclear how many prompts were tested per case study, how many attempts were needed to produce the shown examples, or whether the shown examples are representative. The cases appear to be selected to illustrate specific phenomena without systematic sampling."
     },
     {
       "flag": "Overclaiming from limited evidence",
       "detail": "The paper claims hallucination is 'pervasive' and 'model-agnostic' based on 3 models and 9 case studies. The abstract claims 'sheer impact on software development' without any evidence of real-world impact. The conclusion generalizes to 'all black box large models.'"
     },
     {
       "flag": "No model version specification",
       "detail": "Models are referred to as 'GPT-4', 'Gemini', and 'Copilot' without version numbers or dates. Model behavior changes across versions, so these results may not be reproducible."
     },
     {
       "flag": "Conflation of hallucination with other phenomena",
       "detail": "Some cases (e.g., Case 3 memorized LeetCode solutions, Case 6 runtime errors) may be better characterized as training data contamination, overfitting, or ordinary bugs rather than hallucination. The paper's definition of hallucination is broad enough to encompass many distinct failure modes."
     },
     {
       "flag": "Potential competitor bias",
       "detail": "Three of four authors are from Cisco Research, evaluating products from competitors (OpenAI, Google, Microsoft). No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational work for LLM code generation evaluation."
     },
     {
       "title": "CodeHalu: Code hallucinations in LLMs driven by execution-based verification",
       "authors": ["Yuchen Tian", "Weixiang Yan"],
       "year": 2024,
       "arxiv_id": "2405.00253",
       "relevance": "Directly related taxonomy of code hallucination types with execution-based verification methodology."
     },
     {
       "title": "Exploring and evaluating hallucinations in LLM-powered code generation",
       "authors": ["Fang Liu", "Yang Liu"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "HalluCode: Related taxonomy and evaluation of hallucination recognition capabilities in code LLMs."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLMs on real software engineering tasks from GitHub issues."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "EvalPlus framework for rigorous evaluation of LLM code generation correctness, directly relevant to code quality assessment."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao"],
       "year": 2024,
       "arxiv_id": "2308.10620",
       "relevance": "Comprehensive survey of LLMs in software engineering, relevant as a survey-of-surveys candidate."
     },
     {
       "title": "How secure is code generated by ChatGPT?",
       "authors": ["Raphaël Khoury", "Anderson R. Avila"],
       "year": 2023,
       "arxiv_id": "2304.09655",
       "relevance": "Evaluates security of ChatGPT-generated code, related to code quality and safety assessment."
     },
     {
       "title": "Hallucination is inevitable: An innate limitation of large language models",
       "authors": ["Ziwei Xu", "Sanjay Jain"],
       "year": 2024,
       "arxiv_id": "2401.11817",
       "relevance": "Theoretical work arguing hallucination cannot be completely eliminated from LLMs, foundational for understanding LLM limitations."
     },
     {
       "title": "INSIDE: LLMs' internal states retain the power of hallucination detection",
       "authors": ["Chao Chen", "Kai Liu"],
       "year": 2024,
       "relevance": "Hallucination detection method using LLM internal states, relevant to AI safety and reliability."
     },
     {
       "title": "Generate and pray: Using SALLMS to evaluate the security of LLM generated code",
       "authors": ["Mohammed Latif Siddiq", "Joanna C. S. Santos"],
       "year": 2023,
       "arxiv_id": "2311.00889",
       "relevance": "Security evaluation framework for LLM-generated code, related to code safety assessment methodology."
     },
     {
       "title": "Code security vulnerability repair using reinforcement learning with large language models",
       "authors": ["Nafis Tanveer Islam", "Mohammad Bahrami Karkevandi"],
       "year": 2024,
       "arxiv_id": "2401.07031",
       "relevance": "Uses LLMs for vulnerability repair, relevant to LLM-based code repair capabilities."
     }
   ]
 }