ai-research-survey

scan-v5.json (26073B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "De-Hallucinator: Mitigating LLM Hallucinations in Code Generation Tasks via Iterative Grounding",
     "authors": [
       "Aryaz Eghbali",
       "Michael Pradel"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2401.01701",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All quantitative claims in the abstract (23.3–50.6% edit distance improvement, 23.9–61.0% API recall improvement, 63.2% fixed hallucinations, 15.5% coverage gain) are directly supported by Tables 3 and 4 in the evaluation.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about De-Hallucinator improving code quality; a controlled comparison against baselines (initial prompt, RAG prompt) with Wilcoxon statistical tests and an ablation over hyperparameters k and n provides adequate support for these directional claims.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states 'our conclusions are valid only for these languages' (Python and JavaScript), and the abstract specifies evaluation across 'two code generation tasks, two programming languages, and five state-of-the-art LLMs.'",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether improvement could stem from simply adding more tokens to the prompt rather than the specific iterative API-retrieval mechanism; the RAG vs. iterative comparison partially addresses this but the paper does not frame it as an alternative explanation.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly explains each metric—edit distance quantifies token edits a developer needs to make, exact API match measures the specific phenomenon of interest, and passing tests/coverage directly measure test generation quality—without overclaiming these as general productivity measures.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is dedicated to 'LIMITATIONS AND THREATS TO VALIDITY' and contains multiple distinct limitations beyond a single sentence.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The section names specific threats: the assumption that APIs exist when needed (and a proposed workaround), evaluation restricted to Python and JavaScript, and potential non-representativeness of the project sample; the first two are concrete and specific.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states conclusions are valid only for Python and JavaScript, and notes the project sample may not represent all projects, providing meaningful scope boundaries.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information appears anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors clearly disclose their affiliation as Software Lab, University of Stuttgart, with institutional email addresses.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Hallucination is defined as inventing non-existent APIs; API reference is formally defined in Definition 3.1 with three subtypes (function, class, attribute); initial prompt, RAG prompt, and iterative prompt are each defined with examples.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper lists four explicit contributions at the end of the introduction: empirical motivation, a new technique, a novel algorithm, and empirical evidence of effectiveness across tasks and LLMs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 provides a detailed related work discussion that distinguishes De-Hallucinator from CoCoMIC, RepoCoder, HyDE, TestPilot, and RAG, explaining specifically how each differs in mechanism and coupling to the underlying LLM.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The data availability statement provides two public GitHub URLs (https://github.com/AryazE/dehallucinator and https://github.com/AryazE/testpilot) containing the implementation, datasets, and evaluation scripts.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The code completion dataset is constructed from 11 public GitHub projects with specific commit hashes listed in Table 2; the test generation dataset reuses public JavaScript projects from TestPilot.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions specific libraries (HuggingFace transformers, CodeQL, NLTK, scikit-learn Ball Tree) and hardware, but provides no requirements.txt, Dockerfile, or equivalent environment specification in the paper itself.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper points to GitHub repositories but provides no step-by-step instructions for reproducing the evaluation within the paper itself.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables 3 and 4 report only mean values without confidence intervals, standard deviations, or error bars for any of the main results.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses the Wilcoxon signed-rank test with the Pratt method for all comparative claims; statistically significant results are bolded in Tables 3 and 4.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Relative percentage improvements are reported alongside absolute numbers throughout Tables 3 and 4, providing effect size context (e.g., 23.3–50.6% edit distance reduction).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 50 tasks for the preliminary study, 10 completions per project (440 total), and 12 JavaScript projects is described procedurally but not justified through power analysis or comparable benchmarks.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or confidence intervals are reported for any metric; all tables show only mean values.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Two baselines are included: the conventional initial prompt (no retrieval) and RAG prompt (retrieval from initial prompt only), compared against the iterative De-Hallucinator approach.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baseline LLMs (CodeGen, CodeGen 2.5, UniXCoder, StarCoder+, GPT-3.5-turbo-0125) and TestPilot as the test generation baseline are state-of-the-art as of the 2024 submission.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "RQ3 (Section 5.4) provides a systematic ablation over the number of iterations k ∈ {1,2,3} and the number of API references n ∈ {2,10,20,40} for code completion and n ∈ {3,5,10} for test generation.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Three metrics are used for code completion (edit distance, normalized edit similarity, exact API match) and three for test generation (passing tests, coverage, fixed hallucinations).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "The preliminary study uses two authors independently classifying 50 completion pairs with Cohen's kappa of 0.76; RQ2 includes manual inspection of 20 completion tasks per LLM to verify correct API augmentation.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The evaluation uses a constructed dataset of completion tasks from 11 Python and 12 JavaScript projects; completions already predicted correctly by the baseline are excluded to form a meaningful test set.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results in Table 3 are broken down per LLM model (UniXCoder, CodeGen v1, CodeGen v2.5, StarCoder+), and Table 5 provides per-model breakdown of correct API augmentation rates.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 explicitly discusses failure cases: 'For cases where the approach fails to add the correct API reference into the prompt, the main reason is that the initial completion has low relevance w.r.t. the ground truth.'",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 shows RAG & iterative combined performs worse on coverage than iterative alone; Figure 7 shows n=40 API references hurts performance; these negative results are reported and discussed.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact HuggingFace model IDs are provided (Salesforce/codegen-2B-mono, Salesforce/codegen25-7b-mono, microsoft/unixcoder-base, bigcode/starcoderplus) and the OpenAI model version GPT-3.5-turbo-0125 is specified.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figures 4, 5, and 6 show complete example prompts including the API Reference block format, and Section 3.4 describes the prompt construction with concrete examples.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Key hyperparameters are reported: k=3 iterations, n=20 API references for code completion, n=3 for test generation, max new tokens=256, temperature=0.1, 4 completions per prompt.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The three-stage pipeline (pre-analysis → retrieval → prompt construction → LLM query) is described in detail across Sections 3.2–3.5 with both general description and task-specific instantiation.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1.3 documents dataset construction: removing API usages, filtering functions >25 lines, removing API-related imports, and excluding already-correct completions are all described with rationale.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Specific project commits are listed in Table 2 for all 23 projects, and implementation/datasets are released at the GitHub URLs, enabling independent data reconstruction.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The collection process is documented: randomly sampling from a curated awesome-python list, sampling by application domain, selecting 5 functions per project, applying inclusion/exclusion criteria.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; the study uses public code repositories and internal author annotation for the preliminary study.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from project selection → API extraction via CodeQL → embedding → Ball Tree indexing → retrieval → prompt construction → evaluation is documented across Sections 3–5.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoffs are stated for any of the five evaluated models, despite evaluating on public GitHub code that may have been in training corpora.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper excludes already-correct completions 'to avoid any potential memorizations' but does not analyze or discuss training data overlap with the evaluated GitHub projects.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation projects are popular public GitHub repositories that were almost certainly available before the training cutoffs of the evaluated models; this is not discussed or analyzed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study; the manual annotation is performed by the authors.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; IRB approval is not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 reports retrieval latency (21–227ms for code completion, 0.1–17ms for test generation) and LLM query times (1.3s for GPT-3.5 to 66.7s for CodeGen v2.5 on local GPU).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Hardware is specified (two Nvidia T4 GPUs, single Tesla V100 with 32GB) and pre-analysis times are given per project (under 80 seconds for Python, 3.5s average for JavaScript).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "API hallucinations affect 44% of all function-level code completion tasks and 59% of failed tasks in a preliminary study of 50 functions.",
       "evidence": "Manual classification of 50 code completion tasks with inter-rater agreement Cohen's kappa=0.76; 22 of 37 failed completions had at least one missing/wrong API usage.",
       "supported": "moderate"
     },
     {
       "claim": "De-Hallucinator improves edit distance by 23.3–50.6% over conventional prompts across four LLMs.",
       "evidence": "Table 3 shows reductions from 50.6% (UniXCoder) to 23.3% (CodeGen v1) with Wilcoxon statistical significance; all improvements are statistically significant.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator improves exact API match recall by 23.9–61.0% across four LLMs for code completion.",
       "evidence": "Table 3 reports exact API match counts for initial vs. iterative prompts, with statistically significant improvements for all models.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator fixes 63.2% more hallucinated tests and increases statement coverage by 15.5% for test generation.",
       "evidence": "Table 4 shows iterative prompts improve fixed hallucinations from 19.3 to 31.4 (63.2% relative) and coverage from 32.1 to 37.0 (15.5% relative) with Wilcoxon significance.",
       "supported": "strong"
     },
     {
       "claim": "Iterative prompts (using model output for retrieval) outperform plain RAG prompts (using initial prompt for retrieval).",
       "evidence": "Table 3 consistently shows iterative prompts achieve lower edit distance and higher API match than RAG prompts; Table 4 shows iterative alone achieves higher coverage than RAG & iterative combined.",
       "supported": "strong"
     },
     {
       "claim": "Even k=1 iteration provides clear improvements over the baseline, making the approach useful when LLM query cost is high.",
       "evidence": "Figure 7 shows k=1 already provides substantial relative improvement in exact API match across all four models.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "De-Hallucinator addresses LLM hallucinations of project-specific APIs by iteratively augmenting prompts with retrieved API references, using the model's own (hallucinated) predictions to guide better retrieval. Evaluated on 5 LLMs across 11 Python and 12 JavaScript open-source projects, the approach consistently and significantly improves code completion (23.3–50.6% edit distance reduction, 23.9–61.0% API recall improvement) and test generation (63.2% more hallucinated tests fixed, 15.5% coverage increase). The iterative retrieval mechanism outperforms plain RAG, and even a single iteration provides meaningful gains. The approach requires no fine-tuning and treats the LLM as a black box.",
   "red_flags": [
     {
       "flag": "No variance reported",
       "detail": "All main result tables (Tables 3 and 4) report only mean values with no standard deviation, confidence intervals, or error bars, making it impossible to assess result stability."
     },
     {
       "flag": "Contamination not addressed",
       "detail": "All 11 Python and 12 JavaScript projects are popular public GitHub repositories almost certainly present in the training corpora of the evaluated models; the paper does not analyze or discuss train-test overlap beyond excluding already-correct completions."
     },
     {
       "flag": "Small preliminary study",
       "detail": "The motivational claim that 44% of failed completions involve hallucinated APIs is based on manual inspection of only 50 functions from 10 projects by the authors themselves."
     },
     {
       "flag": "No confound isolation for 'more context'",
       "detail": "The paper does not rule out that improvement comes simply from adding more tokens to the prompt rather than from the specific iterative API-retrieval mechanism; a control adding random API references would strengthen the causal claim."
     }
   ],
   "cited_papers": [
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "relevance": "TestPilot is the baseline system for the test generation task; De-Hallucinator extends TestPilot with iterative API grounding."
     },
     {
       "title": "Repository-Level Code Completion Through Iterative Retrieval and Generation (RepoCoder)",
       "relevance": "Concurrent work on repository-level code completion using iterative retrieval; directly compared to as the closest prior approach."
     },
     {
       "title": "CoCoMIC: Code Completion By Jointly Modeling In-file and Cross-file Context",
       "relevance": "Prior work addressing project-specific context for code completion via fine-tuning; contrasted with De-Hallucinator's black-box approach."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "relevance": "Foundational RAG technique that De-Hallucinator extends with iterative retrieval using model predictions."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex/HumanEval)",
       "relevance": "Key baseline work on evaluating code LLMs; establishes the benchmark context for code generation evaluation."
     },
     {
       "title": "Repository-Level Prompt Generation for Large Language Models of Code",
       "relevance": "Prior work on repository-level prompt generation using a separate trained model; contrasted with De-Hallucinator's model-agnostic approach."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "relevance": "One of four LLMs evaluated in the code completion experiments."
     },
     {
       "title": "An Empirical Evaluation of GitHub Copilot's Code Suggestions",
       "relevance": "Prior empirical study on LLM hallucinations in code generation; motivates the research problem."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "The technique works with off-the-shelf LLMs as black boxes, is deployable in IDEs, and addresses a concrete pain point that developers already report with AI coding assistants."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Using the model's hallucinated output to retrieve better context is a clever observation, but iterative refinement and RAG are established concepts so the insight is incremental."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns; the paper addresses a usability/accuracy problem in code generation tools."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; the paper presents a technical improvement without challenging major assumptions or competing with prominent labs."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Code is publicly released on GitHub for both tasks, and the approach works with any off-the-shelf LLM making it immediately tryable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University of Stuttgart Software Lab is a respected academic group but not a widely-recognized brand in industry or general AI discourse."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "38939558",
         "title": "Large Legal Fictions: Profiling Legal Hallucinations in Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38939558",
         "created_at": "2024-01-10T14:57:07Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }