ai-research-survey

scan-v4.json (32158B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "De-Hallucinator: Mitigating LLM Hallucinations in Code Generation Tasks via Iterative Grounding",
     "authors": [
       "Aryaz Eghbali",
       "Michael Pradel"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2401.01701",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 23.3–50.6% edit distance improvement, 23.9–61.0% recall improvement, 63.2% fixed hallucinations, and 15.5% coverage increase are all directly supported by Tables 3 and 4 in the results.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims De-Hallucinator 'improves' generated code. The experimental design is a controlled comparison: same models, same datasets, with and without De-Hallucinator's augmented prompts. This controlled single-variable manipulation adequately supports the causal claims.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'We implement and evaluate De-Hallucinator for Python and JavaScript, and although our general approach could be applied to any language, our conclusions are valid only for these languages.' The title says 'Code Generation Tasks' broadly, but limitations bound the claims.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses limitations (API availability assumption, language generalization, project representativeness) but does not discuss alternative explanations for the observed improvements, such as whether the gains come from additional context length rather than API-specific retrieval, or whether retrieval noise could explain variable results.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures edit distance, exact API match, and test pass rates, and frames claims at the same granularity ('more accurate predictions', 'mitigated hallucinations'). No broader framing beyond what was measured (e.g., no productivity claims).",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations and Threats to Validity' and provides substantive discussion of multiple limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: API availability assumption (developer may write usage before implementing API), language-specific conclusions (Python and JavaScript only), and project representativeness with specific mitigation (diverse selection of popular projects).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'our conclusions are valid only for these languages' (Python and JavaScript). The paper identifies the specific scenario where the approach fails (APIs not yet implemented).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. No acknowledgments section listing grants or sponsors.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are listed as affiliated with Software Lab, University of Stuttgart. They are not evaluating a commercial product they created.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of funders cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Definition 3.1 formally defines 'API reference' with three subtypes; hallucination and iterative prompting are explained through a concrete running example.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction lists four explicit bullet-point contributions: empirical motivation study, the technique itself, the novel algorithm, and empirical evidence across five LLMs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly contrasts De-Hallucinator with CoCoMIC, RepoCoder, ReACC, HyDE, and TestPilot, explaining mechanistic differences rather than simply listing papers.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Data-Availability Statement provides two GitHub repositories: https://github.com/AryazE/dehallucinator and https://github.com/AryazE/testpilot, described as containing 'implementation, datasets, and evaluation scripts.'",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The Data-Availability Statement states 'Our implementation, datasets, and evaluation scripts are publicly available' at the provided GitHub URLs.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies hardware (Nvidia T4, Tesla V100) and mentions HuggingFace transformers and specific model IDs, but does not provide a requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes the experimental setup in detail (Section 5.1) but does not include step-by-step reproduction instructions or a 'Reproducing Results' section. The code release may contain a README, but the paper itself lacks explicit reproduction steps.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables 3 and 4 report only point estimates (e.g., '50.6%' improvement). No confidence intervals, error bars, or ± notation appear anywhere in the results.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.2 states 'The approach shows statistically significant (using the Wilcoxon test and Pratt method) improvements over the baseline consistently for all metrics and all models.' Same tests used for test generation results (Table 4).",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Tables 3 and 4 report both absolute values and relative improvements with baseline context (e.g., edit distance 47.2→30.1, '36.3% improvement'), providing sufficient information to assess effect magnitude.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation uses 440 code completion tasks (11 projects × 10 tasks × 4 models) and 12 JavaScript projects for test generation. No justification is given for why these sample sizes are sufficient, and no power analysis is discussed.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Results in Tables 3 and 4 are reported as single aggregate values without standard deviation, interquartile range, or any spread measure across projects or tasks.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The paper compares against an initial prompt baseline (no retrieval) and a RAG prompt baseline for code completion. For test generation, the baseline is the original TestPilot implementation (Section 5.1.2).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The baseline models (CodeGen 2022, CodeGen 2.5 2023, UniXCoder 2022, StarCoder+ 2023, GPT-3.5-turbo-0125) were contemporary at the time of writing. TestPilot is a state-of-the-art test generation tool.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Tables 3 and 4 separately report results for RAG prompts and iterative prompts, showing the contribution of each component. Figures 7 and 8 ablate over hyperparameters k and n, and Table 4 compares 'RAG & iterative' vs 'Iterative' alone.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Code completion uses three metrics: edit distance, normalized edit similarity, and exact API match (Section 5.1.4). Test generation uses passing tests, coverage, and fixed hallucinations.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 describes manual classification of 50 completion tasks by two authors with inter-rater agreement (Cohen's kappa 0.76). Section 5.3 manually inspects 20 completion tasks per LLM to assess retrieval quality.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "No explicit dev/test split is described. Hyperparameters k and n are tuned using the same data that reports final results (Figures 7 and 8 show tuning experiments, and the selected defaults are used for Tables 3 and 4).",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by model (Table 3), by prompt type (RAG vs iterative), and by hyperparameter settings (Figures 7, 8). Table 5 breaks down API retrieval success per model. Table 2 lists per-project details.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 discusses failure cases: 'For cases where the approach fails to add the correct API reference into the prompt, the main reason is that the initial completion has low relevance w.r.t. the ground truth.' Section 6 also discusses when the approach cannot help.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 shows that combining RAG & iterative prompts yields lower coverage (33.7%) than iterative alone (37.0%), demonstrating that adding RAG can hurt. RAG prompts alone show minimal improvement for UniXCoder's exact API match (0.0% improvement in Table 3).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact HuggingFace model IDs are provided: 'Salesforce/codegen-2B-mono', 'Salesforce/codegen25-7b-mono', 'microsoft/unixcoder-base', 'bigcode/starcoderplus', and 'GPT-3.5-turbo-0125' with the API snapshot date.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figures 4, 5, and 6 show concrete examples of actual prompts including API reference sections. Section 3.4 describes the prompt construction format. The code is also publicly released containing the implementation.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Key hyperparameters are reported: max tokens = 256 (Section 5.1.2), temperature = 0.1, 4 completions per prompt (test generation), k = 3 iterations, n = 20 API references (code completion), n = 3 (test generation), prompt size limit of 2,048 tokens.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The iterative retrieval-augmented pipeline is described in full detail across Sections 3.1–3.5, including the pre-analysis index, embedding-based retrieval, Ball Tree search, prompt construction, and iterative loop. Figure 3 provides an architectural overview.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1.3 documents dataset construction: random project selection from curated lists, API usage removal for ground truth, multi-line call handling, import removal to prevent data leakage, and filtering of already-correct predictions (with exact counts: 18, 51, 76, and 31 filtered per model).",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The Data-Availability Statement provides links to datasets and evaluation scripts at two GitHub repositories, enabling independent verification of the results.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1.3 describes the collection procedure in detail: random selection of 10 Python projects from awesome-python (by domain sampling), random selection of 5 functions per project with a 25-line filter (preliminary study), and API-usage-removal-based benchmark construction for the main evaluation.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data comes from public open-source GitHub projects selected from a curated list.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline is documented: select projects from curated list → extract API usages → remove API usage lines and related imports → query LLMs → filter already-correct predictions (with counts per model: 18, 51, 76, 31) → evaluate with metrics. For test generation, TestPilot's pipeline is reused with token budget matching.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state training data cutoff dates for any of the five LLMs used (CodeGen, CodeGen 2.5, UniXCoder, StarCoder+, GPT-3.5-turbo), despite evaluating them on code from public GitHub repositories that may have been in their training data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper filters out exact-match completions 'to avoid any potential memorizations' (Section 5.1.3), but does not analyze whether the evaluation projects appeared in model training data. Exact-match filtering only catches the most obvious contamination.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation projects are all popular open-source GitHub repositories (e.g., scikit-learn with 58.5k stars, black with 37.6k stars) that almost certainly appeared in training data. The paper does not discuss this contamination risk beyond filtering exact-match predictions.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study. The evaluation is entirely automated with supplementary manual code inspection.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study uses public open-source code and automated evaluation.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 reports detailed timing: pre-analysis takes under 1 second per 1,000 LoC, retrieval takes 21–227ms per iteration, LLM query time ranges 1.3–66.7 seconds per query. Token limits per package are stated for test generation (130k).",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1.5 specifies hardware (two Nvidia T4 GPUs with 16GB each, single Nvidia Tesla V100 with 32GB, 48-core Intel Xeon at 2.20GHz). Section 5.5 provides per-operation timing breakdowns, and test generation uses a 130k token limit per package.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not report results across multiple random seeds. For test generation, temperature is 0.1 with 4 completions, but no seed variation analysis is performed.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state how many times the full experiment was run. It generates k=3 iterations and 4 completions per prompt (test generation) but does not state whether the entire evaluation was repeated across multiple runs.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "RQ3 studies k∈{1,2,3} and n∈{2,10,20,40} for code completion, n∈{3,5,10} for test generation, but the total search budget (compute spent on hyperparameter exploration) is not reported.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "Hyperparameters k=3, n=20 (code completion) and n=3 (test generation) are selected based on RQ3 experiments (Figures 7, 8) using the same data that reports final results. No separate validation set is used for selection.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Wilcoxon tests are performed across 4 models × 3 metrics for code completion and 3 metrics for test generation, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is mentioned.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors evaluate their own De-Hallucinator system against baselines without acknowledging author-evaluation bias. While baselines are off-the-shelf models (reducing this concern), the benchmark construction and metric selection were done by the same authors.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "For test generation, Section 5.1.2 states 'we set the token limit of De-Hallucinator to the amount of tokens used by the baseline,' explicitly matching compute budgets. Figures 7-8 show performance across different k values (number of queries), implicitly showing performance vs. compute tradeoff.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark is constructed by removing API usages from existing code and treating the original as ground truth. The paper does not discuss whether this constructed benchmark validly measures real-world code completion needs or whether the removal-based approach introduces artifacts.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The same underlying LLMs are used with and without De-Hallucinator, isolating the scaffolding effect. The comparison is controlled: same model, same data, only the prompting strategy changes. Results are shown across four different models.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation projects (e.g., scikit-learn, black, seaborn) are popular open-source projects whose code existed before the models' training cutoffs. The paper does not discuss whether models may have seen the test code during training.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup leaks information not available in real IDE usage scenarios. The API reference augmentation itself is the technique being tested, but no analysis of potential feature leakage in the benchmark construction is provided.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The 10 completion tasks per project are drawn from the same codebase and may share structural similarities (imports, coding style). No discussion of potential non-independence between evaluation examples.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "The paper filters out exact-match predictions to avoid memorization effects, but does not apply a concrete leakage detection method such as canary strings, membership inference tests, n-gram overlap analysis, or temporal splits.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "API hallucinations affect 44% of all studied function-level code completion tasks and 59% of tasks where the baseline LLM fails.",
       "evidence": "Preliminary study on 50 manually classified completions using CodeGen 2.5 with inter-rater Cohen's kappa = 0.76 (Section 2.1).",
       "supported": "moderate"
     },
     {
       "claim": "De-Hallucinator improves edit distance by 23.3–50.6% relative to conventional prompting across four LLMs.",
       "evidence": "Table 3 shows statistically significant improvements (Wilcoxon/Pratt) for all four models on all metrics.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator improves exact API match recall by 23.9–61.0% relative to baseline.",
       "evidence": "Table 3 exact API match row; e.g., CodeGen v2.5 improves from 8.3 to 13.4 (61.0% relative gain), all statistically significant.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator reduces hallucinated tests by 63.2% and increases statement coverage by 15.5% in JavaScript test generation.",
       "evidence": "Table 4 comparing iterative De-Hallucinator vs. TestPilot baseline; coverage and fixed-hallucination improvements are statistically significant.",
       "supported": "strong"
     },
     {
       "claim": "Most improvement is achieved in the first iteration; additional iterations yield diminishing returns.",
       "evidence": "Figure 7 shows k=1 already captures most of the exact API match improvement across all four LLMs; k=2 and k=3 add incrementally.",
       "supported": "strong"
     },
     {
       "claim": "RAG alone (non-iterative) is less effective than iterative retrieval, especially when the initial prompt lacks relevant code.",
       "evidence": "Table 4 shows RAG & iterative combined achieves lower coverage (33.7%) than iterative alone (37.0%) for test generation; Section 5.2 explains the mechanism.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "De-Hallucinator mitigates API hallucinations in LLM-based code generation by iteratively augmenting prompts with project-specific API references derived from the model's own (hallucinated) predictions rather than from the original prompt. Evaluated on 440 code completion tasks across four LLMs and 12 JavaScript test generation projects, the approach consistently and significantly outperforms baselines: 23.3–50.6% edit distance improvement, 23.9–61.0% exact API match improvement, 63.2% fewer hallucinated tests, and 15.5% higher statement coverage. The technique requires no model fine-tuning, works as a black-box wrapper, and most of its benefit is realized in the first iteration, making it practical for production deployment.",
   "red_flags": [
     {
       "flag": "No variance or confidence intervals",
       "detail": "Tables 3 and 4 report mean improvements with no standard deviation, standard error, or confidence intervals; spread across 11 or 12 projects is unknown."
     },
     {
       "flag": "Small preliminary study",
       "detail": "The core motivation claim (44% of completions involve API hallucinations) is based on only 50 manually examined tasks from 10 projects using a single model."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Test projects are public GitHub repos that likely predate training cutoffs; filtering memorized completions is a partial mitigation but no formal contamination analysis is conducted for any of the five models."
     },
     {
       "flag": "No formal sample size justification",
       "detail": "Ten completions per project is chosen for convenience; no power analysis supports that this sample size is adequate for the comparative claims made."
     }
   ],
   "cited_papers": [
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation (TestPilot)",
       "relevance": "Direct baseline system extended for the JavaScript test generation experiments"
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "relevance": "One of the primary LLMs evaluated in code completion experiments"
     },
     {
       "title": "StarCoder: may the source be with you!",
       "relevance": "Largest LLM baseline (15.5B parameters) evaluated in code completion"
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "relevance": "Foundational RAG technique that De-Hallucinator extends with iterative prompting"
     },
     {
       "title": "CoCoMIC: Code Completion By Jointly Modeling In-file and Cross-file Context",
       "relevance": "Key prior work on cross-file context; De-Hallucinator explicitly contrasts its black-box approach vs. CoCoMIC's fine-tuning requirement"
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "relevance": "Concurrent work on iterative retrieval for code completion; distinguished by retrieving code fragments vs. De-Hallucinator's API signatures"
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "relevance": "Related work on RAG-based context selection for code; contrasted with De-Hallucinator's model-agnostic approach"
     },
     {
       "title": "Measuring GitHub Copilot's Impact on Productivity",
       "relevance": "Cited to establish practical relevance and adoption of AI code completion tools"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Presents a usable technique for reducing API hallucinations in code completion with open-source code available, though requires integration work."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The iterative grounding idea is clever but the finding that LLMs hallucinate project-specific APIs is well-known, not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle — purely about improving code generation accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforwardly improves on baselines without challenging any company or popular belief."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub but requires setting up CodeQL, embedding models, and specific LLMs — significant setup effort."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University of Stuttgart authors, not a well-known AI lab; models used (CodeGen, UniXcoder) are not household names."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "38939558",
         "title": "Large Legal Fictions: Profiling Legal Hallucinations in Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38939558",
         "created_at": "2024-01-10T14:57:07Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }