ai-research-survey

scan.json (30941B)

---

 {
   "paper": {
     "title": "De-Hallucinator: Mitigating LLM Hallucinations in Code Generation Tasks via Iterative Grounding",
     "authors": [
       "Aryaz Eghbali",
       "Michael Pradel"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2401.01701"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "De-Hallucinator iteratively augments LLM prompts with project-specific API references retrieved based on the model's own predictions, improving code completion edit distance by 23.3–50.6% and exact API match by 23.9–61.0% across four LLMs. For test generation with GPT-3.5-turbo, it fixes 63.2% of hallucination-induced test failures and increases statement coverage by 15.5%. A preliminary study shows API hallucinations affect 44% of function-level code completion tasks. The first iteration provides the largest gains, with diminishing returns from additional iterations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Data-Availability Statement provides two GitHub repositories: https://github.com/AryazE/dehallucinator and https://github.com/AryazE/testpilot, described as containing 'implementation, datasets, and evaluation scripts.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Data-Availability Statement states 'Our implementation, datasets, and evaluation scripts are publicly available' at the provided GitHub URLs."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies hardware (Nvidia T4, Tesla V100) and mentions HuggingFace transformers and specific model IDs, but does not provide a requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the experimental setup in detail (Section 5.1) but does not include step-by-step reproduction instructions or a 'Reproducing Results' section. The code release may contain a README, but the paper itself lacks explicit reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3 and 4 report only point estimates (e.g., '50.6%' improvement). No confidence intervals, error bars, or ± notation appear anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states 'The approach shows statistically significant (using the Wilcoxon test and Pratt method) improvements over the baseline consistently for all metrics and all models.' Same tests used for test generation results (Table 4)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 report both absolute values and relative improvements with baseline context (e.g., edit distance 47.2→30.1, '36.3% improvement'), providing sufficient information to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 440 code completion tasks (11 projects × 10 tasks × 4 models) and 12 JavaScript projects for test generation. No justification is given for why these sample sizes are sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 3 and 4 are reported as single aggregate values without standard deviation, interquartile range, or any spread measure across projects or tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against an initial prompt baseline (no retrieval) and a RAG prompt baseline for code completion. For test generation, the baseline is the original TestPilot implementation (Section 5.1.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline models (CodeGen 2022, CodeGen 2.5 2023, UniXCoder 2022, StarCoder+ 2023, GPT-3.5-turbo-0125) were contemporary at the time of writing. TestPilot is a state-of-the-art test generation tool."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 separately report results for RAG prompts and iterative prompts, showing the contribution of each component. Figures 7 and 8 ablate over hyperparameters k and n, and Table 4 compares 'RAG & iterative' vs 'Iterative' alone."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Code completion uses three metrics: edit distance, normalized edit similarity, and exact API match (Section 5.1.4). Test generation uses passing tests, coverage, and fixed hallucinations."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes manual classification of 50 completion tasks by two authors with inter-rater agreement (Cohen's kappa 0.76). Section 5.3 manually inspects 20 completion tasks per LLM to assess retrieval quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit dev/test split is described. Hyperparameters k and n are tuned using the same data that reports final results (Figures 7 and 8 show tuning experiments, and the selected defaults are used for Tables 3 and 4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (Table 3), by prompt type (RAG vs iterative), and by hyperparameter settings (Figures 7, 8). Table 5 breaks down API retrieval success per model. Table 2 lists per-project details."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses failure cases: 'For cases where the approach fails to add the correct API reference into the prompt, the main reason is that the initial completion has low relevance w.r.t. the ground truth.' Section 6 also discusses when the approach cannot help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows that combining RAG & iterative prompts yields lower coverage (33.7%) than iterative alone (37.0%), demonstrating that adding RAG can hurt. RAG prompts alone show minimal improvement for UniXCoder's exact API match (0.0% improvement in Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 23.3–50.6% edit distance improvement, 23.9–61.0% recall improvement, 63.2% fixed hallucinations, and 15.5% coverage increase are all directly supported by Tables 3 and 4 in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims De-Hallucinator 'improves' generated code. The experimental design is a controlled comparison: same models, same datasets, with and without De-Hallucinator's augmented prompts. This controlled single-variable manipulation adequately supports the causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'We implement and evaluate De-Hallucinator for Python and JavaScript, and although our general approach could be applied to any language, our conclusions are valid only for these languages.' The title says 'Code Generation Tasks' broadly, but limitations bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses limitations (API availability assumption, language generalization, project representativeness) but does not discuss alternative explanations for the observed improvements, such as whether the gains come from additional context length rather than API-specific retrieval, or whether retrieval noise could explain variable results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures edit distance, exact API match, and test pass rates, and frames claims at the same granularity ('more accurate predictions', 'mitigated hallucinations'). No broader framing beyond what was measured (e.g., no productivity claims)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact HuggingFace model IDs are provided: 'Salesforce/codegen-2B-mono', 'Salesforce/codegen25-7b-mono', 'microsoft/unixcoder-base', 'bigcode/starcoderplus', and 'GPT-3.5-turbo-0125' with the API snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 4, 5, and 6 show concrete examples of actual prompts including API reference sections. Section 3.4 describes the prompt construction format. The code is also publicly released containing the implementation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: max tokens = 256 (Section 5.1.2), temperature = 0.1, 4 completions per prompt (test generation), k = 3 iterations, n = 20 API references (code completion), n = 3 (test generation), prompt size limit of 2,048 tokens."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative retrieval-augmented pipeline is described in full detail across Sections 3.1–3.5, including the pre-analysis index, embedding-based retrieval, Ball Tree search, prompt construction, and iterative loop. Figure 3 provides an architectural overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.3 documents dataset construction: random project selection from curated lists, API usage removal for ground truth, multi-line call handling, import removal to prevent data leakage, and filtering of already-correct predictions (with exact counts: 18, 51, 76, and 31 filtered per model)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations and Threats to Validity' and provides substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: API availability assumption (developer may write usage before implementing API), language-specific conclusions (Python and JavaScript only), and project representativeness with specific mitigation (diverse selection of popular projects)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'our conclusions are valid only for these languages' (Python and JavaScript). The paper identifies the specific scenario where the approach fails (APIs not yet implemented)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Data-Availability Statement provides links to datasets and evaluation scripts at two GitHub repositories, enabling independent verification of the results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.3 describes the collection procedure in detail: random selection of 10 Python projects from awesome-python (by domain sampling), random selection of 5 functions per project with a 25-line filter (preliminary study), and API-usage-removal-based benchmark construction for the main evaluation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public open-source GitHub projects selected from a curated list."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: select projects from curated list → extract API usages → remove API usage lines and related imports → query LLMs → filter already-correct predictions (with counts per model: 18, 51, 76, 31) → evaluate with metrics. For test generation, TestPilot's pipeline is reused with token budget matching."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. No acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are listed as affiliated with Software Lab, University of Stuttgart. They are not evaluating a commercial product they created."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of funders cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the five LLMs used (CodeGen, CodeGen 2.5, UniXCoder, StarCoder+, GPT-3.5-turbo), despite evaluating them on code from public GitHub repositories that may have been in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper filters out exact-match completions 'to avoid any potential memorizations' (Section 5.1.3), but does not analyze whether the evaluation projects appeared in model training data. Exact-match filtering only catches the most obvious contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation projects are all popular open-source GitHub repositories (e.g., scikit-learn with 58.5k stars, black with 37.6k stars) that almost certainly appeared in training data. The paper does not discuss this contamination risk beyond filtering exact-match predictions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The evaluation is entirely automated with supplementary manual code inspection."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses public open-source code and automated evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 reports detailed timing: pre-analysis takes under 1 second per 1,000 LoC, retrieval takes 21–227ms per iteration, LLM query time ranges 1.3–66.7 seconds per query. Token limits per package are stated for test generation (130k)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.5 specifies hardware (two Nvidia T4 GPUs with 16GB each, single Nvidia Tesla V100 with 32GB, 48-core Intel Xeon at 2.20GHz). Section 5.5 provides per-operation timing breakdowns, and test generation uses a 130k token limit per package."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report results across multiple random seeds. For test generation, temperature is 0.1 with 4 completions, but no seed variation analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times the full experiment was run. It generates k=3 iterations and 4 completions per prompt (test generation) but does not state whether the entire evaluation was repeated across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "RQ3 studies k∈{1,2,3} and n∈{2,10,20,40} for code completion, n∈{3,5,10} for test generation, but the total search budget (compute spent on hyperparameter exploration) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters k=3, n=20 (code completion) and n=3 (test generation) are selected based on RQ3 experiments (Figures 7, 8) using the same data that reports final results. No separate validation set is used for selection."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Wilcoxon tests are performed across 4 models × 3 metrics for code completion and 3 metrics for test generation, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own De-Hallucinator system against baselines without acknowledging author-evaluation bias. While baselines are off-the-shelf models (reducing this concern), the benchmark construction and metric selection were done by the same authors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "For test generation, Section 5.1.2 states 'we set the token limit of De-Hallucinator to the amount of tokens used by the baseline,' explicitly matching compute budgets. Figures 7-8 show performance across different k values (number of queries), implicitly showing performance vs. compute tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed by removing API usages from existing code and treating the original as ground truth. The paper does not discuss whether this constructed benchmark validly measures real-world code completion needs or whether the removal-based approach introduces artifacts."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same underlying LLMs are used with and without De-Hallucinator, isolating the scaffolding effect. The comparison is controlled: same model, same data, only the prompting strategy changes. Results are shown across four different models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation projects (e.g., scikit-learn, black, seaborn) are popular open-source projects whose code existed before the models' training cutoffs. The paper does not discuss whether models may have seen the test code during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real IDE usage scenarios. The API reference augmentation itself is the technique being tested, but no analysis of potential feature leakage in the benchmark construction is provided."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 10 completion tasks per project are drawn from the same codebase and may share structural similarities (imports, coding style). No discussion of potential non-independence between evaluation examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "The paper filters out exact-match predictions to avoid memorization effects, but does not apply a concrete leakage detection method such as canary strings, membership inference tests, n-gram overlap analysis, or temporal splits."
       }
     }
   },
   "claims": [
     {
       "claim": "De-Hallucinator improves edit distance by 23.3–50.6% compared to conventional prompts across four LLMs for code completion.",
       "evidence": "Table 3 shows statistically significant improvements (Wilcoxon test, Pratt method) for all four models: UniXCoder 50.6%, CodeGen v1 23.3%, CodeGen v2.5 36.3%, StarCoder+ 24.9%.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator improves exact API match recall by 23.9–61.0% for code completion.",
       "evidence": "Table 3 shows improvements across models: UniXCoder 23.9%, CodeGen v1 55.3%, CodeGen v2.5 61.0%, StarCoder+ 32.0%.",
       "supported": "strong"
     },
     {
       "claim": "De-Hallucinator fixes 63.2% of tests that initially failed due to hallucinated APIs in test generation.",
       "evidence": "Table 4 reports fixed hallucinations increasing from 19.3 (TestPilot baseline) to 31.4 (iterative), a 63.2% relative improvement with GPT-3.5-turbo.",
       "supported": "moderate"
     },
     {
       "claim": "API hallucinations affect 44% of function-level code completion tasks.",
       "evidence": "Preliminary study (Section 2.1) with 50 function-level tasks: 13 correct, 22 of 37 incorrect had missing API usages. Two-author classification with Cohen's kappa = 0.76.",
       "supported": "moderate"
     },
     {
       "claim": "De-Hallucinator achieves 15.5% improvement in statement coverage for test generation.",
       "evidence": "Table 4 shows coverage increasing from 32.1% (TestPilot) to 37.0% (iterative), statistically significant by Wilcoxon test.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No confidence intervals or variance measures",
       "detail": "All results in Tables 3 and 4 are point estimates without confidence intervals, error bars, or standard deviations, despite results aggregating across diverse projects and tasks."
     },
     {
       "flag": "Hyperparameter tuning on evaluation data",
       "detail": "Hyperparameters k and n are selected based on experiments (RQ3, Figures 7-8) using the same data that reports final results. No separate validation set is used, risking overfitting to the evaluation dataset."
     },
     {
       "flag": "Contamination risk from popular training projects",
       "detail": "Evaluation uses highly popular GitHub projects (scikit-learn 58.5k stars, black 37.6k stars) that almost certainly appeared in model training data. Only exact-match filtering is applied, which may not catch near-memorization."
     },
     {
       "flag": "Small preliminary study sample",
       "detail": "The claim that API hallucinations affect 44% of tasks is based on only 50 function-level completions from a single model (CodeGen 2.5 7B). This small sample is presented as motivation for the entire approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex evaluation establishing benchmarks for LLM code generation, foundational to the code generation evaluation methodology."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": [
         "Max Schäfer",
         "Sarah Nadi",
         "Aryaz Eghbali",
         "Frank Tip"
       ],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3334955",
       "relevance": "TestPilot test generation system used as the baseline for the test generation evaluation in this paper."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": [
         "Raymond Li",
         "Loubna Ben Allal"
       ],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM used as one of four evaluated models for code completion."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": [
         "Erik Nijkamp",
         "Bo Pang",
         "Hiroaki Hayashi"
       ],
       "year": 2022,
       "relevance": "Code generation LLM family used as two of the four evaluated models (CodeGen 2B, CodeGen 2.5 7B)."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive nlp tasks",
       "authors": [
         "Patrick Lewis",
         "Ethan Perez",
         "Aleksandra Piktus"
       ],
       "year": 2020,
       "relevance": "Foundational RAG technique that De-Hallucinator extends with iterative retrieval based on model predictions."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": [
         "Jenny T. Liang",
         "Chenyang Yang",
         "Brad A. Myers"
       ],
       "year": 2024,
       "doi": "10.1145/3597503.3608128",
       "relevance": "Survey documenting developer perceptions of AI programming assistant limitations including project-specific API issues."
     },
     {
       "title": "An Empirical Evaluation of GitHub Copilot's Code Suggestions",
       "authors": [
         "Nhan Nguyen",
         "Sarah Nadi"
       ],
       "year": 2022,
       "doi": "10.1145/3524842.3528470",
       "relevance": "Empirical study of Copilot code suggestion quality, documenting hallucination of non-existing APIs."
     },
     {
       "title": "Code Generation Tools (Almost) for Free? A Study of Few-Shot, Pre-Trained Language Models on Code",
       "authors": [
         "Patrick Bareiß",
         "Beatriz Souza",
         "Marcelo d'Amorim",
         "Michael Pradel"
       ],
       "year": 2022,
       "arxiv_id": "2206.01335",
       "relevance": "Study of few-shot LLM code generation capabilities relevant to understanding LLM code generation limitations."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": [
         "Disha Shrivastava",
         "Hugo Larochelle",
         "Daniel Tarlow"
       ],
       "year": 2023,
       "relevance": "Closely related work on repository-level context selection for code completion, training a separate model for context ranking."
     },
     {
       "title": "CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models",
       "authors": [
         "Caroline Lemieux",
         "Jeevana Priya Inala",
         "Shuvendu K Lahiri",
         "Siddhartha Sen"
       ],
       "year": 2023,
       "relevance": "Uses LLMs to augment automated test generation when stuck, relevant to LLM-assisted test generation approaches."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": [
         "Fengji Zhang",
         "Bei Chen",
         "Yue Zhang"
       ],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Concurrent work on iterative retrieval for repository-level code completion, retrieving code fragments rather than API signatures."
     },
     {
       "title": "ReACC: A Retrieval-Augmented Code Completion Framework",
       "authors": [
         "Shuai Lu",
         "Nan Duan",
         "Hojae Han"
       ],
       "year": 2022,
       "arxiv_id": "2203.07722",
       "relevance": "Retrieval-augmented code completion using similar code pieces as dead code, precursor to retrieval-based approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Presents a usable technique for reducing API hallucinations in code completion with open-source code available, though requires integration work."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The iterative grounding idea is clever but the finding that LLMs hallucinate project-specific APIs is well-known, not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle — purely about improving code generation accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforwardly improves on baselines without challenging any company or popular belief."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub but requires setting up CodeQL, embedding models, and specific LLMs — significant setup effort."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University of Stuttgart authors, not a well-known AI lab; models used (CodeGen, UniXcoder) are not household names."
     }
   }
 }