ai-research-survey

scan.json (31364B)

---

 {
   "paper": {
     "title": "Citation-Grounded Code Comprehension: Preventing LLM Hallucination Through Hybrid Retrieval and Graph-Augmented Context",
     "authors": ["Jahidul Arafat"],
     "year": 2025,
     "venue": "arXiv (submitted to ICSE)",
     "arxiv_id": "2512.12117"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided: https://github.com/jahidul-arafat/code_comprehension_sdk_design_with_citation_verification (noted as 'anonymized for review'). The paper also states 'All code, evaluation data, and experiment configurations will be made publicly available under MIT license upon acceptance.' The URL is provided now, so this counts as YES."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses 30 publicly available open-source Python repositories (Flask, Django, FastAPI, NumPy, etc.) with permissive licenses. The paper states all evaluation data will be made publicly available. The repositories are public benchmarks they did not modify."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides detailed version specifications: Python 3.10.12, FAISS 1.7.4, Neo4j CE 5.12.0, Transformers 4.35.0, PyTorch 2.1.0, rank-bm25 0.2.2, NumPy 1.24.3, SciPy 1.11.3. Section 5.2.1 states 'versions pinned in requirements.txt'. Hardware is specified (CloudLab c6420, dual Xeon Gold 6142, 376GB RAM)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the system architecture and pipeline in detail (Section 5), and mentions a run_experiment.py orchestrator with YAML configurations, no explicit step-by-step reproduction instructions (e.g., README with commands to run) are described in the paper itself. The system is described at an architectural level but not with 'run this command' specificity."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates (e.g., 92% citation accuracy, 78% dense-only, 74% sparse-only) without confidence intervals or error bars. Table 7 reports std dev for the Flask/Werkzeug case study (e.g., Citation Accuracy 100% ± 0%), but the main results across 180 questions lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance tests are reported: 'Paired t-test confirmed statistical significance with p < 0.001 and Cohen's d = 1.83' (Section 3.2). Table 3 reports p-values for all metrics. Table 4 reports p < 0.001 for graph expansion improvements. Correlation significance p < 0.01 for model compliance (Section 4.3)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d = 1.83 is reported for the main hybrid vs. single-mode comparison (Section 3.2). Percentage point improvements with baseline context are reported throughout (e.g., '14-18 percentage point improvement' from 74-78% to 92%). Correlation coefficients (r = -0.72) are reported for model compliance vs. hallucination."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 180 questions, 30 repositories, or 6 models were chosen. No power analysis is discussed. The 5-participant developer study (Section 5.4.2) has a very small sample with no justification for the size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We test each question-model combination once (no repeated trials) as temperature 0.2 produces near-deterministic outputs' (Section 5.2.2). No variance across runs is reported since experiments are single-run. Table 7 reports std dev for some metrics in the Flask case study, but these are across questions not across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: sparse-only (BM25), dense-only (BGE), hybrid without graph, and three external baselines (keyword search via GitHub, neural code search via Sourcegraph, ChatGPT without retrieval). Finding 14 in Section 5.4 presents comparative analysis."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are primarily internal ablations (sparse-only, dense-only). External baselines include ChatGPT (GPT-4 November 2023) and Sourcegraph, but no contemporary RAG-based code comprehension systems are compared. The paper acknowledges in Table 1 that no prior system combines hybrid retrieval with graph expansion and citation verification, but does not compare against recent RAG frameworks like RAGA or LangChain-based systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts systematic ablations: sparse-only vs. dense-only vs. hybrid (RQ1), text-only vs. graph-expanded (RQ2), three packing strategies (RQ5), and hyperparameter sweeps across fusion weights, candidate counts, context budgets, and graph parameters (RQ4). Section 4.1-4.5 present detailed ablation results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: citation accuracy, precision, recall, F1-score, evidence diversity (unique files), hallucination rate, self-citation rate, retrieval latency, and end-to-end time. Tables 3, 4, 5, and 7 present these across different dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4.2 describes a developer evaluation with 5 participants performing 2-hour comprehension tasks. Manual review of 30% of responses (324 responses) for hallucination detection is also reported (Section 4.3). The system received 4.2/5 average usefulness rating."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No separation between development and test sets is described. The paper does not mention holding out any questions or repositories for tuning versus final evaluation. Hyperparameters (alpha=0.45, k=28, gamma=0.25) appear to be tuned and evaluated on the same 180 questions and 30 repositories."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by retrieval mode (Table 3), by model (Table 5), by query type (cross-file 62.3% vs. single-file 37.7%), by architectural pattern (exception handling 30%, utility delegation 40%, configuration coupling 29%), and by packing strategy (Section 4.5). The Flask/Werkzeug case study provides per-repository detail."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed extensively. Section 4.1 identifies three failure patterns for sparse retrieval and three for dense retrieval. Section 4.3 identifies four failure modes for model citation compliance. Section 5.4.2 identifies 4 common failure modes from developer evaluation. Section 5.5 discusses limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 8% of responses were flagged as hallucinations, sparse-only and dense-only both underperform, per-chunk 120 lines reduced accuracy to 90% (Section 4.4), and Mistral-7B achieved only 62% compliance with 19% hallucination. The 5-participant study reported limitations users found."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 92% citation accuracy (supported by Table 3), zero hallucinations (supported by mechanical verification results), 14-18pp improvement over baselines (supported by Section 4.1), 62% cross-file evidence discovery (supported by Section 3.2 Finding 2), and 24pp citation completeness improvement from graph expansion (supported by Table 4). All claims are backed by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies: removing graph expansion reduces cross-file completeness by 24pp (Table 4), changing packing strategy changes diversity by 18% (Section 4.5). These are controlled single-variable manipulations. The correlation claim (r=-0.72 between compliance and hallucination) is appropriately stated as correlation, not causation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Comprehension' generally but results are only on Python repositories. The abstract does not bound this to Python. Section 5.5.1 acknowledges 'Current implementation targets Python codebases exclusively' but the title and abstract present findings as general code comprehension results. The paper says extending to other languages 'requires language-specific AST parsers' but does not bound claims in the abstract or title."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, could the 92% accuracy be due to the specific question design favoring cross-file patterns? Could the 14-18pp improvement be attributed to the specific repositories chosen rather than hybrid retrieval superiority? The limitations section (5.5) discusses scope limitations but not alternative explanations for the results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model filenames with quantization formats are specified in Section 5.2.1: 'llama-3-groq-8b-8192-tool-use-preview-Q4_K_M.gguf', 'codellama-13b-instruct.Q4_K_M.gguf', 'mistral-7b-instruct-v0.3.Q4_K_M.gguf', 'deepseek-coder-6.7b-instruct.Q4_K_M.gguf', 'Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf', 'Phi-3-mini-4k-instruct-Q4_K_M.gguf'. BGE-base-en-v1.5 is also specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the system prompt in natural language: 'System prompt enforces citation format: \"You must cite code locations using [file:start-end] format...\"' (Section 5.2.2). However, the full prompt text used for LLM generation is not provided. The paper describes what the prompt does but does not provide the complete actual prompt used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters are reported in Section 5.2.2: fusion weights alpha=0.45, beta=0.55; k=28 candidates; graph bonus gamma=0.25; decay delta=0.6; context budget 11,000-12,000 characters; per-chunk 100 lines. LLM parameters: temperature 0.2, top-p 0.9, top-k 40, max tokens 1024, repetition penalty 1.1."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described in extensive detail: parallel BM25 and FAISS retrieval, hybrid fusion with normalization, graph expansion via Neo4j BFS, submodular context packing, LLM generation with citation format requirements, and mechanical citation verification. Section 3.1.3 and Section 5.1 describe the full system architecture with four core modules."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.3 describes the four-stage preprocessing pipeline in detail: AST parsing for chunking, embedding generation with BGE, FAISS index construction, and graph loading from import statements. Section 3.1.1 describes repository selection criteria. Section 3.1.2 describes question generation methodology with template-based and issue-mining approaches."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Limitations and Future Work' provides extensive discussion across six subsections: Language and Paradigm Coverage (5.5.1), Scalability Boundaries (5.5.2), Query Understanding Limitations (5.5.3), Citation Granularity Trade-offs (5.5.4), Evaluation Methodology (5.5.5), and Privacy and Security Considerations (5.5.6)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses specific threats: Python-only evaluation (5.5.1), FAISS degradation at 100K+ chunks (5.5.2), single-run experiments with temperature 0.2 (implied in 5.2.2), questions skewing toward architectural comprehension (5.5.5), repositories selected for popularity not representative sampling (5.5.5), and the 5-participant developer study's small size."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5.1 explicitly states 'Current implementation targets Python codebases exclusively.' Section 5.5.5 states 'questions skew toward architectural comprehension over debugging or performance analysis, repositories selected for popularity and documentation quality rather than representative sampling.' Section 5.5.3 lists unsupported query types: temporal, counterfactual, and debugging queries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper states code and data will be 'made publicly available under MIT license upon acceptance' but this is conditional on acceptance. No raw data (per-question results, retrieval logs, per-response analysis) is currently available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 describes repository selection with four criteria (open-source, active development, moderate complexity, domain diversity). Section 3.1.2 describes question generation through template-based (80-120 questions) and issue mining (40-60 questions from 500+ candidates). Table 2 lists all 30 repositories with metadata."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the 5-participant developer evaluation (Section 5.4.2), the paper states '5 software developers (3 with Flask experience, 2 without)' but does not describe how they were recruited, from what population, or whether selection could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented: repository selection criteria (Section 3.1.1), question generation with counts (Section 3.1.2: template-based 80-120 + issue mining 40-60 from 500+ candidates = 180 final), preprocessing stages (Section 5.1.3), and evaluation protocol (Section 5.2.2). The question distribution is reported: 62.3% cross-file, 37.7% single-file, 45.2% API-specific, 54.8% conceptual."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation states 'Presidential and Woltosz Graduate Research Fellow, Department of Computer Science and Software Engineering, Auburn University.' The acknowledgments mention 'computational resources provided by Auburn University.' The fellowship funding is disclosed through the affiliation."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is clearly disclosed: 'Jahidul Arafat — Principal Investigator (PI); Presidential and Woltosz Graduate Research Fellow, Department of Computer Science and Software Engineering, Auburn University, Alabama, USA.' The paper evaluates open-source tools and local LLMs, not the author's employer's products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funding comes from Auburn University through a graduate fellowship. The university has no financial stake in the outcome of the research on hybrid code retrieval systems. The evaluated tools are open-source (FAISS, Neo4j Community Edition, BGE, LM Studio)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is found in the paper. While the research appears academic with no obvious commercial conflicts, the absence of a formal competing interests declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 6 LLMs on code comprehension tasks but does not state the training data cutoff dates for any of the models used. For the ChatGPT baseline, it mentions 'November 2023' as the version date but not the training cutoff. The models could have seen the evaluated repositories during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the 30 Python repositories (Flask, Django, NumPy, etc.) were in the training data of the evaluated models. These are extremely popular repositories almost certainly present in training data. The paper briefly mentions 'training data staleness' but does not address the overlap with evaluation data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 30 repositories used for evaluation (Flask, Django, PyTorch, etc.) are among the most popular Python projects and were certainly available online before any model's training cutoff. The paper does not address whether model familiarity with these codebases could inflate citation accuracy or comprehension quality metrics."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The 5-participant developer evaluation (Section 5.4.2) is not pre-registered. No link to a pre-registration platform is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the developer evaluation study involving 5 human participants (Section 5.4.2)."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The only demographic information provided is '5 software developers (3 with Flask experience, 2 without)' (Section 5.4.2). No experience levels, years of experience, gender, geographic distribution, or other demographics are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for the 5 developer participants. The paper does not describe who was eligible, how they were screened, or why 5 was chosen."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The developer study uses a within-subjects design (participants first use their tools, then the system), but no randomization or counterbalancing of task order is described. All participants appear to follow the same sequence, introducing potential order effects."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described for the developer evaluation. Participants knew which tool they were using (their preferred tools vs. the proposed system). The within-subjects design with no blinding could introduce expectation bias."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about whether all 5 participants completed all tasks. Attrition or dropout is not discussed."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed latency is reported: end-to-end 16.5 seconds average (Section 3.2), per-stage breakdown (BM25 85ms, FAISS 120ms, graph 27ms, packing 45ms, verification 15ms, LLM generation 11.4s). Per-model generation times are reported in Table 5 (8.5-12.1 seconds). The Flask case study reports 10.1s average total latency (Table 7). No monetary API costs since local inference is used."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.2 states 'Total experimental budget: 180 questions × 6 models × 5 ablation conditions = 5,400 trials requiring approximately 180 machine-hours across 6 repositories.' Hardware specified as CloudLab c6420 machines with dual Intel Xeon Gold 6142 CPUs, 376GB RAM."
       }
     }
   },
   "claims": [
     {
       "claim": "Hybrid retrieval combining BM25 and BGE dense embeddings achieves 92% citation accuracy, outperforming single-mode approaches by 14-18 percentage points.",
       "evidence": "Section 3.2 Finding 1 and Table 3 report 92% hybrid vs. 78% dense-only and 74% sparse-only, with paired t-test p < 0.001 and Cohen's d = 1.83.",
       "supported": "strong"
     },
     {
       "claim": "62.3% of code comprehension questions require cross-file evidence spanning multiple modules.",
       "evidence": "Section 3.2 Finding 2 reports manual analysis of 180 questions showing 112/180 (62.3%) require multi-file citations, broken down into three patterns: exception handling (34), utility delegation (45), configuration coupling (33).",
       "supported": "moderate"
     },
     {
       "claim": "Graph expansion discovers average 11.8 cross-file neighbors per query, increasing cross-file citation completeness by 24 percentage points (from 58% to 82%).",
       "evidence": "Table 4 and Section 4.2 Finding 7 report these numbers with p < 0.001 significance. Analysis covers 112 cross-file questions.",
       "supported": "strong"
     },
     {
       "claim": "Citation verification prevents hallucination with zero false positives across 1,080 verified responses.",
       "evidence": "Section 3.2 Finding 3 reports 100% precision on hallucination detection. Manual inspection of 30% sample (324 responses) found 97% true positive rate (31/32 genuine hallucinations).",
       "supported": "moderate"
     },
     {
       "claim": "The system achieves 100% citation accuracy on Flask and Werkzeug with zero hallucinations.",
       "evidence": "Table 7 and Section 5.4 Finding 13 report 100% accuracy across 30 questions with 156 citations verified. However, this is on only one of the 30 repository pairs and may not generalize.",
       "supported": "moderate"
     },
     {
       "claim": "Developer evaluation shows citation-grounded approach reduces time-to-understanding by 60% compared to manual code search.",
       "evidence": "Section 5.4.2 Finding 15 reports reduction from 6.8 minutes to 2.7 minutes across 5 developers. Answer completeness improved from 65% to 95%.",
       "supported": "weak"
     },
     {
       "claim": "Model citation compliance varies from 88% (DeepSeek-Coder) to 62% (Mistral) with strong negative correlation (r = -0.72) between self-citation rate and hallucination rate.",
       "evidence": "Table 5 and Section 4.3 Finding 9 present per-model compliance and hallucination statistics across 1,080 responses (180 questions x 6 models), with p < 0.01 for the correlation.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper develops a hybrid retrieval system combining BM25 sparse matching, BGE dense embeddings, and Neo4j graph expansion for citation-grounded code comprehension. Evaluated across 30 Python repositories with 180 queries and 6 LLMs, the system achieves 92% citation accuracy, outperforming single-mode baselines by 14-18 percentage points. Graph expansion via import relationships discovers cross-file evidence in 62% of architectural queries, improving citation completeness by 24 percentage points, while mechanical citation verification prevents hallucination with zero false negatives across 1,080 verified responses.",
   "red_flags": [
     {
       "flag": "No train-test separation for hyperparameter tuning",
       "detail": "Hyperparameters (alpha=0.45, k=28, gamma=0.25) appear to be tuned on the same 180 questions used for final evaluation. No held-out test set is described. This could overfit parameters to the evaluation set, inflating reported accuracy."
     },
     {
       "flag": "Single-run experiments with no variance reporting",
       "detail": "Each question-model combination is tested once with temperature 0.2, justified as 'near-deterministic.' No repeated runs or variance across seeds are reported. Temperature 0.2 does not produce identical outputs, and result stability is unverified."
     },
     {
       "flag": "Very small user study (N=5) with no controls",
       "detail": "The developer evaluation (Section 5.4.2) uses only 5 participants, has no randomization or counterbalancing of tool order, no blinding, no IRB approval mentioned, and no demographic reporting. The within-subjects design with fixed ordering (manual first, then system) introduces learning and order effects that could inflate the 60% time improvement claim."
     },
     {
       "flag": "Contamination risk with popular repositories",
       "detail": "The 30 evaluated repositories (Flask, Django, NumPy, PyTorch, etc.) are among the most popular Python projects, almost certainly present in the training data of all evaluated models. The paper does not discuss how model familiarity with these codebases could affect results."
     },
     {
       "flag": "Generalization claims exceed evidence",
       "detail": "The title and abstract present findings as applicable to 'code comprehension' generally, but the system is Python-only, evaluated on popular well-documented repositories, with questions skewed toward architectural comprehension. The paper acknowledges this in limitations but the framing is broader than the evidence supports."
     },
     {
       "flag": "Solo author claiming PI status on own work",
       "detail": "The paper lists a single author (a graduate student) who designates themselves as 'Principal Investigator (PI).' This is unusual for a graduate student paper and no advisor or collaborators are listed as co-authors, though the acknowledgments thank 'our advisor.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on evaluating LLM code generation capabilities (Codex/HumanEval), directly relevant to code generation evaluation methodology."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke"],
       "year": 2023,
       "relevance": "Empirical study of developer interaction with code-generating models, relevant to understanding AI coding assistant productivity."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used as baseline for code embeddings in retrieval, relevant to code understanding models."
     },
     {
       "title": "GraphCodeBERT: Pre-training Code Representations with Data Flow",
       "authors": ["Daya Guo"],
       "year": 2021,
       "relevance": "Code representation model incorporating data flow graphs, relevant to structural code understanding approaches."
     },
     {
       "title": "CodeRetriever: Retrieved Code is Better Than You Think",
       "authors": ["Yao Li"],
       "year": 2022,
       "relevance": "Hybrid keyword and embedding code retrieval system, directly comparable baseline for code search methodology."
     },
     {
       "title": "The Causes of and Solutions to Hallucinations in Language Models",
       "authors": ["Tianyi Zhang"],
       "year": 2023,
       "arxiv_id": "2311.09417",
       "relevance": "Survey of LLM hallucination including code-specific hallucination patterns, directly relevant to AI safety and reliability."
     },
     {
       "title": "SelfCheck: Detecting LLM Hallucinations Without External Knowledge",
       "authors": ["Potsawee Manakul"],
       "year": 2023,
       "relevance": "Self-consistency based hallucination detection method, comparable approach to the citation verification mechanism evaluated."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis"],
       "year": 2020,
       "relevance": "Foundational RAG paper that the citation-grounded approach extends, relevant to understanding RAG methodology."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM evaluated in this study and widely used for code generation benchmarks."
     },
     {
       "title": "Mistral 7B",
       "authors": ["Albert Q Jiang"],
       "year": 2023,
       "arxiv_id": "2310.06825",
       "relevance": "General-purpose LLM evaluated in this study, relevant to understanding model capability variation across architectures."
     },
     {
       "title": "LLMs for Code Generation: Assessing Correctness and Hallucinations",
       "authors": ["Yu Huang"],
       "year": 2023,
       "relevance": "Study characterizing LLM hallucination specifically in code generation tasks, directly relevant to AI code reliability assessment."
     },
     {
       "title": "Expectations vs. Experience: Empirical Effects of Code Generation Assistants",
       "authors": ["Priyan Vaithilingam"],
       "year": 2022,
       "relevance": "Empirical study of developer experience with code generation tools, relevant to AI programming productivity evaluation."
     }
   ]
 }