ai-research-survey

scan.json (31412B)

---

 {
   "paper": {
     "title": "An Empirical Study of Retrieval-Augmented Code Generation: Challenges and Opportunities",
     "authors": [
       "Zezhou Yang",
       "Sirong Chen",
       "Cuiyun Gao",
       "Zhenhao Li",
       "Xing Hu",
       "Kui Liu",
       "Xin Xia"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2501.13742",
     "doi": "10.1145/3717061"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Retrieval-augmented framework (RAF) consistently improves code generation performance across CodeGen, UniXcoder, and CodeT5 on CONCODE, CoNaLa, and HearthStone datasets. BM25, despite being the simplest retrieval technique requiring no training, outperforms more complex neural retrieval methods including CodeBERT, UniXcoder, and CoCoSoDa on most configurations. Sequential Integration Fusion is the most cost-effective fusion strategy, while Sketch Filling Fusion achieves the best performance at 2-7x training cost. The framework also benefits LLMs (ChatGLM3, CodeLlama, DeepSeek-Coder) when used via prompt engineering at inference time.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a code repository at https://github.com/watreyoung/RACG (footnote 4) and retrieval-augmented datasets at a Google Drive link (footnote 3)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (CONCODE, CoNaLa, HearthStone) and additionally releases retrieval-augmented datasets via Google Drive (footnote 3). The standard benchmarks are all publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch, Huggingface, and hardware (Intel Xeon Platinum 8276, NVIDIA A100 80G), but does not provide a requirements.txt, Dockerfile, or specific library versions. Section 4.4 states only the framework and hardware without sufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper does not include step-by-step reproduction instructions. Section 4.4 says 'all the hyper-parameter settings of pre-trained models are the same as the original corresponding papers' without providing specific reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results tables (Tables 3-8) report only point estimates. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states: 'We perform a statistical significance test (t-test), and the results show that the models with the RAF outperform the original models at the significance level at 0.05 (p-value 0.035).'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout (e.g., 'an average improvement of 41.60% in the EM metric' in Section 5.2) and Tables 4 and 6 include percentage improvements in parentheses alongside absolute values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of three models, three datasets, or the specific dataset sizes. No power analysis is discussed. The paper states datasets are 'well recognized' and 'widely-used' but does not justify sample adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported in any table. All results appear to be from single experimental runs without reporting result stability across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares retrieval-augmented models against base models (CodeGen, UniXcoder, CodeT5 fine-tuned on original datasets). Table 4 compares five different retrieval techniques against each other and the baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The pre-trained models (CodeGen, UniXcoder, CodeT5) and LLMs (ChatGLM3-6B, CodeLlama-7B, DeepSeek-Coder-6.7B) were contemporary at the time of writing. CoCoSoDa is described as achieving 'state-of-the-art performance' for code search."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies retrieval techniques (RQ2), fusion strategies (RQ3), number of retrieved snippets (Section 5.4.1), and ordering of snippets (Table 8), effectively ablating different components of the retrieval-augmented framework."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: Exact Match (EM), BLEU-4, Edit Distance (ED), SimilarityAST, and CodeBLEU, as described in Section 4.3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated using the five metrics. The case studies in Section 6.2 are qualitative illustrations by the authors, not systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes separate train/test/validation splits for all three datasets. CONCODE has 100k/2k/2k, CoNaLa has 2179/500/200, HearthStone has 533/66/66. Results are reported on test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (CONCODE, CoNaLa, HearthStone), by model, by retrieval technique (Table 4), and by fusion strategy (Table 5). This provides detailed per-category analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses RetroMAE's performance degradation (7.74% and 1.32% BLEU drops on CONCODE). Section 5.4.2 discusses VDF's decreased performance compared to SEF. Section 6.2.2 analyzes why UniXcoder retrieves suboptimal results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RetroMAE decreases CodeGen performance by 7.74% BLEU on CONCODE and causes 81.33% drop on HearthStone (Table 4). VDF performs worse than SEF across all datasets (Table 5). CoCoSoDa decreases performance on HearthStone for CodeGen."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about RAF improving performance, BM25's effectiveness, SIF's convenience, and SFF's further improvement are all supported by Tables 3-8 and the analysis in Sections 5.2-5.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claims (RAF improves performance) are supported by controlled comparisons: same models with and without retrieval augmentation, same models with different retrieval techniques, same models with different fusion strategies. The single-variable manipulation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Finding 2 states 'BM25 is proven to be the most effective retrieval technique for code generation' — an overly broad claim from 3 datasets and 3 models. The title claims 'Code Generation' generally while testing only Java and Python. The threats section acknowledges uncertainty about larger models but the findings themselves are stated without bounds."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses why RetroMAE underperforms (NLP vs code domain gap) and why VDF fails (encoder/decoder limitations), it does not discuss alternative explanations for the main findings, such as whether improvements come from data augmentation effects rather than retrieval quality, or whether memorization of training examples explains the gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures BLEU, CodeBLEU, and EM but frames results as 'code generation performance' without discussing the gap between n-gram matching metrics and actual code correctness or functional correctness. The paper acknowledges pass@k datasets exist (Section 2.2) but does not use them or discuss why proxy metrics may be insufficient."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 specifies model names with parameter counts: CodeGen-MONO (350M), UniXcoder (126M), CodeT5 (223M). LLMs are specified as ChatGLM3-6B, CodeLlama-7B, DeepSeek-Coder-6.7B. Pre-training data sources are listed."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "For LLM experiments in Section 6.1, the paper states 'The prompts are constructed following [43]' without providing the actual prompt text. For fine-tuned models, the input format is described conceptually but actual prompts/templates are not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4 states 'all the hyper-parameter settings of pre-trained models are the same as the original corresponding papers' without listing specific values. No learning rate, batch size, epoch count, or temperature settings are reported. BM25 parameters b and k are mentioned but values are not given."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper studies a retrieval-augmented framework with standard fine-tuning and prompting, not agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes dataset organization, splits, and how the CoNaLa validation set was created (200 random instances from training). Section 4.4 describes data format ('organized in the form of <Natural Language Description, Code Snippets> and stored in JSON files'). The retrieval database construction is described in Section 3.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 'Threats to Validity' contains three substantive subsections: generalization of model results, replication of experiments, and limited dataset."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific: (1) uncertainty about applicability to larger models or different architectures, (2) device and parameter setting effects on replication, (3) CONCODE's preprocessing makes it unintuitive for humans — 'it is difficult for human developers to write code the same as the ground truth.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 states findings may not apply to 'larger models or models with differing architectures' and acknowledges 'a distinct gap between the data within these datasets and the real, specific development environment context.' These are specific scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Retrieval-augmented datasets are released via Google Drive (footnote 3) and code is available at the GitHub repository (footnote 4). The underlying benchmarks (CONCODE, CoNaLa, HearthStone) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes each dataset's origin and construction: CONCODE from ~33k Java GitHub projects, CoNaLa from 2,879 Stack Overflow annotations, HearthStone from 665 card implementations. Table 1 provides detailed statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-phase pipeline is clearly documented: Retrieval Phase retrieves top-k code snippets (Section 3.2), Fusion Phase integrates them with input (Section 3.3), Generation Phase produces code (Section 3.4). Section 4.4 describes the learning process. Dataset splits and validation creation are documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is mentioned anywhere in the paper text. One author is from Huawei Technologies but no funding disclosure accompanies this."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology (Shenzhen), Concordia University, Zhejiang University, and Huawei Technologies Co., Ltd."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. One author is from Huawei Technologies, which has commercial interest in code generation (PanGu-Coder is cited), creating a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data sources are mentioned (The Pile, BigQuery, BigPython for CodeGen; CodeSearchNet for UniXcoder and CodeT5) but no training data cutoff dates are provided for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark data (CONCODE, CoNaLa, HearthStone) could appear in the pre-training corpora of the evaluated models. The models are pre-trained on large code corpora that could contain these benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CONCODE (2018), CoNaLa (2018), and HearthStone (2016) were all published years before the pre-trained models were trained. Models trained on GitHub/StackOverflow data could easily contain these benchmarks, but this contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of pre-trained models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports inference costs (in seconds) for each fusion strategy across all three datasets. Table 7 reports retrieval costs per 50 instances for each retrieval technique. Section 6.1.2 discusses cost trade-offs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 specifies hardware (two Intel Xeon Platinum 8276 CPUs, two NVIDIA A100 80G). Table 5 reports total training time (e.g., 128-923 min on CONCODE). Table 7 reports per-epoch training costs and retrieval costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single experimental runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated anywhere. It is unclear whether results are from single runs or averaged over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper defers to original papers for hyperparameter settings without indicating whether any search was performed for the retrieval-augmented configurations."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The selection of k=5 retrieved snippets is justified by experiments in Section 5.4.1 showing performance across k=1,3,5,7,10. The choice of ascending ordering is justified by Table 8. Model selections reference prior empirical comparisons."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs a single t-test (p-value 0.035) but runs many comparisons across 3 models × 3 datasets × 5 retrieval techniques × 5 metrics without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate all retrieval techniques and fusion strategies themselves without acknowledging the bias of evaluating their own implementations. No independent evaluation or acknowledgment of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 directly reports both performance metrics and training/inference costs for each fusion strategy side by side. Section 6.1.2 explicitly discusses the 'trade-off between performance improvement and computational costs.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 acknowledges that CONCODE's 'pre-processing makes human hard to understand the code intuitively' and that 'it is difficult for human developers to write code the same as the ground truth,' questioning whether these benchmarks reflect real-world development."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The study evaluates fine-tuned models and prompted LLMs directly without scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. The benchmarks (2016-2018) were created before the pre-trained models' training periods, meaning solutions could have been in training data. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The retrieval database is the training set, meaning the model has already been fine-tuned on the same data it retrieves from. The paper does not discuss whether this creates a feature leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Only partially addressed for CONCODE, which uses 'repository-based partitioning' to separate domains. No independence analysis is performed for CoNaLa or HearthStone, and no systematic discussion of non-independence is provided."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "Retrieval-augmented framework universally improves code generation performance across different pre-trained models and datasets.",
       "evidence": "Table 3 shows average improvements of 6.79% EM, 11.45% BLEU, 6.93% SimilarityAST, and 8.72% CodeBLEU on CONCODE; 3.74%, 18.42%, 15.51%, and 16.75% on CoNaLa; 41.60%, 9.01%, 11.25%, and 8.69% on HearthStone (Section 5.2). Statistical significance test yields p-value 0.035.",
       "supported": "strong"
     },
     {
       "claim": "BM25 is the most effective retrieval technique for the retrieval-augmented framework, outperforming neural methods.",
       "evidence": "Table 4 shows all three models achieve highest gains with BM25 on CONCODE and HearthStone. On CoNaLa, BM25 is optimal for CodeT5 (25.69% BLEU improvement) and suboptimal for CodeGen (Section 5.3). CoCoSoDa sometimes matches BM25 on CoNaLa.",
       "supported": "moderate"
     },
     {
       "claim": "Sequential Integration Fusion is the most recommended fusion strategy balancing cost and performance.",
       "evidence": "Table 5 shows SIF training time is the shortest (285 min vs 923 min for SEF on CONCODE). SIF achieves competitive or best performance on CoNaLa. On HearthStone, SFF outperforms SIF but costs 107 min vs 60 min (Section 5.4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Sketch Filling Fusion achieves the highest performance improvement among fusion strategies.",
       "evidence": "Table 5 shows SFF yields highest BLEU on CONCODE (40.84) and HearthStone (81.89). Section 5.4.2 reports average 14.83% BLEU and 8.05% CodeBLEU improvement for SFF based on SEF. However, SFF performs worse than SIF on CoNaLa.",
       "supported": "moderate"
     },
     {
       "claim": "RAF is effective for LLMs in code generation during inference without fine-tuning.",
       "evidence": "Table 6 shows substantial improvements: ChatGLM BLEU ratio of 198.67 on HearthStone with BM25, CodeLlama BLEU ratio of 147.00 on HearthStone with BM25. All three LLMs improve across all datasets and retrieval techniques (Section 6.1).",
       "supported": "moderate"
     },
     {
       "claim": "More complex retrieval techniques do not necessarily lead to better code generation results.",
       "evidence": "Table 4 shows RetroMAE (a neural retrieval model) decreases CodeGen's BLEU by 7.74% and causes 81.33% drop on HearthStone. BM25, requiring no training, outperforms trained code search models in most configurations (Section 5.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or multiple seeds reported",
       "detail": "All results appear to be from single experimental runs. No standard deviations, confidence intervals, or seed sensitivity analysis is reported despite deep learning results being highly sensitive to random initialization (Henderson et al., 2018)."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The benchmarks (CONCODE 2018, CoNaLa 2018, HearthStone 2016) predate all evaluated models' training periods. Models trained on GitHub/StackOverflow code could contain these benchmark solutions, but no contamination analysis is performed."
     },
     {
       "flag": "Overclaiming from limited scope",
       "detail": "Finding 2 states 'BM25 is proven to be the most effective retrieval technique for code generation' from only 3 datasets (2 languages) and 3 models. The use of 'proven' overstates the evidence. The title implies general code generation but tests are limited to Java and Python."
     },
     {
       "flag": "No human evaluation of generated code",
       "detail": "All evaluation relies on automated metrics (BLEU, CodeBLEU, EM) which measure surface-level similarity. No functional correctness evaluation (pass@k) or human assessment of code quality is performed despite the paper framing results as 'code generation performance.'"
     },
     {
       "flag": "Huawei affiliation without conflict disclosure",
       "detail": "One author (Kui Liu) is from Huawei Technologies, which has commercial code generation products (PanGu-Coder). No funding disclosure or competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark, foundational to LLM-based code generation evaluation."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2023,
       "relevance": "One of three pre-trained models evaluated in this study; pioneered conversational code generation."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang"],
       "year": 2021,
       "relevance": "One of three pre-trained models evaluated; unified encoder-decoder architecture for code tasks."
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "authors": ["Md. Rizwan Parvez", "Wasi Uddin Ahmad"],
       "year": 2021,
       "relevance": "REDCODER: key prior work on retrieval-augmented code generation enhancing both generation and summarization."
     },
     {
       "title": "Skcoder: A sketch-based approach for automatic code generation",
       "authors": ["Jia Li", "Yongmin Li"],
       "year": 2023,
       "arxiv_id": "2302.06144",
       "relevance": "Introduces Sketch Filling Fusion, one of the four fusion strategies evaluated in this study."
     },
     {
       "title": "DocPrompting: Generating Code by Retrieving the Docs",
       "authors": ["Shuyan Zhou", "Uri Alon"],
       "year": 2023,
       "relevance": "Retrieval-augmented code generation through documentation retrieval; alternative RAG approach for code."
     },
     {
       "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
       "authors": ["Hung Le", "Yue Wang"],
       "year": 2022,
       "relevance": "Uses RL to improve code generation from pre-trained models; alternative approach to improving code quality."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming - The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of three LLMs used in the RAF extension experiments; represents modern open-source code LLMs."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of three LLMs used in the RAF extension experiments; Meta's open code generation model."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick S. H. Lewis", "Ethan Perez"],
       "year": 2020,
       "relevance": "Foundational RAG paper defining the retriever-and-generator framework that this work extends to code generation."
     },
     {
       "title": "An Empirical Comparison of Pre-Trained Models of Source Code",
       "authors": ["Changan Niu", "Chuanyi Li"],
       "year": 2023,
       "relevance": "Empirical comparison of pre-trained code models that informed model selection in this study."
     },
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["Akari Asai", "Zeqiu Wu"],
       "year": 2024,
       "relevance": "Active retrieval approach for RAG, cited as future work direction for retrieval-augmented code generation."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "First bimodal pre-trained model for code; used as both a retrieval technique and baseline in this study."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners building RAG systems for code generation can directly apply the recommendation to use BM25 + Sequential Integration Fusion."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that simple BM25 outperforms neural retrieval models is mildly surprising but aligns with known patterns in information retrieval."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward empirical study with no controversy or conflict."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code repository is available on GitHub but requires substantial setup with GPU hardware and multiple models to replicate."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from academic institutions and Huawei; not associated with high-profile AI labs."
     }
   }
 }