ai-research-survey

scan.json (35352B)

---

 {
   "paper": {
     "title": "A Deep Dive into Retrieval-Augmented Generation for Code Completion: Experience on WeChat",
     "authors": [
       "Zezhou Yang",
       "Ting Peng",
       "Cuiyun Gao",
       "Chaozheng Wang",
       "Hailiang Huang",
       "Yuetang Deng"
     ],
     "year": 2025,
     "venue": "IEEE International Conference on Software Maintenance and Evolution (ICSME)",
     "arxiv_id": "2507.18515",
     "doi": "10.1109/ICSME64153.2025.00062"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "Both identifier-based and similarity-based RAG consistently improve code completion on WeChat's proprietary C++ codebase across 26 open-source LLMs (0.5B–671B parameters), with similarity-based RAG substantially outperforming identifier-based RAG (e.g., DeepSeek-V3 achieves 42.7% higher CodeBLEU). Among retrieval techniques, BM25 (lexical) and GTE-Qwen (semantic) achieve the best individual performance, and their combination yields optimal results for 7B+ models due to minimal overlap in retrieved candidates (64–76% completely distinct). A developer survey with 3 participants confirms the combined approach produces higher-quality completions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL, GitHub link, or archive is provided anywhere in the paper. The preprocessing algorithm and retrieval pipeline are described but not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark (100 examples) and retrieval corpus (1,669 repositories) are from WeChat's proprietary codebase and are not released. No download links or public data are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (8× A100 40GB, 8× H20 96GB, 16× H20 96GB) and vLLM is mentioned for deployment, but no requirements.txt, Dockerfile, library versions, or detailed environment setup is provided. FP16/FP8 precision is mentioned but this alone is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithmic description (Algorithm 1) describes preprocessing logic but does not constitute reproduction instructions for the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I, II, and III are reported as point estimates (e.g., '51.12/61.96') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'similarity-based RAG substantially performs better' and 'BM25 and GTE-Qwen-based retrieval techniques demonstrate superior performance' based solely on comparing numbers without any statistical significance tests (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context throughout, e.g., 'CB/ES metrics improvement from 29.79/48.56 to 51.12/61.96, representing a 71.60% and 27.59% relative increase' (Section IV-A), and 'DeepSeek-V3 shows an enhancement from 35.23/54.85 to 60.28/73.11 using GTE-Qwen retrieval technique, corresponding to a 71.1% and 33.3% increase.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 100 examples and the developer survey has only 3 participants evaluating 52 examples. No justification is given for why these sample sizes are adequate, and no power analysis is conducted."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic output, and results are from single runs. No variance, standard deviation, or spread measures are reported across any experimental conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Base models without RAG serve as baselines ('base' column in Table I). Each RAG method is compared against the base model performance, and different RAG methods are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes 26 contemporary LLMs including DeepSeek-V3 (2024), Qwen2.5-Coder (2024), Llama-3.3 (2024), and other recent model series, representing current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 compares identifier-based vs. similarity-based RAG with different retrieval types (message definition, class definition, function declaration, function definition). RQ2 compares 5 retrieval techniques with incomplete vs. complete queries. RQ3 explores combinations of techniques. These constitute ablation over retrieval components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two automated metrics are used: CodeBLEU (CB) measuring structural and semantic code similarity, and Edit Similarity (ES) measuring token-level edit distance. The developer survey adds a 1-5 human rating scale."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A describes a developer survey with 3 developers evaluating 52 code completions on a 1-5 scale, comparing BM25, GTE-Qwen, and their combination across 3 LLMs. Error patterns are also categorized."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 100-example benchmark is used for all evaluation. There is no mention of a separate dev/validation split for tuning hyperparameters (e.g., number of retrieved results, BM25 parameters). The same 100 examples are used for all results."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 7 domains (client call, connection, kv, colib, mq, utils, encoding) and easy/hard difficulty levels (Figure 1), but results in Tables I–III are reported only as overall averages across all 100 examples. No per-domain or per-difficulty breakdowns are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The developer survey (Section V-A, Figure 2c) identifies three error categories: Missing or Incorrect Logic (52%), Extra Logic (30%), and Nonexistent Function Call (17%), with per-model error distributions analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: CodeLlama-70B-Instruct shows degradation with RAG in most conditions; combination of retrieval techniques shows 'limited or even negative impact' for models below 7B (Section IV-C); CodeBERT 'consistently underperforms' other semantic retrieval techniques."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three key claims are supported: (1) both RAG methods effective with similarity-based superior — Table I; (2) BM25 and GTE-Qwen achieve superior performance — Table II; (3) combination of lexical and semantic yields optimal results — Table III."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'RAG methods consistently outperform base models') are justified by controlled comparisons where the same model is evaluated with and without RAG under identical conditions. The single-variable manipulation (adding RAG context) is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Finding 1 states 'Both types of RAG methods can consistently improve code completion performance across different models and scales in closed-source repositories,' generalizing from one C++ codebase (WeChat) to 'closed-source repositories' broadly. The paper tests only C++ code in one organization's codebase but does not bound claims to this setting in the findings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C discusses specific alternative explanations: internal validity (parameter settings and hardware could influence results), external validity (WeChat's codebase may differ from other organizations'), and construct validity (automated metrics may not capture semantic correctness). Section V-B discusses the gap between training and application scenarios for semantic retrieval."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures CodeBLEU and Edit Similarity and frames these as 'code completion performance' — a reasonable match. Section V-C explicitly acknowledges the proxy gap: 'automated metrics (CodeBLEU and Edit Similarity) to measure code quality... these metrics might not fully capture the semantic correctness and functionality of generated code,' and supplements with the developer survey."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided for all 26 LLMs (e.g., 'Qwen2.5-Coder-14B-Instruct', 'Llama-3.3-70B-Instruct', 'DeepSeek-V3-671B/37B'). Retrieval models are also specified (e.g., 'GTE-Qwen2-1.5B-instruct'). All models are obtained from 'official Hugging Face repositories' (Section III-D3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompt templates are described conceptually (e.g., 'we develop four distinct prompt templates to help LLMs understand different types of background knowledge,' Section II-C3) and Section III-D3 states 'we design our prompts in Chinese wrapped in C++ comment format,' but the actual prompt text is never provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature=0 for all models, number of retrieved results=4, prompt length <2k tokens, FP16/FP8 precision, CodeBLEU weights α=β=γ=δ=0.25 (Section III-C). BM25 formula parameters k and b are described but exact values are not given."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The RAG pipeline is a fixed multi-step process (retrieve → prompt → generate) without retry logic, feedback mechanisms, or autonomous tool use."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 provides detailed pseudocode for the data preprocessing pipeline, addressing file segmentation, recursive dependencies, auto-generated code, and macro handling. Section III-D1 describes the tree-sitter implementation and regex-based protobuf extraction."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C 'Threats to Validity' provides substantive discussion of internal, external, and construct validity threats across three dedicated paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: internal validity addresses that 'performance of these deep learning-based models can be influenced by multiple factors, including parameter settings and hardware devices'; external validity notes 'experiments are conducted on the specific enterprise codebase in WeChat group'; construct validity specifically identifies that 'automated metrics might not fully capture the semantic correctness and functionality of generated code in real development scenarios.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While external validity mentions the WeChat-specific limitation, there is no explicit listing of untested settings (e.g., no mention that only C++ was tested, that only function-level completion was evaluated, that proprietary code patterns may not apply to other industries, etc.)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The benchmark (100 examples), retrieval corpus (1,669 projects), and developer survey responses are all proprietary and not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark construction is described in detail (Section II-A): three senior developers with 5+ years experience, four annotation rules (function significance, context selection, difficulty classification, quality assurance), cross-validation, three weeks of effort, 100 examples across 7 domains. Retrieval corpus: 1,669 internal projects spanning multiple business units."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark annotators are described as 'three senior developers from our group, each with over five years of industrial experience.' Developer survey participants are 'three developers from our group (excluding the authors).' No description of how these specific individuals were selected or whether selection could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: Algorithm 1 details preprocessing from raw source files to retrieval corpus. Section II-A describes benchmark construction with four rules and cross-validation. Section II-B describes retrieval corpus construction with filtering of duplicates and format standardization."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in a footnote: 'This research is supported by National Key R&D Program of China (No. 2022YFB3103900), National Natural Science Foundation of China (No. 62472126), Natural Science Foundation of Guangdong Province, Shenzhen-Hong Kong Jointly Funded Project, and Shenzhen Basic Research.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: three authors from Tencent (Guangzhou, China) and two from The Chinese University of Hong Kong. Tencent is the parent company of WeChat, the codebase being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government research programs (NSFC, National Key R&D Program, provincial and municipal science foundations) that do not have a financial stake in whether RAG improves code completion at Tencent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Three authors are Tencent employees evaluating methods on Tencent's codebase, but this potential commercial interest is not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 26 LLMs evaluated. While the benchmark is proprietary (reducing contamination risk), the paper does not state when any model's training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test overlap. The proprietary nature of the WeChat codebase inherently mitigates contamination risk, but this is never explicitly discussed as a contamination consideration."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of benchmark contamination. While the custom proprietary benchmark is unlikely to be in training data, the paper does not leverage this as a methodological advantage or discuss contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the benchmark annotation or the developer survey."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the developer survey or the benchmark annotation process involving human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark annotators are described as having 'over five years of industrial experience,' but no other demographics are reported. Developer survey participants are described only as 'three developers from our group' with no experience level, role, or other demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for selecting the benchmark annotators or developer survey participants beyond being 'from our group.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The developer survey is a within-subjects evaluation where all participants rate all conditions, not an experimental study with treatment/control assignment requiring randomization."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether developer survey participants were blinded to which retrieval technique produced which code completion. The evaluation design description does not address blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported. The paper does not state whether all 3 developer survey participants completed all evaluations."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for any of the 26 models or retrieval techniques, despite evaluating models ranging from 0.5B to 671B parameters on industrial-scale codebases."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware configurations are described (8× A100 40GB, 8× H20 96GB, 16× H20 96GB) but total GPU hours, wall-clock time, or total computational budget for the experiments is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic output, producing single-run results. No seed sensitivity analysis is conducted, and the paper does not discuss whether non-determinism from other sources (GPU floating-point, batching) could affect results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Temperature=0 implies single deterministic runs, but this is not explicitly confirmed as single-run evaluation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The number of retrieved results (k=4) is chosen to fit prompt length constraints, and BM25 parameters k and b are referenced but their values and how they were selected are not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of k=4 retrieved results is justified only by prompt length constraints ('less than 2k tokens'), not by systematic evaluation. No validation set is used to select hyperparameters."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes hundreds of comparisons across 26 models × 10 RAG conditions (Tables I–III) without any statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement the RAG pipeline and compare it against base model performance without acknowledging potential author-implementation bias. The retrieval techniques are standard, but the preprocessing algorithm and prompt templates are the authors' own design."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While results are grouped by model scale (0.5B+ to 671B+), the actual compute cost of each RAG method (retrieval + inference) is not compared. The added cost of retrieval over base model inference is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C explicitly discusses construct validity: 'We use automated metrics (CodeBLEU and Edit Similarity) to measure code quality... these metrics might not fully capture the semantic correctness and functionality of generated code in real development scenarios.' The developer survey is conducted to partially address this gap."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated under identical RAG configurations within each comparison (same retrieval technique, same number of results, same prompt template per method). Cross-model comparisons in each column of Tables I–III use matched conditions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The paper does not discuss whether any model's training data could include code patterns similar to the proprietary benchmark, or the temporal relationship between model training and benchmark creation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The paper does not consider whether the complete code snippets used as queries in Table II's 'Complete' condition leak answer information that would not be available in real code completion scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the 100 benchmark examples. Examples from the same domain (e.g., 29 client_call examples) may share patterns, but potential non-independence is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. While the proprietary nature of the code inherently reduces contamination risk, no formal detection (canary strings, membership inference, n-gram overlap) is employed."
       }
     }
   },
   "claims": [
     {
       "claim": "Both identifier-based and similarity-based RAG methods consistently improve code completion performance across different models and scales in closed-source repositories, with similarity-based RAG performing substantially better.",
       "evidence": "Table I shows improvements across 26 LLMs. For example, DeepSeek-V3 achieves 42.24/61.75 CB/ES with identifier-based RAG (func-def) and 60.28/73.11 with similarity-based RAG (GTE-Qwen), representing 42.7% and 18.4% improvement of similarity over identifier-based (Section IV-A).",
       "supported": "strong"
     },
     {
       "claim": "BM25 (lexical) and GTE-Qwen (semantic) achieve superior performance among retrieval techniques for similarity-based RAG.",
       "evidence": "Table I shows BM25 achieving 55.67/69.18 and GTE-Qwen reaching 55.29/68.21 for DeepSeek-V2.5, outperforming CodeBERT, UniXcoder, and CoCoSoDa. Table II confirms this pattern across model scales (Section IV-A, IV-B).",
       "supported": "strong"
     },
     {
       "claim": "GTE-Qwen demonstrates superior performance with incomplete code contexts as queries, unlike other retrieval techniques that perform better with complete code snippets.",
       "evidence": "Table II shows that most retrieval techniques improve with complete queries, but GTE-Qwen achieves better results with incomplete queries for larger models (e.g., DeepSeek-V3: 60.28/73.11 incomplete vs. 58.85/71.02 complete). Finding 2 in Section IV-B.",
       "supported": "moderate"
     },
     {
       "claim": "Combining BM25 and GTE-Qwen yields optimal performance in similarity-based RAG for most LLMs 7B+.",
       "evidence": "Table III shows BM25+GTE-Qwen achieving 63.62/75.26 CB/ES for DeepSeek-V3 and 63.73/72.25 for Qwen2.5-32B. However, the combination shows 'limited or even negative impact' for models below 7B (Section IV-C).",
       "supported": "strong"
     },
     {
       "claim": "Lexical and semantic retrieval techniques have minimal overlap in retrieved candidates, suggesting they capture fundamentally different aspects of code similarity.",
       "evidence": "Out of 100 test examples, there are 76, 74, and 64 completely distinct retrieved samples when comparing BM25 with UniXcoder, CoCoSoDa, and GTE-Qwen respectively (Section IV-C).",
       "supported": "moderate"
     },
     {
       "claim": "The developer survey confirms that the BM25+GTE-Qwen combination consistently achieves higher scores than either technique alone.",
       "evidence": "Figure 2a shows average scores across 3 LLMs, and Figure 2b shows win-rate analysis where the combined technique outperforms single techniques in about half of test cases. However, the survey involves only 3 developers on 52 examples (Section V-A).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny benchmark size",
       "detail": "The evaluation benchmark contains only 100 examples for evaluating 26 models across 10+ RAG conditions. With such a small sample, individual examples can disproportionately affect results, yet no statistical significance testing is performed on any comparison."
     },
     {
       "flag": "Company evaluating on its own codebase",
       "detail": "Three of six authors are Tencent employees evaluating RAG methods on Tencent's WeChat codebase. The benchmark was annotated by 'senior developers from our group' and the developer survey involved 'three developers from our group.' This creates multiple points where organizational bias could influence results."
     },
     {
       "flag": "No statistical tests on any comparison",
       "detail": "Across Tables I–III, hundreds of comparisons are made (26 models × 10 RAG conditions) with claims of 'superiority' and 'consistent improvement,' but no statistical significance tests are conducted. Small differences in CodeBLEU/ES on 100 examples may not be meaningful."
     },
     {
       "flag": "Developer survey with 3 participants",
       "detail": "The developer survey (Section V-A) uses only 3 developers evaluating 52 examples. This is far too small for reliable conclusions, yet the results are presented as validation of the automated findings."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single deterministic runs (temperature=0). While this ensures reproducibility, there is no analysis of sensitivity to other sources of variation (prompt wording, retrieval parameters, benchmark selection), making it impossible to assess result stability."
     },
     {
       "flag": "Proprietary data prevents any independent reproduction",
       "detail": "Neither the benchmark (100 examples), retrieval corpus (1,669 repositories), nor any code is released. The prompts are in Chinese and not provided. No external researcher can verify or reproduce any result in this paper."
     }
   ],
   "cited_papers": [
     {
       "title": "GraphCoder: Enhancing Repository-Level Code Completion via Code Context Graph-based Retrieval and Language Model",
       "authors": ["W. Liu", "A. Yu", "D. Zan", "B. Shen", "W. Zhang", "H. Zhao", "Z. Jin", "Q. Wang"],
       "year": 2024,
       "arxiv_id": "2406.07003",
       "relevance": "Proposes graph-based RAG for repository-level code completion, directly relevant to code generation evaluation methodology."
     },
     {
       "title": "Language Models for Code Completion: A Practical Evaluation",
       "authors": ["M. Izadi", "J. Katzy", "T. van Dam", "M. Otten", "R. M. Popescu", "A. van Deursen"],
       "year": 2024,
       "relevance": "Practical evaluation of LLMs for code completion at ICSE 2024, directly relevant to LLM code generation capability assessment."
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "authors": ["T. Liu", "C. Xu", "J. J. McAuley"],
       "year": 2024,
       "relevance": "Benchmark for repository-level code completion, relevant to evaluation methodology for code generation systems."
     },
     {
       "title": "How Practitioners Expect Code Completion?",
       "authors": ["C. Wang", "J. Hu", "C. Gao", "Y. Jin", "T. Xie", "H. Huang", "Z. Lei", "Y. Deng"],
       "year": 2023,
       "relevance": "Studies developer expectations for code completion tools, relevant to understanding practical AI coding tool impact."
     },
     {
       "title": "REPOFUSE: Repository-Level Code Completion with Fused Dual Context",
       "authors": ["M. Liang", "X. Xie", "G. Zhang", "X. Zheng", "P. Di", "W. Jiang", "H. Chen", "C. Wang", "G. Fan"],
       "year": 2024,
       "arxiv_id": "2402.14323",
       "relevance": "Repository-level code completion using dependency and similar code retrieval, directly comparable RAG approach for code generation."
     },
     {
       "title": "Repoformer: Selective Retrieval for Repository-Level Code Completion",
       "authors": ["D. Wu", "W. U. Ahmad", "D. Zhang", "M. K. Ramanathan", "X. Ma"],
       "year": 2024,
       "relevance": "Selective retrieval approach for repository-level code completion, relevant to RAG for code generation evaluation."
     },
     {
       "title": "Dataflow-Guided Retrieval Augmentation for Repository-Level Code Completion",
       "authors": ["W. Cheng", "Y. Wu", "W. Hu"],
       "year": 2024,
       "relevance": "Dataflow-guided RAG for code completion presented at ACL 2024, directly relevant to code generation methodology."
     },
     {
       "title": "ReACC: A Retrieval-Augmented Code Completion Framework",
       "authors": ["S. Lu", "N. Duan", "H. Han", "D. Guo", "S. Hwang", "A. Svyatkovskiy"],
       "year": 2022,
       "relevance": "Early retrieval-augmented code completion framework at ACL 2022, foundational work for RAG in code generation."
     },
     {
       "title": "Studying LLM Performance on Closed-and Open-source Data",
       "authors": ["T. Ahmed", "C. Bird", "P. Devanbu", "S. Chakraborty"],
       "year": 2024,
       "arxiv_id": "2402.15100",
       "relevance": "Studies the open-source vs closed-source performance gap for LLMs, directly relevant to distribution shift in code generation evaluation."
     },
     {
       "title": "FT2Ra: A Fine-Tuning-Inspired Approach to Retrieval-Augmented Code Completion",
       "authors": ["Q. Guo", "X. Li", "X. Xie", "S. Liu", "Z. Tang", "R. Feng", "J. Wang", "J. Ge", "L. Bu"],
       "year": 2024,
       "relevance": "Retrieval-augmented code completion approach at ISSTA 2024, relevant to code generation methods evaluation."
     },
     {
       "title": "CodeBLEU: a Method for Automatic Evaluation of Code Synthesis",
       "authors": ["S. Ren", "D. Guo", "S. Lu", "L. Zhou", "S. Liu", "D. Tang", "N. Sundaresan", "M. Zhou", "A. Blanco", "S. Ma"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Defines the CodeBLEU metric used widely for evaluating code generation quality, foundational evaluation methodology."
     },
     {
       "title": "STALL+: Boosting LLM-based Repository-level Code Completion with Static Analysis",
       "authors": ["J. Liu", "Y. Chen", "M. Liu", "X. Peng", "Y. Lou"],
       "year": 2024,
       "arxiv_id": "2406.10018",
       "relevance": "Uses static analysis to improve LLM-based code completion, relevant to agentic code generation approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly applicable to practitioners building RAG-based code completion systems for proprietary codebases, with specific guidance on retrieval technique selection."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results largely confirm expected benefits of RAG for code completion; the finding that BM25+semantic combination works best is mildly interesting but not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward empirical evaluation."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, data, or demo is released; everything is proprietary to WeChat/Tencent."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "WeChat/Tencent is widely recognized, though not as closely associated with AI research as OpenAI or Google DeepMind."
     }
   }
 }