ai-research-survey

scan.json (26055B)

---

 {
   "paper": {
     "title": "Building A Coding Assistant via the Retrieval-Augmented Language Model",
     "authors": [
       "Xinze Li",
       "Hanbin Wang",
       "Zhenghao Liu",
       "Shi Yu",
       "Shuo Wang",
       "Yukun Yan",
       "Yukai Fu",
       "Yu Gu",
       "Ge Yu"
     ],
     "year": 2024,
     "venue": "ACM Trans. Inf. Syst.",
     "arxiv_id": "2410.16229",
     "doi": "10.1145/3695868"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All codes are available at https://github.com/NEUIR/CONAN' in the preamble section (page 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: CodeSearchNet, Concode, CgCSN, PY150, JavaCorpus, HumanEval, and MBPP. All are standard public benchmarks referenced with citations and available independently."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4 mentions 'PyTorch and Huggingface transformers' and 'OpenMatch' but provides no version numbers, requirements file, Dockerfile, or detailed environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-style reproduction guide or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 5-12 are reported as point estimates only. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'CONAN achieves convincing performance' and 'significantly outperforms previous retrieval augmented code generation models') but reports no statistical significance tests such as p-values, t-tests, or bootstrap tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context throughout, e.g., 'surpasses the state-of-the-art models REDCODER-EXT with an average of approximately 3.1% and 0.6% improvements on CsCSN and Concode datasets' (Sec 5.1), and 'approximately 7% improvements on CgCSN' with full tables showing baseline and proposed method scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why these particular datasets and their sizes are sufficient. Dataset sizes are reported in Table 2 but there is no discussion of whether they are adequate for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance measures, or multi-run results are reported. All results appear to be from single runs with no indication of run-to-run variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included across multiple categories: retrieval models (BM25, CodeBERT, CodeT5, SCODE-R), PLMs (GPT-2, CodeGPT, PLBART, UniXcoder, CodeT5), PLM w. RAG (REDCODER, ReACC), and LLMs (DeepSeek-Coder, CodeQwen), documented in Section 4.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models such as REDCODER (2021), ReACC (2022), UniXcoder (2022), CodeRetriever (2022), DeepSeek-Coder (2024), and CodeQwen1.5 (2024). These represent recent and competitive work in the area."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 presents ablation studies removing RAG, FID, and Dual-View components individually. Table 11 further ablates the pretraining strategies (MEP, CDA) and compares masking strategies (span vs entity mask). Section 5.2 discusses results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: EM, BLEU, CBLEU (CodeBLEU) for code generation; smoothed BLEU-4 for summarization; EM and ES (edit similarity) for code completion; MRR@100 for retrieval; Pass@k for HumanEval/MBPP. Documented in Section 4.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluations are automated metrics. For a system claiming to be a 'code assistant,' human evaluation of output quality or utility would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All datasets have explicit train/dev/test splits shown in Table 2 and Table 4. Results are reported on test sets. The paper also states it excludes target code/documentation from retrieval databases to prevent information leakage (Section 4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (code generation, summarization, completion), by dataset (Concode, CgCSN, CsCSN, PY150, JavaCorpus, HumanEval, MBPP), and by programming language (Python, Java, Ruby, JavaScript, Go, PHP) in Tables 5-10."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses that CONAN does not outperform ReACC on code completion and explains why (different backbone architectures). Section 5.3 and Figure 5 analyze when retrieved documents help vs. hurt (pred==gold vs pred!=gold groups)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CONAN underperforms ReACC on code completion (Table 7), that the dual-view method shows 'less effectiveness in the code summarization task' (Section 5.2), and that MEP alone 'shows almost the same performance as the baseline model' (Section 5.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CONAN 'achieves convincing performance on different code generation tasks and significantly outperforms previous retrieval augmented code generation models,' which is supported by Tables 5-8 showing improvements over REDCODER-EXT and other baselines across most tasks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (Tables 9, 11) use controlled single-variable manipulation to support causal claims about component contributions (e.g., removing RAG, FID, or Dual-View individually). Language like 'the improvements mainly derive from' is supported by these ablations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to build 'A Coding Assistant' generally, and the abstract claims it 'aims to build a code assistant by mimicking the knowledge-seeking behaviors of humans during coding.' However, experiments are limited to Python and Java on specific benchmarks. No testing on other languages like C++, TypeScript, or Rust, and no real-world deployment evaluation. The paper does not bound these generalization limits."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the improvements could stem from the larger effective context window rather than the dual-view representation, or from data leakage between CodeSearchNet-derived training and test sets. No such alternatives are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with size are given: CodeT5-base as backbone, Deepseek-Coder-6.7b-Instruct, CodeQwen1.5-7B-Chat (Section 4.3-4.4). These are specific enough to identify the exact models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "When using LLMs (DeepSeek-Coder, CodeQwen), the paper provides no actual prompt text. It describes the process in equations (Eq. 14: concatenating generated knowledge with query) but does not show the actual prompt templates or system instructions used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 reports learning rates (1e-4, 2e-5, 1e-5, 5e-5), batch sizes (128, 64, 1), epochs (10, 12, 1), warmup settings (0.1 proportion, 1000 steps), optimizers (Adam, AdamW), and for LLM inference: temperature 0.2 and max generation length 512 tokens."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The retrieval-augmented pipeline is described in detail in Section 3: CONAN-R retrieves top-N documents, CONAN-G uses FID architecture to encode them, and the dual-view representation concatenates code documentation with code snippets. Section 3.4 describes how CONAN assists LLMs. The architecture is illustrated in Figure 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: CodeSearchNet filtering for CgCSN/CsCSN (Section 4.1), AST parsing to remove unparseable examples, entity identification using BytesIO and tree_sitter (Section 4.1), retrieval database construction with deduplication and exclusion of target answers (Section 4.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The paper goes directly from evaluation results and case studies (Section 5) to Conclusion (Section 6) with no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper, neither specific nor generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were not tested, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The datasets used (CodeSearchNet, Concode, HumanEval, MBPP, PY150, JavaCorpus) are all publicly available and independently verifiable. The GitHub repository is provided for code. Raw experimental data (individual predictions) are not released, but the input data is fully public."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data sources, construction of retrieval databases, filtering procedures, and dataset statistics (Tables 1-4). Pretraining data construction from CodeSearchNet is described with examples (Figure 3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. All data comes from standard public benchmarks (CodeSearchNet, HumanEval, MBPP, etc.)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: raw CodeSearchNet → filtering → code-documentation pair construction → entity identification → masking (Section 4.1, Figure 3). Retrieval database construction steps with size counts are provided in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding: 'Natural Science Foundation of China under Grant (No. 92267201, No. 62206042 and No. U23B2019), the Joint Funds of Natural Science Foundation of Liaoning Province (No. 2023-MSBA-081), and the Fundamental Research Funds for the Central Universities (No. N2416012).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Northeastern University (China), Tsinghua University, and Chinese Academy of Sciences. The paper evaluates their own proposed system, not a commercial product, so there is no undisclosed corporate conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are Chinese government science foundations (NSFC, Liaoning Province, Central Universities). These are independent research grants with no financial stake in the outcome of comparing code retrieval models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses DeepSeek-Coder and CodeQwen1.5 LLMs for evaluation on HumanEval and MBPP but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper excludes target answers from retrieval databases to prevent information leakage from the RAG pipeline, but does not discuss whether the LLMs (DeepSeek-Coder, CodeQwen) may have seen HumanEval or MBPP test cases during pretraining."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) and MBPP (published 2021) are widely available online. DeepSeek-Coder and CodeQwen were trained after 2021 and may have seen these benchmarks. The paper does not address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All evaluation is automated on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The system involves retrieval + generation + LLM inference but the practical cost of this pipeline is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or total computational budget is stated. Section 4.4 mentions a GPU memory limitation affecting batch size but does not quantify the overall compute."
       }
     }
   },
   "claims": [
     {
       "claim": "CONAN outperforms previous retrieval-augmented code generation models (REDCODER-EXT) with approximately 3.1% average BLEU improvement on CsCSN and 0.6% on Concode datasets.",
       "evidence": "Table 5 shows CONAN achieving BLEU scores of 23.5/26.5 on CsCSN Python/Java summarization (vs REDCODER-EXT 20.9/22.9) and 42.8 BLEU on Concode code generation (vs REDCODER-EXT 42.5). Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "CONAN achieves more significant improvements on CgCSN (approximately 7%) than on Concode (0.6%), demonstrating ability to handle longer code generation.",
       "evidence": "Table 6 shows CONAN achieving 32.9/37.7 BLEU on CgCSN Python/Java vs REDCODER-EXT 24.4/29.0, a substantial improvement. Average code length is 98 for CgCSN vs 27 for Concode (Table 2). Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "CONAN can serve as an effective assistant for LLMs, achieving approximately 10% improvement in EM when used with DeepSeek-Coder and CodeQwen.",
       "evidence": "Table 5 shows DSCoder-6.7b-Ins going from 0 EM to 24.2 EM with CONAN on Concode, and CQwen1.5-7B-Chat going from 0 to 24.2 EM. Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "The improvements of CONAN mainly derive from external retrieved knowledge, with FID architecture providing additional 3% improvement and dual-view representation providing 1.85% improvement on code generation.",
       "evidence": "Table 9 ablation study shows w/o RAG → w/o FID gives ~7.6% improvement, w/o FID → CONAN gives ~3% improvement, and w/o Dual-View → CONAN gives ~1.85% on code generation. Section 5.2.",
       "supported": "strong"
     },
     {
       "claim": "CONAN-R achieves state-of-the-art zero-shot code retrieval, outperforming CodeRetriever by about 2% on CodeSearch and 14% on Adv.",
       "evidence": "Table 10 shows CONAN-R zero-shot achieving 70.9 overall MRR on CodeSearch (vs CodeRetriever 69.1) and 46.1 MRR on Adv (vs CodeRetriever 34.7). The Adv improvement is actually ~11.4 absolute points. Section 5.4.",
       "supported": "moderate"
     },
     {
       "claim": "Code-Documentation Alignment (CDA) is the primary driver of retrieval improvements, while Masked Entity Prediction (MEP) alone shows little improvement.",
       "evidence": "Table 11 shows MEP alone has virtually no zero-shot improvement over vanilla CodeT5 (0.03 vs 0.03), while CDA alone jumps to 45.01. Adding MEP to CDA further improves from 45.01 to 46.08. Section 5.4.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CONAN, a retrieval-augmented code assistant combining a structure-aware retriever (CONAN-R) and a dual-view code representation generator (CONAN-G), outperforms prior RAG-based code generation models on code generation, summarization, and completion benchmarks. The system's improvements primarily derive from incorporating external retrieved code knowledge via the FID architecture, with the code-documentation alignment pretraining being the most impactful component. When used as an assistant for LLMs like DeepSeek-Coder and CodeQwen, CONAN improves their performance by filtering and denoising retrieved code documents into shorter, higher-quality inputs. However, CONAN underperforms specialized decoder-only models (ReACC) on code completion tasks, suggesting the T5-based architecture has limitations for autoregressive completion.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper makes numerous claims about one method outperforming another based solely on comparing point estimates without any significance tests. Given the absence of variance reporting, it is impossible to know whether the reported differences are reliable."
     },
     {
       "flag": "No variance or multi-run reporting",
       "detail": "All results appear to be from single experimental runs. Without multiple runs or seed variation analysis, the reported numbers could be artifacts of random initialization or data ordering."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity discussion. For a system paper proposing a 'code assistant,' this omission is significant since there is no discussion of when the approach fails, what languages/domains it does not cover, or what the practical deployment challenges are."
     },
     {
       "flag": "Benchmark contamination risk with LLMs",
       "detail": "HumanEval and MBPP are widely available online. The paper evaluates DeepSeek-Coder and CodeQwen on these benchmarks without discussing whether these models may have seen the test cases during pretraining, which would inflate the LLM baseline and potentially the CONAN-augmented scores."
     },
     {
       "flag": "Overly broad claims relative to evaluation scope",
       "detail": "The paper titles itself a 'Coding Assistant' but only evaluates on Python and Java with specific benchmark datasets. No real-world deployment, no multi-language generalization beyond 6 languages for retrieval only, and no user studies are conducted."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark used for evaluating code generation capabilities of LLMs."
     },
     {
       "title": "Retrieval Augmented Code Generation and Summarization",
       "authors": ["Md Rizwan Parvez", "Wasi Ahmad", "Saikat Chakraborty", "Baishakhi Ray", "Kai-Wei Chang"],
       "year": 2021,
       "relevance": "Proposes REDCODER framework for retrieval-augmented code generation, a key baseline for RAG-based code systems."
     },
     {
       "title": "ReACC: A Retrieval-Augmented Code Completion Framework",
       "authors": ["Shuai Lu", "Nan Duan", "Hojae Han"],
       "year": 2022,
       "relevance": "Presents a retrieval-augmented code completion framework combining sparse and dense retrieval with autoregressive generation."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven C.H. Hoi"],
       "year": 2021,
       "relevance": "Foundational code-oriented pretrained language model used as the backbone for CONAN, central to code generation research."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Seminal pretrained model for code, widely used as a baseline in code generation and retrieval research."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "relevance": "Recent code LLM used as a baseline for evaluating CONAN's LLM-assistant capabilities."
     },
     {
       "title": "Docprompting: Generating code by retrieving the docs",
       "authors": ["Shuyan Zhou", "Uri Alon", "Frank F Xu", "Zhengbao Jiang", "Graham Neubig"],
       "year": 2022,
       "relevance": "Uses documentation retrieval for code generation with FID architecture, directly related to retrieval-augmented code generation approaches."
     },
     {
       "title": "RepoFusion: Training Code Models to Understand Your Repository",
       "authors": ["Disha Shrivastava", "Denis Kocetkov", "Harm de Vries", "Dzmitry Bahdanau", "Torsten Scholak"],
       "year": 2023,
       "relevance": "Repository-level code understanding using retrieval augmentation, relevant to contextual code generation research."
     },
     {
       "title": "CodeRetriever: A Large Scale Contrastive Pre-Training Method for Code Search",
       "authors": ["Xiaonan Li", "Yeyun Gong"],
       "year": 2022,
       "relevance": "State-of-the-art code retrieval model using contrastive pre-training, key baseline for evaluating code search effectiveness."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark for evaluating program synthesis capabilities of language models."
     },
     {
       "title": "Leveraging Passage Retrieval with Generative Models for Open Domain Question Answering",
       "authors": ["Gautier Izacard", "Edouard Grave"],
       "year": 2021,
       "relevance": "Proposes the Fusion-in-Decoder (FID) architecture used as the generation backbone in CONAN-G."
     },
     {
       "title": "GraphCodeBERT: Pre-training Code Representations with Data Flow",
       "authors": ["Daya Guo", "Shuo Ren", "Shuai Lu"],
       "year": 2021,
       "relevance": "Code representation model incorporating data flow graphs, used as a baseline and as the foundation for CodeRetriever."
     }
   ]
 }