ai-research-survey

scan.json (26723B)

---

 {
   "paper": {
     "title": "CODERAG-BENCH: Can Retrieval Augment Code Generation?",
     "authors": [
       "Zora Zhiruo Wang",
       "Akari Asai",
       "Xinyan Velocity Yu",
       "Frank F. Xu",
       "Yiqing Xie",
       "Graham Neubig",
       "Daniel Fried"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2406.14497"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a project website (https://code-rag-bench.github.io/) and mentions releasing a codebase for reproducible evaluations. The abstract and introduction explicitly state the benchmark provides a 'consistent interface for retrieval, augmented generation, and evaluation.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark aggregates publicly available datasets (HumanEval, MBPP, LiveCodeBench, DS-1000, ODEX, RepoEval, SWE-bench-Lite, CodeSearchNet) and creates a diverse retrieval datastore from five public sources. The project website is provided for access."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using specific tools (pyserini, sentence-transformers) and hardware (A100 GPU with 80GB), but does not provide a requirements.txt, Dockerfile, or detailed environment specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix C discusses result reproduction differences and the paper describes experimental setups, the paper itself does not include step-by-step reproduction instructions or commands. Reproduction details may exist on the project website, but the paper text does not contain them."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (pass@1, NDCG@10) without confidence intervals, error bars, or any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'dense embedding models frequently surpass BM25', 'GPT-4o achieves a 27.4% gain') but reports no statistical significance tests to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context, e.g., 'GPT-4o achieves a 27.4% gain on SWE-Bench and a 6.9% gain on the harder ODEX subset' and 'Jina-v2-code outperforms GIST-base and BGE-base by 7.4 and 6.6 average NDCG@10.' Baseline values and improvements are consistently provided together."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of examples in each dataset or why these particular dataset sizes are sufficient. For instance, LiveCodeBench uses 400 examples and ODEX-hard uses 29 examples from the 20 least common libraries, with no power analysis or size justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper generates only one response per example (temperature=0.2, sample one response) and reports single-run numbers. No standard deviation, variance, or multiple-run statistics are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes extensive baselines: 10 retrieval models (BM25, various dense and proprietary retrievers) and 10 generation models, plus no-retrieval and gold-document baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models as of 2024: GPT-4o, DeepSeekCoder, StarCoder2, Llama3, CodeGemma, Claude-3 variants, Gemini-1.5 variants, and top-ranked MTEB retrievers like SFR-Embedding-Mistral."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple ablation-style analyses: comparing different retrieval sources individually (Section 4, Tables 7-8), varying document counts (Appendix E, Figure 6), comparing chunking strategies (Table 9), and constructing the ODEX-hard subset to isolate library familiarity effects (Section 3.3, Appendix D)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: NDCG@10, Precision, and Recall for retrieval (Section 2.4), and pass@k for code generation. Different tasks use appropriate metrics (ndcg@10 for code retrieval, execution-based pass@1 for generation)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper relies entirely on automated metrics (execution-based pass@1 and NDCG@10). While manual analysis is mentioned for specific failure modes (Section 4, Appendix F), there is no systematic human evaluation of the generated code quality or relevance of retrieved documents beyond the canonical document annotation process."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets (HumanEval, MBPP, DS-1000, ODEX, RepoEval, SWE-bench-Lite). LiveCodeBench is specifically included to address contamination concerns with problems collected after training cutoffs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task type (basic programming, open-domain, repository-level, code retrieval) in Tables 3, 5, and 6, and further by individual dataset within each category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendix F discuss failure modes: models copying functions from context, generating over-complicated programs, Claude models responding with explanations instead of code. Figure 5 illustrates distracting context examples. Appendix A.2 provides concrete examples of both helpful and distracting retrieved documents."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: RACG does not improve DeepSeekCoder generations due to overcomplicated outputs (Section 3.4), GPTs show no gains on open-domain tasks with common libraries (Section 3.3), reranking within optimal range 'greatly degrades the results' (Section 4), and open retrieval sources are less useful than local code for repository-level tasks (Table 8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (1) 'retrieving high-quality contexts improves code generation' is supported by Table 5 gold vs. w/o results; (2) 'retrievers often struggle to fetch useful contexts' is supported by Table 3 low NDCG scores on open-domain and SWE-bench; (3) 'generators face limitations in using those contexts effectively' is supported by negative RACG results in Tables 6 and 11-13."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily ablation-style: adding/removing retrieval context and measuring the effect. The controlled experimental design (same model with/without documents, gold vs. retrieved) adequately supports claims like 'retrieval significantly enhances performance.' The ODEX-hard experiment (Appendix D) provides controlled evidence for the library familiarity hypothesis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds scope: 'we focus on coding tasks using Python programming language, but extrapolating to other languages may bring additional challenges.' The title asks 'Can Retrieval Augment Code Generation?' rather than asserting universal effectiveness. Footnote 1 also states the Python-only scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for key results: the library memorization hypothesis for why GPT models don't benefit from open-domain retrieval (Section 3.3, verified in Appendix D), the over-complication hypothesis for DeepSeekCoder's degradation with contexts (Section 3.4), and the context-copying problem for ODEX failures (Appendix F)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'gpt-3.5-turbo-0125' and 'gpt-4o' but 'gpt-4o' lacks a specific version/snapshot date. Other models are specified by name but not version: 'Claude-3-haiku', 'Claude-3-sonnet', 'Gemini-1.5-flash', 'Gemini-1.5-pro', 'Llama3-8B', 'Command-R'. Marketing names without snapshot dates do not count as specified versions per the schema."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C states 'we use the same prompt for each dataset' and 'zero-shot prompts without any additional instructions,' but the actual prompt text is not provided in the paper or appendix. The paper describes what prompts do in natural language rather than providing the actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature t=0.2, top_p=0.95 for generation (Section 3.1); BM25 parameters k1=1.2, b=0.75; batch sizes for encoding (64 for most models, 8 for SFR-Mistral); top-5 documents prepended; n=21 sampling for SWE-bench."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It evaluates a straightforward retrieve-then-generate pipeline without agent loops, tool use, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data curation process in detail: how retrieval sources were collected (from ClueWeb22, RedPajama-1T), how canonical documents were annotated (automatic parsing + manual verification), and how each dataset was integrated. Table 1 shows dataset sizes and Table 2 shows corpus statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The paper has a dedicated 'Limitations' section after the Conclusion (starting at the text marker 'Limitations') discussing task and language diversity, model coverage, and methodological scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific concerns: Python-only focus may not generalize to other languages, vanilla retrieval/generation methods used rather than advanced techniques, and results 'may not represent all model behaviors.' Appendix C specifically discusses reproduction variances for individual models (3-5 points lower for CodeGemma, few-shot vs zero-shot differences)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: Python-only (footnote 1), focus on basic retrieval methods without advanced RACG techniques (Limitations section), exclusion of tasks like code debugging. The title frames the work as a question rather than a definitive answer."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is built on publicly available datasets (HumanEval, MBPP, LiveCodeBench, DS-1000, ODEX, RepoEval, SWE-bench-Lite, CodeSearchNet) and the retrieval corpora are from public sources (ClueWeb22, RedPajama-1T, devdocs.io). The project website provides access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes data collection in detail: programming problems sourced from existing datasets (Section 2.1), retrieval documents from five specific sources with sizes and average lengths (Section 2.2, Table 2), and canonical document annotation with automatic + manual verification (Section 2.3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper uses standard public benchmarks and automated evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented across Section 2: dataset selection and integration (2.1), retrieval source collection with specific origins for each source (2.2), canonical document annotation process with automatic parsing followed by manual verification (2.3), and the evaluation pipeline (2.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgment section thanks individuals for discussions and feedback but does not mention any grants, funding agencies, or corporate sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University, University of Washington, and University of Southern California. The authors are from academic institutions, not from companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure does not confirm the work is unfunded — the authors are at major research universities that typically have funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the models evaluated. It acknowledges contamination concerns ('it is unclear whether models suffer from data contamination on HumanEval and MBPP') and includes LiveCodeBench with 'problems collected from coding websites after the training cutoff of LMs that we consider,' but does not specify what those cutoff dates are."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly discusses contamination risk: 'due to limited public knowledge about model training data, it is unclear whether models suffer from data contamination on HumanEval and MBPP.' LiveCodeBench is specifically included 'to decrease the risk of contamination' with problems after training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper proactively addresses benchmark contamination by including LiveCodeBench alongside potentially contaminated benchmarks (HumanEval published 2021, MBPP published 2021). Section 2.1 explicitly acknowledges the contamination risk and includes a contamination-free alternative."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study. All evaluation is automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses public benchmarks and automated evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or subjective evaluation requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports encoding latency, search latency, model storage, and index storage for retrieval models. Section 3.2 discusses the efficiency-effectiveness tradeoff: SFR-Mistral requires '5x larger index storage and adds nearly 100x latency to encode documents' compared to GIST-base."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions using an A100 GPU (Appendix B), total computational budget (GPU hours, total API spend for proprietary model calls across all experiments) is not stated. The paper runs extensive experiments across 10 retrievers and 10 generators on 8 datasets, but the total cost is never quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Retrieving high-quality (canonical) documents significantly improves code generation across tasks, with GPT-4o achieving a 27.4% gain on SWE-Bench and 6.9% gain on ODEX-hard.",
       "evidence": "Table 5 shows pass@1 improvements from w/o to gold across models and datasets. GPT-4o SWE-bench: 2.3 to 30.7 (Section 3.3). ODEX-hard: 20.7 to 27.6 (Section 3.3, Table 5).",
       "supported": "strong"
     },
     {
       "claim": "Dense embedding models frequently surpass BM25 on code retrieval tasks, contrary to findings in text-centric domains.",
       "evidence": "Table 3 shows retrieval NDCG@10 where models like SFR-Mistral (67.0 avg) and Jina-v2-code (65.4 avg) outperform BM25 (57.7 avg) on average. Discussed in Section 3.2.",
       "supported": "strong"
     },
     {
       "claim": "Code-specific retrieval models outperform general-purpose models at similar parameter scales.",
       "evidence": "Section 3.2: 'Jina-v2-code outperforms GIST-base and BGE-base by 7.4 and 6.6 average NDCG@10, respectively, while Voyage-code significantly outperforms OpenAI-03.' Supported by Table 3 numbers.",
       "supported": "moderate"
     },
     {
       "claim": "RACG is particularly effective for less common libraries where models lack parametric knowledge.",
       "evidence": "The ODEX-hard subset (20 least common libraries) shows 20.3-40.1% improvement with retrieval (Appendix D, Table 10), compared to limited gains on the full ODEX set. Verified across multiple retrieval models.",
       "supported": "strong"
     },
     {
       "claim": "Pre-retrieval chunking achieves the highest scores compared to post-retrieval and reranking-based chunking strategies.",
       "evidence": "Table 9 shows pre-retrieval chunking outperforming full text, first chunk, and reranking on most sources for HumanEval with BM25.",
       "supported": "moderate"
     },
     {
       "claim": "Top-performing retrieval models do not always lead to the best end-to-end RACG outcomes.",
       "evidence": "Section 3.4 states this and Table 6 shows examples where the best retriever varies by task and generation model. For instance, OpenAI reranking performs best on RepoEval but not on basic programming tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Models can be easily distracted by irrelevant retrieved contexts, sometimes degrading performance below the no-retrieval baseline.",
       "evidence": "Table 5 shows red-highlighted cells where gold < w/o. Table 6 shows DeepSeekCoder degradation on some tasks. Appendix F discusses Claude models copying context and generating overcomplicated code.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CODERAG-BENCH demonstrates that retrieval-augmented code generation (RACG) can substantially improve code generation when high-quality documents are retrieved, with gains up to 27.4% on SWE-Bench. However, current retrieval models struggle to find useful documents especially for open-domain and repository-level tasks, and generation models often fail to effectively utilize retrieved contexts, sometimes degrading below no-retrieval baselines. Code-specific retrieval models outperform general-purpose ones at similar scales, and RACG is most effective for problems involving less common libraries where models lack parametric knowledge.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across Tables 3-13 are single point estimates without confidence intervals, error bars, or variance across runs, despite using temperature=0.2 sampling which introduces stochasticity."
     },
     {
       "flag": "Single-sample generation",
       "detail": "The paper generates only one response per example (except SWE-bench with n=21 majority voting), making results sensitive to sampling randomness while reporting no variance."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Numerous comparative claims ('surpass', 'outperform', 'significantly improves') are made based on raw number comparisons without any significance testing."
     },
     {
       "flag": "Potential data leakage in HumanEval/MBPP evaluation",
       "detail": "The paper acknowledges HumanEval and MBPP contamination risk but still reports and draws conclusions from results on these datasets. While LiveCodeBench is included as a mitigation, many comparisons rely on the potentially contaminated benchmarks."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and pass@k metric, foundational for code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for repository-level code generation from GitHub issues, used as a key evaluation task."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Proposed sampling and majority-vote reranking strategy for SWE-bench, adopted in this paper's experimental setup."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code generation benchmark with problems collected after model training cutoffs."
     },
     {
       "title": "Docprompting: Generating code by retrieving the docs",
       "authors": ["Shuyan Zhou", "Uri Alon"],
       "year": 2023,
       "relevance": "Prior work on retrieval-augmented code generation using library documentation."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Agent-based approach to software engineering using LLMs with custom computer interfaces."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major open-source code LLM evaluated as a generation baseline in this benchmark."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "relevance": "Open code LLM series with reproducibility certification, used as baseline."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "relevance": "Prior work on retrieval-augmented repository-level code completion."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Critical evaluation of LLM code generation quality and evaluation methodology."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieve-then-generate paradigm."
     },
     {
       "title": "ARKS: Active retrieval in knowledge soup for code generation",
       "authors": ["Hongjin Su"],
       "year": 2024,
       "arxiv_id": "2402.12317",
       "relevance": "Prior work on retrieval strategies specifically for code generation tasks."
     }
   ]
 }