ai-research-survey

scan.json (24165B)

---

 {
   "paper": {
     "title": "Retrieval-Augmented Code Generation: A Survey with Focus on Repository-Level Approaches",
     "authors": [
       "Yicheng Tao",
       "Yao Qin",
       "Yepang Liu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.04905",
     "doi": "10.48550/arXiv.2510.04905"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials are mentioned. The survey could have released its paper corpus, analysis scripts, or taxonomy data but did not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The list of 110 surveyed papers and their annotations/categorizations are not released as a downloadable dataset. No supplementary data files are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. As a survey, this could have included tools used for analysis or bibliometric processing, but none are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The exact search keywords are not listed ('we derived and refined a set of search terms' without specifying them), making the search unreproducible."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments. Statistical uncertainty measures are not applicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments or statistical comparisons."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments. No effect sizes are applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments requiring sample size justification."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper positions itself against prior surveys [22-28] in Section 1, noting that 'existing surveys mainly focus on snippet-level generation or general software engineering tasks, with limited discussion of retrieval-based or repository-level methods.' This comparison justifies the survey's contribution."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The prior surveys cited for comparison (references 22-28) include recent works from 2024-2025, representing the current state of survey literature in this area."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed breakdowns by multiple dimensions: retrieval strategy (graph vs non-graph, Section 4.1), training paradigm (Section 4.2), agent architecture level (Level 0/1/2, Section 4.3), downstream tasks (Section 4.4), programming language (Section 4.5, Figure 9), backbone models (Section 4.6), and venue distribution (Figure 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Limitations of Existing Approaches' discusses where current RACG approaches fail: context window constraints, graph-based RAG complexity and noise, insufficient dataset scale, and limited deployment readiness."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.4 discusses cases where RAG does not help: 'when repositories are relatively small and well-structured, LC models can match or even outperform RAG' (Peng et al.). The paper also notes 'RAG is not universally superior.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'comprehensive review,' taxonomy by 'generation strategies, retrieval modalities, model architectures, training paradigms, and evaluation protocols,' and a 'unified analytical framework.' All of these are delivered in the paper body through Sections 4-5 and the taxonomy figures."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey makes no original causal claims. Interpretive statements about trends (e.g., 'Python dominates... likely due to its popularity') are appropriately hedged with 'likely' or 'may be due to,' and causal claims about RAG effectiveness are attributed to cited work."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 bounds the search to January 2023 – August 2025, specific venues, and specific platforms. Inclusion/exclusion criteria (Section 3.3) further bound the scope. The title itself scopes to 'Repository-Level Approaches.'"
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy paper that presents no original empirical results requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no measurements or proxies of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used by the survey itself."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used in this survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the data collection pipeline: manual venue review → keyword-based automated search across ACM DL, IEEE Xplore, arXiv, OpenReview, ACL Anthology → 579 candidates → screening with inclusion/exclusion criteria (Section 3.3) → quality assessment (Section 3.4) → backward snowballing (Section 3.5) → 110 papers. Criteria at each stage are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 discusses limitations of the RACG approaches surveyed (context windows, graph complexity, etc.), but there is no discussion of the survey's own methodological limitations — e.g., search coverage gaps, keyword selection bias, or limitations of the manual annotation process."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity of the survey itself are discussed. There is no mention of potential biases in paper selection, limitations of the search strategy, or risks of the manual categorization process."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The scope is bounded by time period (Jan 2023 – Aug 2025), venues (listed in Section 3.2), platforms (five bibliographic databases), and explicit exclusion criteria (Section 3.3: papers without LMs, papers using RAG only as minor component, vulnerability/clone detection, etc.)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The list of 110 papers, their categorizations, and annotations are not available for download. No supplementary materials are provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the two-stage data collection: manual review of leading venues followed by automated keyword searches across five platforms, with the search period (Jan 2023 – Aug 2025) and platforms explicitly stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a literature search, not a standard benchmark, but the search methodology is documented in Section 3."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (manual review → keyword search → 579 candidates → screening → quality assessment → snowballing → 110 papers), but intermediate counts are missing. How many were excluded at each stage (screening, quality assessment, snowballing additions) is not stated — only the starting (579) and ending (110) counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University, Chinese University of Hong Kong, and Southern University of Science and Technology."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 follows a structured review approach 'inspired by established guidelines in software engineering literature [30, 27, 22]' with defined research questions (Section 3.1), systematic data collection (Section 3.2), inclusion/exclusion criteria (Section 3.3), quality assessment (Section 3.4), and snowballing (Section 3.5). However, no PRISMA flow diagram is provided and exact search keywords are not listed."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 describes quality assessment as a screening criterion (relevance, methodological clarity, retrieval-generation integration, reproducibility), but once papers pass screening, all 110 are treated equally. There is no quality scoring rubric, risk-of-bias assessment, or structured evaluation of included studies' methodological rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Publication bias is not discussed. The paper notes arXiv dominates (37.3%) but does not consider whether included papers skew toward positive results, whether negative findings are underrepresented, or whether the venue distribution introduces systematic bias."
       }
     }
   },
   "claims": [
     {
       "claim": "This is the first comprehensive survey on Retrieval-Augmented Code Generation (RACG) emphasizing repository-level reasoning challenges.",
       "evidence": "The paper positions against prior surveys [22-28] in Section 1, noting existing surveys 'mainly focus on snippet-level generation or general software engineering tasks, with limited discussion of retrieval-based or repository-level methods.'",
       "supported": "moderate"
     },
     {
       "claim": "Python dominates RACG research with 79 occurrences, far ahead of Java (38) and other languages.",
       "evidence": "Figure 9 (Section 4.5) shows the programming language distribution analyzed from explicit mentions in papers.",
       "supported": "strong"
     },
     {
       "claim": "75.2% of RACG systems are non-agent (Level 0), with only 15.8% being fully autonomous agents (Level 2).",
       "evidence": "Figure 7 shows the distribution pie chart; Section 4.3 provides the three-tier classification framework with categorization of individual systems.",
       "supported": "strong"
     },
     {
       "claim": "RAG effectiveness is context-dependent: RAG excels in large/complex repositories but long-context LLMs can match or surpass RAG in smaller, structured settings.",
       "evidence": "Section 4.1.4 cites Peng et al. [112] showing LC models matching RAG in small repos, and multiple works (Chen et al. [114], Yang et al. [115]) demonstrating RAG advantages in specific contexts.",
       "supported": "moderate"
     },
     {
       "claim": "Small open-source models (<10B) and proprietary models are significantly more popular than large open-source models (>10B) as base generation models in RACG.",
       "evidence": "Figure 10 (Section 4.6) shows the distribution of base generation models adopted in recommended configurations across the surveyed papers.",
       "supported": "strong"
     },
     {
       "claim": "Contain and Invoke are foundational edge types in graph-based RAG, forming part of a de facto standard three-tier abstraction (Module-Class-Function).",
       "evidence": "Table 2 in Section 4.1.2 compares 25 graph-based RAG methods, showing Contain appears in nearly all and Invoke in most, with Module-Class-Function nodes co-occurring frequently.",
       "supported": "strong"
     },
     {
       "claim": "BM25 retrieval achieves competitive or best performance for code matching in multiple studies.",
       "evidence": "Section 4.1.4 cites Chen et al. [114] finding BM25 retrieves best for code matching, Yang et al. [115] finding BM25 balances simplicity and effectiveness, and Wang et al. [123] confirming BM25's balance of effectiveness and efficiency.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey reviews 110 papers on Retrieval-Augmented Code Generation, providing a taxonomy covering non-graph-based and graph-based retrieval strategies, training paradigms, and a three-tier agent architecture classification (Level 0-2). The analysis reveals Python dominance (79 occurrences), the prevalence of non-agent systems (75.2%), and that RAG effectiveness is context-dependent rather than universally superior. Key gaps identified include limited multilingual support, insufficient benchmark diversity, and the underexploration of data/control flow edges in graph-based approaches.",
   "red_flags": [
     {
       "flag": "No artifacts released",
       "detail": "The survey provides no downloadable corpus of the 110 papers, their categorizations, or analysis data. The exact search keywords used for automated search are not disclosed, making the search unreproducible."
     },
     {
       "flag": "No quality assessment of included studies",
       "detail": "All 110 papers are treated equally once they pass an initial screening. There is no quality scoring or risk-of-bias assessment of included studies, meaning findings from rigorous controlled experiments are weighted the same as preliminary arXiv preprints."
     },
     {
       "flag": "No discussion of publication bias",
       "detail": "The survey does not consider whether included papers skew toward positive RAG results, whether negative findings are underrepresented, or whether the 37.3% arXiv dominance introduces systematic quality or reporting bias."
     },
     {
       "flag": "Missing survey self-limitations",
       "detail": "Section 5.1 discusses limitations of the surveyed approaches, but the paper never discusses limitations of the survey itself — potential search coverage gaps, keyword selection bias, categorization subjectivity, or the impact of the 579→110 filtering on representativeness."
     },
     {
       "flag": "Opaque filtering pipeline",
       "detail": "The paper reports going from 579 candidate papers to 110 retained but provides no intermediate counts showing how many were excluded at each stage (inclusion/exclusion screening, quality assessment, snowballing additions)."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM-based automated software engineering at the repository level."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Foundational Level 2 agentic framework for automated code repair with custom Agent-Computer Interface."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang", "Jacky Keung", "Jin Liu", "Daoguang Zan", "Yi Mao", "Jian-Guang Lou", "Weizhu Chen"],
       "year": 2023,
       "relevance": "Foundational RACG work introducing iterative retrieval and the RepoEval benchmark for repository-level code completion."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "General-purpose agentic platform for automated software development, demonstrating scalable collaborative agent systems."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, widely used for evaluating code generation capabilities of LLMs."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Level 2 autonomous agent integrating AST traversal, iterative retrieval, and spectrum-based fault localization for program repair."
     },
     {
       "title": "CodexGraph: Bridging large language models and code repositories via code graph databases",
       "authors": ["Xiangyan Liu", "Bo Lan", "Zhiyuan Hu"],
       "year": 2025,
       "relevance": "Bridges LLMs with code graph databases using dual-agent architecture for repository-level reasoning."
     },
     {
       "title": "Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Analyzes the Agentless approach demonstrating that simple locate-repair-verify pipelines can match complex agents on SWE-bench."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Major open-source code LLM family supporting 92+ programming languages, widely used as backbone in RACG systems."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen", "Sungju Kim", "Sunghun Kim"],
       "year": 2025,
       "relevance": "Prior survey on LLM code generation that this paper positions against as lacking RACG/repository-level focus."
     },
     {
       "title": "CodeRAG-bench: Can retrieval augment code generation?",
       "authors": ["Zora Zhiruo Wang", "Akari Asai", "Xinyan Velocity Yu"],
       "year": 2025,
       "relevance": "Benchmark specifically for evaluating retrieval-augmented code generation, directly relevant to RACG evaluation practices."
     },
     {
       "title": "RLCoder: Reinforcement Learning for Repository-Level Code Completion",
       "authors": ["Yanlin Wang", "Yanli Wang", "Daya Guo"],
       "year": 2025,
       "relevance": "Applies reinforcement learning to train retrievers for repository-level code completion, demonstrating RL's value in RACG."
     }
   ]
 }