ai-research-survey

scan.json (24756B)

---

 {
   "paper": {
     "title": "CAST: Enhancing Code Retrieval-Augmented Generation with Structural Chunking via Abstract Syntax Tree",
     "authors": ["Yilin Zhang", "Xinran Zhao", "Zora Zhiruo Wang", "Chenyang Yang", "Jiayi Wei", "Tongshuang Wu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.15655"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/yilinjz/astchunk (mentioned in the abstract footnote and in Section A.6). The code is stated as open source under the MIT License (Table 9)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: RepoEval, CrossCodeEval, and SWE-bench Lite, all with public links and open licenses listed in Table 9."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.1 mentions hardware (8 Nvidia A100 40G GPUs, 8 Nvidia A6000 40G GPUs, CUDA 12) and vLLM for inference, but there is no requirements.txt, Dockerfile, or detailed library version listing to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the algorithm is provided in pseudocode (Algorithm 1) and the code repository is linked, the paper does not include step-by-step reproduction instructions (e.g., specific commands to run, a README with reproduction steps for the main experiments)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-8 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CAST improves over fixed-size chunking across multiple settings but provides no statistical significance tests. Comparisons are made purely by comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute point improvements with baseline context throughout, e.g., 'boosting Recall@5 by 4.3 points on RepoEval' and 'Pass@1 by 2.67 points on SWE-bench generation' (Abstract, Section 3.2). Tables show both CAST and baseline numbers, allowing readers to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why specific benchmarks or their sizes were chosen. SWE-bench Lite is a 300-problem subset, but no discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across experimental runs is reported. Results appear to be from single runs (for Pass@1 with t=0.2) with no mention of multiple seeds or repetitions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "CAST is consistently compared against fixed-size line-based chunking as the baseline across all experiments (Tables 1-8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is fixed-size line-based chunking, which is the simplest approach. No comparison is made against other structure-aware or semantic chunking methods. The paper mentions CodeCRAG (Du et al., 2025) as related work but does not compare against it."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 presents ablation studies: split-only vs. split-then-merge (Table 2), context length selection (Table 3), and maximum chunk size sensitivity (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: nDCG@5, Precision@5, Recall@5 for retrieval; Pass@1, Pass@8, exact match (EM), and edit similarity (ES) for generation (Section 3.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via metrics on benchmark datasets. Human evaluation of chunk quality or generated code quality could be relevant to validating whether AST-based chunks are indeed more 'semantically coherent' as claimed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets (RepoEval, CrossCodeEval, SWE-bench Lite) with standard splits. No tuning on test data is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides per-language breakdowns (Python, Java, C#, TypeScript) on CrossCodeEval. Results are also broken down by retriever model and generator model across all tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases or show examples where CAST performs worse than the baseline. The only negative result is in the ablation (BGE-large on SWE-bench), but this is not analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows that increasing context length from 4000 to 8000 can hurt performance for CodeSage+StarCoder2 (73.2 → 69.2). Table 7 shows cases where CAST with BGE-large performs worse than fixed-size (13.3 vs 14.6 for Claude). These negative results are present in the data though not deeply discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'boosting Recall@5 by 4.3 points on RepoEval' and 'Pass@1 by 2.67 points on SWE-bench generation' are supported by Table 1 (e.g., CodeSage Recall@5: 83.9 vs 82.1 = 1.8; GIST Recall@5: 75.0 vs 70.7 = 4.3; and SWE-bench Pass@1 with BGE+Claude: 16.3 vs 13.7 = 2.6). The claims are reasonably supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that AST-based chunking improves performance. The ablation studies (Section 4) provide controlled single-variable manipulations: split-only vs split-then-merge, varying chunk sizes, and varying context lengths. These constitute adequate ablation designs for the causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Enhancing Code Retrieval-Augmented Generation' broadly, but results are limited to three specific benchmarks (RepoEval, CrossCodeEval, SWE-bench Lite) with specific retrievers and generators. The abstract and conclusion make broad claims about 'scaling retrieval-enhanced code intelligence' without bounding to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For instance, the score mapping technique (Appendix A.2) could introduce systematic bias, and the distributional differences between AST-based and fixed-size chunks could confound comparisons. Neither is discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'claude-3.7-sonnet' and 'gemini-2.5-pro-0325' (Section 3.1). While Gemini has a snapshot date, Claude 3.7 Sonnet does not have a snapshot date or API version. Open models are specified by name and size (StarCoder2-7B, CodeLlama-7B-Python) but without version hashes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide the actual prompts used to query the generation models. It describes the pipeline at a high level but the specific prompt text or templates used for code generation are not included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports temperature (t=0.2 for Pass@1, t=0.8 for Pass@8), top-p (0.95), number of samples (1 for Pass@1, 8 for Pass@8), max_context_length (4000 for RepoEval/SWE-bench, 10000 for CrossCodeEval), and max_chunk_size (2000)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. CAST is a preprocessing (chunking) step in a standard RAG pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The AST-based chunking algorithm is documented in detail (Section 2, Algorithm 1). The score mapping technique for metric comparability is described in Appendix A.2. The chunk size metric (non-whitespace characters rather than lines) is explained in Section 2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the conclusion, discussing contextual awareness, multi-view of code, and inner execution dynamics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses future directions (multi-level information, multiple code views, execution traces) rather than specific threats to validity of the current study. It does not address whether the improvements could be due to confounds, the fairness of the score mapping technique, or the generalizability concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses future directions but does not explicitly state what the results do NOT show. It does not bound the claims to the specific benchmarks, languages, or models tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmarks are publicly available, the paper does not release the raw experimental outputs (e.g., generated code, retrieval results, per-instance scores) for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data comes from established public benchmarks (RepoEval, CrossCodeEval, SWE-bench Lite), all described with citations in Section 3.1. The selection of SWE-bench Lite (300-problem subset where each issue is solvable by editing a single file) is documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: source code is parsed into AST, chunked via the split-then-merge algorithm (Algorithm 1), indexed, retrieved with various retrievers, and fed to generators. The score mapping technique for metric alignment is also documented (Appendix A.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding: ONR Award N000142312840, OpenAI Research Credit program, Amazon AI Research Gift Fund, and Gemma Academic Program GCP Credit Award."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Carnegie Mellon University (5 authors) and Augment Code (1 author, Jiayi Wei). The affiliation with Augment Code, a code AI company, is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI, Amazon, and Google (Gemma program) are all companies with products in the code generation and RAG space. While the funding appears to be unrestricted grants/credits, these companies have financial interest in advancing code RAG technology. Additionally, one author is affiliated with Augment Code, a code AI company."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included. One author (Jiayi Wei) is affiliated with Augment Code, a code AI company that could benefit from improved code RAG techniques, but no financial interest disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models (StarCoder2-7B, CodeLlama-7B, Claude 3.7 Sonnet, Gemini 2.5 Pro) on benchmarks but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark data (RepoEval, CrossCodeEval, SWE-bench) could have been in the training data of the models used. This is particularly relevant for Claude and Gemini, which are large closed-source models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RepoEval and SWE-bench were published in 2023/2024, before the likely training cutoffs of Claude 3.7 and Gemini 2.5. The paper does not discuss this contamination risk. However, since CAST is compared against fixed-size chunking using the same models, contamination would affect both conditions equally, partially mitigating (but not eliminating) the concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs for Claude 3.7 Sonnet or Gemini 2.5 Pro, nor wall-clock time for chunking or generation. Given that CAST involves AST parsing as an additional step, the computational overhead vs. fixed-size chunking is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the hardware is mentioned (8 A100 GPUs, 8 A6000 GPUs), the total compute budget (GPU hours, API costs, total experiment time) is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CAST boosts Recall@5 by 4.3 points on RepoEval retrieval.",
       "evidence": "Table 1 shows GIST-base retriever achieves Recall@5 of 75.0 with CAST vs 70.7 with fixed-size chunking, a 4.3 point gain.",
       "supported": "strong"
     },
     {
       "claim": "CAST improves Pass@1 by 2.67 points on SWE-bench generation.",
       "evidence": "Table 1 shows BGE+Claude achieves Pass@1 of 16.3 with CAST vs 13.7 with fixed-size chunking (2.6 points) and CodeSage+Claude at 16.7 vs 14.0 (2.7 points).",
       "supported": "moderate"
     },
     {
       "claim": "The merging step in CAST is necessary for good performance.",
       "evidence": "Table 2 shows split-only approach significantly degrades performance across all retrievers. For BGE-base, nDCG drops from 71.1 to 53.5 and Pass@1 from 51.7 to 48.3.",
       "supported": "strong"
     },
     {
       "claim": "CAST provides cross-language consistency with up to 4.3 points gain on CrossCodeEval.",
       "evidence": "Table 1 shows CodeSage retriever achieves Identifier Match EM of 39.9 with CAST vs 36.3 with fixed-size, a 3.6 point gain. Table 8 shows per-language breakdowns with consistent gains, with largest gains on TypeScript.",
       "supported": "moderate"
     },
     {
       "claim": "Higher precision in retrieval tends to convert into better generation performance.",
       "evidence": "Section 3.2 discusses this correlation across Tables 1-8, noting recall and nDCG correlate weakly with downstream quality.",
       "supported": "weak"
     },
     {
       "claim": "StarCoder2-7B sees an average of 5.5 points gain on RepoEval.",
       "evidence": "Table 6 shows Pass@1 improvements across retrievers for StarCoder2: BGE-base (51.7 vs 47.5 = 4.2), BGE-large (48.8 vs 45.8 = 3.0), GIST-base (57.9 vs 51.2 = 6.7), GIST-large (61.7 vs 59.2 = 2.5), CodeSage (73.2 vs 67.6 = 5.6), Jina (80.7 vs 75.1 = 5.6). Average is approximately 4.6 points, not 5.5.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CAST proposes AST-based chunking for code RAG pipelines, using a recursive split-then-merge algorithm that respects syntactic boundaries. The method consistently improves retrieval metrics (1-4 points in Precision and Recall) and generation metrics (2-6 points in Pass@1) across three benchmarks (RepoEval, CrossCodeEval, SWE-bench Lite) compared to fixed-size line-based chunking. Ablation studies show the merging step is critical, and performance is sensitive to chunk size and context length parameters. The approach is language-agnostic, showing gains across Python, Java, C#, and TypeScript.",
   "red_flags": [
     {
       "flag": "Single weak baseline",
       "detail": "The only baseline is naive fixed-size line-based chunking. No comparison against other structure-aware chunking methods (e.g., CodeCRAG which is mentioned in related work) or semantic chunking approaches. This makes it impossible to assess whether CAST specifically is good or whether any structure-aware approach would yield similar gains."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, error bars, or variance across runs. For Pass@1 with temperature 0.2, a single sample is used, making results sensitive to randomness. No significance tests are performed despite comparative claims."
     },
     {
       "flag": "Score mapping technique may introduce bias",
       "detail": "Appendix A.2 describes a score mapping technique to make retrieval scores comparable across different chunk distributions. This non-standard technique could systematically favor one method, but its impact is not analyzed or validated."
     },
     {
       "flag": "Claimed average gain does not match data",
       "detail": "Section 1 claims 'StarCoder2-7B sees an average of 5.5 points gain on RepoEval' but computing from Table 6 yields approximately 4.6 points average gain across retrievers."
     },
     {
       "flag": "Potential funder conflict not acknowledged",
       "detail": "One author is affiliated with Augment Code (a code AI company), and funding comes from OpenAI, Amazon, and Google — all with commercial interests in code generation technology. No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Key benchmark for evaluating LLMs on real-world software engineering tasks, used as one of three main evaluation datasets."
     },
     {
       "title": "CodeRAG-Bench: Can retrieval augment code generation?",
       "authors": ["Zora Zhiruo Wang", "Akari Asai", "Xinyan Yu", "Frank F. Xu", "Yiqing Xie", "Graham Neubig", "Daniel Fried"],
       "year": 2024,
       "arxiv_id": "2406.14497",
       "relevance": "Benchmark framework for evaluating retrieval-augmented code generation, directly relevant to this survey's scope on code generation methodology."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Proposes a non-agentic approach to LLM-based software engineering, directly relevant to comparing agentic vs non-agentic approaches."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and the Pass@k metric for evaluating code generation, foundational to the evaluation methodology used in this and many other papers."
     },
     {
       "title": "Starcoder 2 and the stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code LLM used as a primary generation model in the experiments, relevant to understanding code LLM capabilities."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as a generation model in the experiments, relevant to the survey's coverage of code generation models."
     },
     {
       "title": "An empirical study on llm-based agents for automated bug fixing",
       "authors": ["Xiangxin Meng", "Zexiong Ma", "Pengfei Gao", "Chao Peng"],
       "year": 2024,
       "arxiv_id": "2411.10213",
       "relevance": "Empirical study on LLM-based agents for software engineering tasks, directly relevant to the survey scope."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that established the paradigm used in code generation, relevant to understanding the RAG methodology evaluated in surveyed papers."
     },
     {
       "title": "CodeGRAG: Bridging the gap between natural language and programming language via graphical retrieval augmented generation",
       "authors": ["Kounianhua Du", "Jizheng Chen", "Renting Rui"],
       "year": 2025,
       "arxiv_id": "2405.02355",
       "relevance": "Uses graph structure of code for RAG, directly relevant alternative approach to structure-aware code retrieval."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang", "Jacky Keung"],
       "year": 2023,
       "relevance": "Repository-level code completion using retrieval, relevant to evaluating RAG-based code generation approaches."
     }
   ]
 }