ai-research-survey

scan.json (18680B)

---

 {
   "paper": {
     "title": "Context-Augmented Code Generation Using Programming Knowledge Graphs",
     "authors": ["Iman Saberi", "Fatemeh Fard"],
     "year": 2025,
     "venue": "ICML 2025",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. No mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: PythonAlpaca (Huggingface link provided) and Tutorials dataset from Wang et al. (2024). HumanEval and MBPP are public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a single A100 GPU' and Neo4j version 5.20.0, but no requirements.txt, Dockerfile, or comprehensive dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README or step-by-step reproduction instructions are provided. The methodology is described but not as executable reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '40%' pass@1) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., 'up to 20%') by comparing raw numbers across methods. No statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-4 report absolute pass@1 percentages for all methods and baselines, providing baseline context (e.g., CodeLlama-7B goes from 33% NoRAG to 40% Block-PKG on HumanEval)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has 164 problems and MBPP has ~400. No justification for whether these sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding (temperature=0) is used, so results are deterministic per run. However, no variance across different random seeds or configurations is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: NoRAG, BM25, VoyageEmb, Func-BM25, Func-PKG, Block-PKG, plus an 'Ideal Reranker' upper bound (Tables 1-4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Voyage-Code-2 (2024), BM25, and the paper compares against CodeRAG-Bench (Wang et al., 2024). Models used are recent (DeepSeek-Coder, Llama3.1, StarCoder2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 presents an ablation study on tree pruning (Table 7), and A.2 evaluates the effectiveness of individual re-ranker steps (Tables 8-10)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only pass@1 is used as the evaluation metric. No other metrics (e.g., pass@10, CodeBLEU) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is included. Evaluation is entirely automated via pass@1 test execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and MBPP are standard held-out benchmarks with predefined test cases used for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 provides per-topic breakdowns across 10 categories on MBPP (Figures 4, 5, 10). Error analysis by type is in Table 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ5 discusses error types introduced by RAG (Table 5). Section A.4 discusses retrieval challenges including string manipulation failures. Case studies in A.9."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that DeepSeek-Coder shows less improvement on HumanEval, that RAG can degrade correct solutions, and that PKG underperforms on string manipulation and data structures topics."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'up to 20% pass@1 accuracy gains' and '34% improvement over baselines on MBPP' are supported by Tables 1-2 (e.g., CodeLlama-7B on MBPP: 38% NoRAG to 58% Reranked = 20pp gain)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about PKG improving performance are supported by controlled ablation studies (tree pruning ablation in Table 7, re-ranker step evaluation in Tables 8-10) with single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Generation' broadly, but experiments are limited to Python only on two benchmarks (HumanEval, MBPP) with 7B-13B models. No explicit bounding of generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, improvements could partly come from the re-ranker's test execution filtering rather than PKG retrieval quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by name and size (e.g., 'CodeLlama-7B', 'DeepSeek-Coder-7B', 'Llama3.1-8B') but no specific version identifiers or snapshot dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with actual code are provided in Appendix sections A.6.1, A.7.1, and A.8.1 for CodeLlama, StarCoder2, and DeepSeek-Coder respectively."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0, max_new_tokens=512, and greedy decoding are specified in Section 4 (Evaluation Metric)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The PKG retrieval pipeline, tree pruning, and re-ranking mechanism are described in detail in Section 2 with mathematical formulations and figures."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents preprocessing: 143,000 Q&A pairs → 115,000 Python functions extracted → PKG with 425,058 nodes. Tutorial dataset processing is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Section A.4 discusses retrieval challenges but not broader study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential confounds or methodological limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do not show or what settings are excluded. The paper does not bound its claims to Python or specific model sizes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-problem results, generated code outputs) is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes data sources: PythonAlpaca with 143,000 Q&A pairs and Tutorials dataset with 76,600 entries. Extraction counts are provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets and benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from dataset selection through block extraction, graph extraction, encoding, and Neo4j import is documented in Section 2.1 with counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as being from University of British Columbia Okanagan, with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (CodeLlama, DeepSeek-Coder, StarCoder2, Llama3.1)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or MBPP appeared in the training data of the models used. This is a significant concern given these are widely-used benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published before all models' training cutoffs. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports retrieval time (~3 seconds per query), token budget analysis (Table 11), and Table 6 provides time and storage costs for PKG construction."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports total processing times (301 min for PKG, 241 for VoyageAI, 44 for BM25), storage usage (12,530 MB for PKG), and experiments used a single A100 GPU."
       }
     }
   },
   "claims": [
     {
       "claim": "PKG-based retrieval improves pass@1 accuracy by up to 20% compared to NoRAG on HumanEval and MBPP.",
       "evidence": "Tables 1-2 show improvements across models. E.g., CodeLlama-7B on MBPP: 38% (NoRAG) to 58% (Reranked) = 20pp gain.",
       "supported": "strong"
     },
     {
       "claim": "PKG with re-ranking achieves up to 34% improvement over baselines on MBPP.",
       "evidence": "Table 2 shows StarCoder2-7B: BM25 at 25%, Reranked at 62%, a 37pp difference. CodeLlama-7B: VoyageEmb at 32%, Reranked at 58%, a 26pp difference.",
       "supported": "moderate"
     },
     {
       "claim": "Block-wise retrieval (finer granularity) outperforms function-wise retrieval across most models.",
       "evidence": "Tables 1-2 show Block-PKG consistently outperforms Func-PKG (e.g., StarCoder2-7B on HumanEval: 59% vs 56%).",
       "supported": "strong"
     },
     {
       "claim": "Tree pruning improves retrieval quality.",
       "evidence": "Table 7 (ablation study) shows Block-PKG with pruning outperforms without pruning across all models (1-11pp gains).",
       "supported": "strong"
     },
     {
       "claim": "RAG can introduce new error types even while reducing others.",
       "evidence": "Table 5 shows assertion errors decrease but name errors and type errors can increase when using Block-PKG.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes Programming Knowledge Graphs (PKG) for RAG-based code generation, evaluated on HumanEval and MBPP with five models (7B-13B). PKG with re-ranking improves pass@1 by up to 20pp over NoRAG. Finer-grained block-wise retrieval outperforms function-wise retrieval. However, RAG can degrade performance on certain problem categories (string manipulation, data structures) and introduce new error types while reducing others.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval and MBPP were published in 2021, well before the training data cutoffs of all models used. The paper does not discuss whether these benchmarks appeared in training data, which could inflate baseline and method results alike."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only pass@1 with greedy decoding is reported. No pass@k for k>1, no code quality metrics, no functional correctness beyond test cases."
     },
     {
       "flag": "No statistical testing",
       "detail": "All comparisons are based on raw percentage differences with no significance tests, confidence intervals, or variance measures. With small benchmark sizes (164 HumanEval problems), observed differences could be due to chance."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any formal limitations or threats-to-validity discussion, despite making broad claims about code generation improvement."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for LLM code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used for evaluating code generation capabilities."
     },
     {
       "title": "CodeRAG-Bench: Can retrieval augment code generation?",
       "authors": ["Zora Zhiruo Wang"],
       "year": 2024,
       "arxiv_id": "2406.14497",
       "relevance": "Directly related benchmark studying RAG for code generation, key baseline and motivation for this work."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the primary code LLMs evaluated in code generation benchmarks."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the barrier of closed-source models in code intelligence",
       "authors": ["Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Code LLM evaluated in the study, relevant to understanding model capabilities for code generation."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The next generation",
       "authors": ["Anton Lozhkov"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Code LLM used as baseline model in code generation benchmarks."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent approach to code generation, related agentic coding methodology."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Comprehensive RAG survey relevant to understanding retrieval-augmented approaches for LLMs."
     },
     {
       "title": "DocPrompting: Generating code by retrieving the docs",
       "authors": ["Shuyan Zhou"],
       "year": 2022,
       "relevance": "Prior work on documentation-based RAG for code generation, directly compared in this paper."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Repository-level code completion using iterative RAG, related retrieval-based code generation approach."
     }
   ]
 }