ai-research-survey

scan.json (20165B)

---

 {
   "paper": {
     "title": "Completion by Comprehension: Guiding Code Generation with Multi-Granularity Understanding",
     "authors": ["Xinkui Zhao", "Rongkai Liu", "Yifan Zhang", "Chen Zhi", "Lufei Zhang", "Guanjie Cheng", "Yueshen Xu", "Shuiguang Deng", "Jianwei Yin"],
     "year": 2025,
     "venue": "Manuscript submitted to ACM",
     "arxiv_id": "2512.04538"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We make the code publicly available at the link' but no actual URL is provided — just the word 'link' as a placeholder. No working repository URL found."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks CrossCodeEval and RepoEval, and reuses publicly available evaluation implementations from prior studies [8, 52]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'one Tesla A100 GPU, with 40 GB memory' but provides no requirements.txt, Dockerfile, or library version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The experimental setup section describes settings but not how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 and Figures 7-8 report only point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoCo 'consistently outperforms' baselines but provides no statistical significance tests. Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '20.2% gains in EM' and provides absolute numbers in tables (e.g., from 24.91% to 29.93%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes. The RepoEval subset is stated as 1600 samples but no rationale is given for why this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the random seed is set to 123' suggesting single-run experiments. No variance, standard deviation, or multi-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: RawRAG, RepoCoder, and RLCoder (Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "RLCoder (2024) is described as the state-of-the-art method. RepoCoder (2023) is also recent. These are reasonable contemporary baselines for repository-level code completion."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Two ablation studies are conducted: RQ2 removes Code Comprehension and Structural-Aware Re-Ranker individually (Section 5.2), and RQ5 tests individual granularity levels (Section 5.5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Exact Match (EM), Edit Similarity (ES), Identifier Exact Match (ID.EM), and F1-score (Section 4.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is performed. All evaluation is automated via EM, ES, ID.EM, and F1 against ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets from CrossCodeEval and RepoEval, which have predefined splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (CrossCodeEval Python, CrossCodeEval Java, RepoEval Line, RepoEval API) and by model (CodeLlama-7B, Yi-Coder-1.5B, DeepSeekCoder-1B) in Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case study (Section 5.6) shows a success case. No failure cases or error analysis of where CoCo produces incorrect completions are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CoCo_all does not significantly outperform CoCo_pro (Section 5.5), and that the re-ranker provides only 'relatively modest' gains (Section 5.2). The Jaccard similarity limitation is acknowledged."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 20.2% gains in EM' which is supported by Table 1 (Yi-Coder-1.5B on CrossCodeEval Python: 24.91 → 29.93, ~20.2% relative improvement). Claims of consistent outperformance are supported across Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('multi-granularity context improves completion') are supported by controlled ablation studies (RQ2, RQ5) that systematically remove components and measure impact."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Guiding Code Generation' generally but experiments are limited to Python and Java on two benchmarks with three relatively small models (1B-7B). The abstract claims the framework is 'model-agnostic' without testing on larger or proprietary models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 6) discusses limitations of static analysis tools but does not consider alternative explanations for the observed improvements (e.g., whether gains come from simply providing more context tokens rather than structured comprehension)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified as 'DeepSeekCoder-1B', 'Codellama-7B', 'Yi-Coder-1.5B' without specific version identifiers or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6 shows a prompt template with placeholders (e.g., 'file_info 1', 'pro_info 1', 'code example 1') but the actual fill values and complete prompt text are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Random seed (123) and max tokens (64) are stated. However, temperature, top-p, and other sampling parameters are not reported. The teleportation factor α for PageRank and top-k values for context selection are not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "CoCo is a pipeline framework (static analysis + retrieval + generation), not an agentic scaffolding system with retry logic or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The methodology (Section 3) describes in detail how code is parsed at function, file, and project levels using AST parsers and Tree-sitter, how the graph-based context selection filters information, and how the re-ranker processes candidates."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides substantive discussion of limitations across three paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: CFG construction missing complex execution paths, context inflation exceeding model windows, inability to handle non-entity code elements like comments and annotations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what settings or claims are excluded. It does not bound its claims to the tested languages (Python, Java) or model sizes (1B-7B)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs, generated code samples, or per-example results are made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (CrossCodeEval, RepoEval) are well-described with references. The paper describes using Python and Java subsets of CrossCodeEval and line/API subsets of RepoEval (1600 samples)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from code repository input through AST parsing, multi-granularity context extraction, graph-based selection, retrieval, re-ranking, and prompt construction is documented in Section 3 with algorithms and figures."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Section 9 is titled 'Acknowledgments' but contains no content — no funding sources are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Zhejiang University, State Key Laboratory of Mathematical Engineering and Advanced Computing, and Xidian University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The empty Acknowledgments section suggests non-disclosure rather than absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained code LLMs (DeepSeekCoder, CodeLlama, Yi-Coder) on benchmarks but does not state training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark examples (CrossCodeEval, RepoEval) could have appeared in the training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CrossCodeEval (2023) and RepoEval (2023) were published before or around the training of the models used, yet no contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports wall-clock inference times and overhead in seconds for each benchmark (e.g., 132.4s overhead on CrossCodeEval Python, ~1.9% increase)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (Tesla A100 40GB) but total compute budget, GPU hours, or total experiment time is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CoCo achieves up to 20.2% gains in Exact Match over state-of-the-art baselines.",
       "evidence": "Table 1: Yi-Coder-1.5B on CrossCodeEval Python, RLCoder EM=24.91 vs CoCo EM=29.93, ~20.2% relative improvement.",
       "supported": "moderate"
     },
     {
       "claim": "CoCo consistently outperforms all baselines across all backbone LLMs and benchmarks.",
       "evidence": "Table 1 shows CoCo outperforming RawRAG, RepoCoder, and RLCoder across all 4 benchmarks and 3 models on all 4 metrics. However, only single-run results with no statistical tests.",
       "supported": "moderate"
     },
     {
       "claim": "CoCo can be seamlessly integrated into existing methods as a plug-in, consistently enhancing performance.",
       "evidence": "Table 2 shows RawRAG+CoCo and RepoCoder+CoCo both improve over their base methods across all benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "The integration overhead of CoCo is negligible (below 5% latency increase).",
       "evidence": "Table 3 shows overhead ranging from 1.0% to 2.1% across benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "Project-level information contributes the most to code generation performance among the three granularity levels.",
       "evidence": "Section 5.5 and Figure 8 show CoCo_pro outperforming CoCo_func and CoCo_file on CrossCodeEval Python.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoCo proposes a comprehension-driven approach to repository-level code completion that extracts multi-granularity context (function, file, project level) via static analysis and uses Personalized PageRank for context selection. On CrossCodeEval and RepoEval benchmarks with three small code LLMs (1B-7B parameters), CoCo achieves up to 20.2% relative EM improvement over RLCoder. The framework adds less than 2.1% latency overhead and can be integrated as a plug-in into existing RAG-based methods. Ablations show project-level context provides the largest gains, and the graph-based context selection is important for filtering noise from combined multi-granularity input.",
   "red_flags": [
     {
       "flag": "Single-run evaluation",
       "detail": "All experiments use a single random seed (123) with no variance reporting across multiple runs, making it impossible to assess result stability."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Claims of outperformance are based solely on comparing point estimates without any significance tests, despite some improvements being small (e.g., <2 percentage points)."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Benchmarks from 2023 are used with models that may have been trained on data including those benchmarks, but no contamination analysis is performed."
     },
     {
       "flag": "Placeholder code release link",
       "detail": "The paper claims code is publicly available but the actual URL is missing — only the word 'link' appears as a placeholder, suggesting the artifact is not actually released."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Key hyperparameters like temperature, top-p, PageRank teleportation factor α, and top-k values for context selection are not reported, hindering reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming–The Rise of Code Intelligence",
       "authors": ["Daya Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major open-source code LLM used as backbone in code generation evaluations."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family widely used as baseline in code generation research."
     },
     {
       "title": "Crosscodeeval: A diverse and multilingual benchmark for cross-file code completion",
       "authors": ["Yangruibo Ding"],
       "year": 2023,
       "relevance": "Key benchmark for evaluating repository-level code completion systems."
     },
     {
       "title": "Repocoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Influential RAG-based repository-level code completion method used as baseline."
     },
     {
       "title": "RLCoder: Reinforcement learning for repository-level code completion",
       "authors": ["Yanlin Wang"],
       "year": 2024,
       "arxiv_id": "2407.19487",
       "relevance": "State-of-the-art RL-based retriever for repository-level code completion, primary baseline."
     },
     {
       "title": "Graphcoder: Enhancing repository-level code completion via code context graph-based retrieval and language model",
       "authors": ["Wei Liu"],
       "year": 2024,
       "arxiv_id": "2406.07003",
       "relevance": "Graph-based retrieval framework for code completion, related structural approach."
     },
     {
       "title": "Dataflow-guided retrieval augmentation for repository-level code completion",
       "authors": ["Wei Cheng"],
       "year": 2024,
       "arxiv_id": "2405.19782",
       "relevance": "Dataflow-guided retrieval for repository code completion, closely related work."
     },
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM coding capabilities on real-world software engineering tasks."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler"],
       "year": 2022,
       "relevance": "Empirical study on productivity impact of code completion tools in practice."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Comprehensive survey of RAG techniques applied to LLMs."
     }
   ]
 }