ai-research-survey

scan.json (23082B)

---

 {
   "paper": {
     "title": "COCOMIC: Code Completion By Jointly Modeling In-file and Cross-file Context",
     "authors": [
       "Yangruibo Ding",
       "Zijian Wang",
       "Wasi Uddin Ahmad",
       "Murali Krishna Ramanathan",
       "Ramesh Nallapati",
       "Parminder Bhatia",
       "Dan Roth",
       "Bing Xiang"
     ],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2212.10007"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/amazon-science/cocomic (footnote 1 in Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states they release a dataset for statement-level code completion (contribution 3: 'We release a diverse and high-quality dataset on statement-level code completion'). The dataset is linked via the same GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Transformers (Wolf et al., 2020) and training on 8 Nvidia A100s (Appendix D), but does not provide a requirements.txt, Dockerfile, or detailed dependency list with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Appendix D mentions hardware and training time but does not give commands or scripts to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-5 and 10 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims COCOMIC 'significantly outperforms' baselines (e.g., Table 4 caption) but no statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as relative percentage improvements with baseline context. For example, '33.94% relative increase in exact match' comparing COCOMIC (21.39%) to the finetuned CodeGen baseline (15.97%). Tables provide raw numbers enabling readers to compute effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set contains 6,888 prompts (Section 5.1) but there is no justification for why this size is adequate, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D explicitly states: 'We perform one round of experiments only as it is very expensive to repeat the experiments many times.' No variance, standard deviation, or multi-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: zero-shot CodeGen, finetuned CodeGen, and CodeGen with cross-file context prepended as plain text (Section 5.3, Table 1). Additional baseline variants are in Appendix E.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeGen (Nijkamp et al., 2022) was contemporary to the paper's submission in December 2022. The concurrent work by Shrivastava et al. (2022) and Zhang et al. (2023) is discussed in related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies are provided: random vs. CCFINDER entities (Table 3), 1-hop vs. 2-hop retrieval (Table 3), [SUM] vs. mean pooling (Table 4), locale vs. MTL vs. no relations (Table 5), and additional baseline variants (Appendix E)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Code Match EM, BLEU-4, Identifier Match EM, Identifier Precision, Identifier Recall, and Perplexity (Section 5.3, Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated via exact match, BLEU-4, identifier match, and perplexity. Human evaluation of code quality or correctness would have strengthened the claims about practical utility."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is divided into 80%/10%/10% train/validation/test splits (Section 5.1). Additionally, the test set explicitly excludes projects used as dependencies by training projects to prevent information leakage."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate metrics across the full test set. There is no breakdown by project size, complexity, number of cross-file dependencies, or programming pattern."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses failure cases: COCOMIC performs 5-7% lower than finetuned CodeGen when cross-file context is absent. Case studies in Appendix F show qualitative examples of both successes and baseline failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are reported: the MTL approach for encoding entity relations achieved 97.2% accuracy on the auxiliary task but 'hardly improves COCOMIC in code completion' (Section 6.4, Table 5). Also, COCOMIC degrades 5-7% without cross-file context (Limitations section)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '33.94% relative increase in exact match and a 28.69% relative increase in identifier matching' are directly supported by Table 1 (COCOMIC 21.39% vs. finetuned CodeGen 15.97% EM, and 31.26% vs. 24.29% ID EM)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about cross-file context improving code completion. These are justified through controlled ablation studies: comparing COCOMIC with/without cross-file context, random vs. CCFINDER entities, and different encoding strategies (Tables 1, 3, 4, 5)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds generalization: 'Our work focuses on Python language' and acknowledges the technique is not tested on larger models ('our work does not substantiate that our proposed technique would boost the performance of language models of any size'). The paper also notes it is a 'proof-of-concept' for Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the improvements. For example, it does not consider whether the gains come from additional training data exposure rather than the cross-file architecture, or whether the test set is biased toward cases where cross-file context is easy to retrieve."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The backbone model is specified as 'CodeGen-350M-Mono' (Section 5.2), which is an exact model variant with specific size and training data specification."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "COCOMIC is a fine-tuned model, not a prompting-based approach. The model is trained end-to-end with cross-file context encoded via the [SUM] mechanism, not through prompting an LLM."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported in Section 5.2: 'finetune the model for 5 epochs with max sequence length of 2,048 tokens and learning rate of 5e-5 with 5% warm-up steps then cosine annealing.' The retrieval uses k=2 hops, max 128 entities, each up to 128 tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "COCOMIC is not an agentic system. It is a fine-tuned language model with a retrieval component (CCFINDER), not an agentic scaffold with tools, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: projects filtered by size (5-5k Python files), train/val/test split (80/10/10), test set excludes dependency projects, prompts cut at cross-file context locations, samples filtered for AST parseability and local API calls, yielding 6,888 test prompts (Section 5.1, Appendix C)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after Section 8 (Conclusion) with three substantive subsections: extension to other languages, performance without cross-file context, and impact on different model sizes."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: Python-only scope, single model size (350M), degraded performance without cross-file context (5-7% drop), and the hypothesis that larger models would benefit similarly is explicitly not substantiated."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope boundaries are stated: Python only, project-level context only (not third-party packages), CodeGen-350M only, and the authors explicitly state 'our work does not substantiate that our proposed technique would boost the performance of language models of any size.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released via the GitHub repository (https://github.com/amazon-science/cocomic), and the data stems from the publicly available Python Package Index (PyPI)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: projects collected from PyPI, filtered for permissive licenses, filtered out projects with too few files (<=5) or too many nodes (>=5k), resulting in 60,891 projects (Section 5.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public repository (PyPI)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: PyPI collection -> license filtering -> size filtering (60,891 projects) -> 80/10/10 split with dependency exclusion -> prompt creation at cross-file context locations -> AST parseability and local API filtering -> 6,888 test prompts. Filtering criteria are specified at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Most authors are from AWS AI Labs, but there is no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Yangruibo Ding from Columbia University, all other authors from AWS AI Labs (Amazon). Email addresses confirm the affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are from AWS AI Labs (Amazon), which has a commercial interest in code completion tools. The funder (Amazon/AWS) is not independent of the outcome, as improved code completion directly benefits their products. This conflict is not disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper. Authors at Amazon/AWS may hold equity or other financial interests related to code completion technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses CodeGen-350M-Mono as the backbone but does not state the training data cutoff date for this pretrained model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses train-test overlap by ensuring test set projects are not dependencies of training projects (Section 5.1: 'we only include projects that were not used as dependencies by any training projects in the test set'). However, this addresses their fine-tuning data, not the pretrained model's data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is built from PyPI, which is publicly available. The CodeGen pretrained model may have trained on PyPI packages, but this potential contamination between the pretrained model's training data and the benchmark is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported. The paper does not mention how long CCFINDER takes to retrieve context or how much inference time the [SUM] encoding adds compared to standard CodeGen."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D states: 'We train our models on a machine with 8 Nvidia A100s. Each job takes around 50 hours (i.e., 400 GPU hours) to train.'"
       }
     }
   },
   "claims": [
     {
       "claim": "COCOMIC improves over finetuned CodeGen by 33.94% relative increase in exact match for code completion.",
       "evidence": "Table 1 shows COCOMIC achieves 21.39% EM vs. finetuned CodeGen at 15.97% EM, a relative improvement of (21.39-15.97)/15.97 = 33.94%.",
       "supported": "strong"
     },
     {
       "claim": "COCOMIC improves identifier matching by 28.69% relative increase over finetuned CodeGen.",
       "evidence": "Table 1 shows COCOMIC achieves 31.26% ID Match EM vs. finetuned CodeGen at 24.29%, a relative improvement of (31.26-24.29)/24.29 = 28.69%.",
       "supported": "strong"
     },
     {
       "claim": "CCFINDER retrieves 27.07% more relevant identifiers than in-file context alone.",
       "evidence": "Table 2 shows in-file context covers 75.19% of identifiers, while cross-file context brings recall to 95.55%, a 27.07% increase. This measures identifier coverage, not end-to-end model performance.",
       "supported": "strong"
     },
     {
       "claim": "The [SUM] token representation significantly outperforms mean pooling for cross-file context encoding.",
       "evidence": "Table 4 shows [SUM] achieves 21.39% Code EM vs. mean pooling at 16.78%, and 31.26% ID EM vs. 25.01%.",
       "supported": "moderate"
     },
     {
       "claim": "Adding cross-file context does not degrade perplexity when cross-file context is not required.",
       "evidence": "Table 1 shows COCOMIC achieves the lowest perplexity (2.69) across all settings, lower than all baselines including those without cross-file context.",
       "supported": "moderate"
     },
     {
       "claim": "Locales improve cross-file context representation while multi-task learning with edge prediction does not.",
       "evidence": "Table 5 shows locale-based COCOMIC (21.39% EM) outperforms no-relations (20.27%) and MTL (20.01%). MTL achieves 97.2% edge prediction accuracy but does not improve code completion.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "COCOMIC, a framework that jointly models in-file and cross-file context for code completion, achieves a 33.94% relative improvement in exact match over a finetuned CodeGen-350M baseline on a Python dataset from PyPI. The key innovation is compressing cross-file entities into single-token representations via a [SUM] token, enabling the model to incorporate up to 16,384 tokens of cross-file context within limited input length. Ablation studies show that the static retrieval tool CCFINDER retrieves 27.07% more relevant identifiers than in-file context alone, and that the [SUM] representation substantially outperforms mean pooling. However, all results are single-run on one model size (350M) and one language (Python), limiting generalizability.",
   "red_flags": [
     {
       "flag": "Single-run experiments",
       "detail": "Appendix D explicitly states 'We perform one round of experiments only as it is very expensive to repeat the experiments many times.' Without variance estimates, it is impossible to assess whether the reported improvements are stable or due to training randomness."
     },
     {
       "flag": "No significance tests",
       "detail": "Claims of 'significantly outperforms' (e.g., Table 4 caption) are made without any statistical significance tests. All reported differences are raw number comparisons."
     },
     {
       "flag": "Company evaluating its own product",
       "detail": "6 of 8 authors are from AWS AI Labs (Amazon). Amazon has commercial interest in code completion tools. No conflicts of interest statement is provided."
     },
     {
       "flag": "Potential pretrained model contamination",
       "detail": "The benchmark is built from PyPI (public Python packages). CodeGen was pretrained on GitHub code which likely includes many of these same packages. The paper addresses fine-tuning data leakage but not pretrained model contamination."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Introduces Codex and HumanEval, foundational work on LLM-based code generation evaluation."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2022,
       "relevance": "The backbone model used in COCOMIC; open-source code LLM for code generation."
     },
     {
       "title": "Grounded copilot: How programmers interact with code-generating models",
       "authors": ["Shraddha Barke", "Michael B James", "Nadia Polikarpova"],
       "year": 2022,
       "relevance": "Studies developer interaction with AI code generation tools, relevant to understanding productivity impact."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle", "Daniel Tarlow"],
       "year": 2022,
       "relevance": "Proposes repository-level prompt engineering for code LLMs, a closely related approach to cross-file context."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Concurrent work on repository-level code completion using iterative retrieval, directly comparable approach."
     },
     {
       "title": "DocCoder: Generating code by retrieving and reading docs",
       "authors": ["Shuyan Zhou", "Uri Alon", "Frank F Xu"],
       "year": 2022,
       "relevance": "Retrieval-augmented code generation using API documentation, related approach to using external context for code completion."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F Xu", "Uri Alon", "Graham Neubig"],
       "year": 2022,
       "relevance": "Systematic evaluation methodology for code LLMs, relevant to benchmarking and evaluation design."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan"],
       "year": 2022,
       "relevance": "Code infilling model relevant to code completion approaches and evaluation."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "relevance": "Large-scale code generation system evaluated on competitive programming, relevant to code generation capability assessment."
     },
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee"],
       "year": 2022,
       "relevance": "Discusses hallucination in generation models, which COCOMIC addresses by providing cross-file context to reduce code hallucination."
     }
   ]
 }