ai-research-survey

scan.json (24914B)

---

 {
   "paper": {
     "title": "CodeFill: Multi-token Code Completion by Jointly Learning from Structure and Naming Sequences",
     "authors": ["Maliheh Izadi", "Roberta Gismondi", "Georgios Gousios"],
     "year": 2022,
     "venue": "ICSE '22",
     "arxiv_id": "2202.06689",
     "doi": "10.1145/3510003.3510172"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/saltudelft/codefill (footnote 1, Section 1). The abstract also states 'We publicly release our source code and datasets.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We publicly release our source code and datasets' in the abstract. One dataset (ETH 150K/PY117K) is a public benchmark from Raychev et al. The other (PY1690K) was collected by the authors and shared via the same GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions the hardware used (two GeForce GTX 1080 Ti GPUs, Intel Xeon CPU, 128G RAM) and the HuggingFace library, but there is no requirements.txt, Dockerfile, or detailed listing of library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the approach and configuration (Section 4.3) but does not include step-by-step reproduction instructions, a README with commands, or scripts to replicate experiments. While code is released, the paper itself does not provide reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-8 and Figure 6 are reported as single point estimates (e.g., 'MRR: 70.9%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes claims that 'CodeFill surpasses all baselines' and 'outperforms the state of the art' but provides no statistical significance tests (no p-values, t-tests, or other hypothesis tests). Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports baseline context alongside improvements, e.g., 'MRR: 70.9% vs. 66.2% and 67.8%' and 'ROUGE-L: 63.7% vs. 52.4% and 59.2%' in the abstract and throughout the results tables, providing enough context to understand the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses two datasets (29M and 425M LOC) but does not justify why these sizes are adequate, discuss power analysis, or acknowledge whether smaller datasets might yield different conclusions. No justification for the 90/10 train/test split is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All reported results appear to be from single runs. There is no mention of multiple runs, standard deviation, variance across seeds, or any spread measure. Tables 3-8 report only single numbers per metric."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares CodeFill against six baselines: n-gram+LSTM, Pointer Mixture, OpenVocab, T-XL+Bi-LSTM, GPT-C (IntelliCode Compose), and TravTrans+ (Section 2.4, Tables 3-8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines span 2017-2021: TravTrans+ (ICSE 2021), GPT-C (FSE 2020), OpenVocab (ICSE 2020), T-XL+Bi-LSTM (ICPC 2020). For a 2022 paper, these are contemporary and include the state of the art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 presents an ablation study examining vanilla GPT-2 vs. MTL with hard-parameter sharing vs. soft-parameter sharing vs. adding the statement completion task. Each component's contribution is measured (Section 5.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Accuracy and MRR for token-level prediction (Tables 3-6), and METEOR and ROUGE-L for statement-level prediction (Figure 6). Runtime latency is also reported (Table 8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates code completion quality, which involves subjective developer experience, but uses only automated metrics (Accuracy, MRR, METEOR, ROUGE-L). No human evaluation or user study is included to assess whether the completions are actually useful to developers in practice."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'We then use 90% of PY117K for fine-tuning the model on the tasks... and finally the last 10% of PY117K for evaluation.' PY1690K is used exclusively for pre-training, ensuring no train-test overlap."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4 and 5 provide per-category breakdowns by token type (Identifier, Keyword, Punctuation, Literals, Operators) and by leaf node type (Attribute Access, Names, Function names, Numeric constant). Table 6 provides cardinal point results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases, error examples, or qualitative analysis of where CodeFill produces incorrect completions. Results are presented only in aggregate metrics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) shows that hard-parameter sharing MTL performs slightly worse than soft-parameter sharing on MRR (79.6 vs 79.5), and the paper notes that TTP was excluded from fine-tuning because the model quickly learns type sequences. The paper also notes CodeFill is 'on par' (not superior) for MRR on keywords and punctuation (Section 5.1.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'MRR: 70.9% vs. 66.2% and 67.8%' for single token and 'ROUGE-L: 63.7% vs. 52.4% and 59.2%' for multi-token prediction. These are supported in Table 6 (cardinal point MRR) and Figure 6 (SLP ROUGE-L at n=4). The claim of public code/data release is supported by the GitHub link."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) provides controlled single-variable manipulation to support claims like 'employing the MTL technique to train the models jointly on multiple tasks indeed helps the model learn better' and 'training on two different granularity also helps them learn better.' Each component is added incrementally with measured impact."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states Python is the target language (Section 3.2) and the threats to validity section acknowledges 'further studies are needed to validate and generalize our findings to other programming languages' (Section 7). The title mentions 'code completion' broadly but the scope is bounded in the body."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 7) discusses generic threats (dataset quality, baseline implementation errors, evaluation metrics) but does not consider specific alternative explanations for why CodeFill outperforms baselines. For example, whether the gains come from the larger model size (258M vs. 119-177M params) rather than the architectural innovations is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-2 Transformer' and 'HuggingFace library' but does not specify which version of GPT-2 (small, medium, large) or the HuggingFace library version. Section 4.3 mentions implementation details but not specific model version identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains models from scratch using standard language modeling objectives. No prompting is involved."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports: learning rate 0.00001, maximum sequence length 2048, 100 epochs, beam width 5 (Section 3.2.4). Section 3.2 reports task probability distribution (20%/40%/40% for TTP/TVP/SC). Table 1 provides post-processing weights. 'We set the remaining parameters to default values' is stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CodeFill is a standard neural model architecture with no agent loop, tool use, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the full preprocessing pipeline: comment removal, AST extraction, module/library alias extraction, tokenization, BPE encoding, string/numeric literal normalization, and indentation handling. Section 4.2 describes dataset collection and deduplication using the method from Allamanis [2]."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to the Validity' provides a dedicated subsection covering internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: using replication packages from baseline studies to avoid implementation errors, using only Python datasets (external validity), the need to validate on other languages, data quality from open-source GitHub repos, and suitability of ROUGE-L/METEOR metrics adapted from NLP."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats to validity section mentions Python-only evaluation and the need for other languages, it does not explicitly state what the results do NOT show. It does not bound claims to specific repository types, developer experience levels, or completion scenarios beyond what was tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states datasets are publicly released. PY117K is based on the public ETH 150K dataset. PY1690K was collected from GHTorrent and is released at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes data collection: ETH 150K from Raychev et al. (repositories with permissive licenses, parsed ASTs, removed obfuscated files), PY1690K from GHTorrent (all non-forked Python repositories with more than 20 stars, 58k repositories). Deduplication method from Allamanis [2] is cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data comes from standard public code repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: raw GitHub repos → filter by stars/license → deduplication (Allamanis method) → PY117K and PY1690K → preprocessing (Section 3.1: comment removal, AST extraction, BPE encoding) → 90/10 train/eval split. Table 2 provides dataset statistics at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses: 'This work has received funding from the European Union's Horizon 2020 research and innovation programme under grant number 825328 (FASTEN project), and also the NWO MIPL project, grant number 628.008.003.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed as affiliated with Delft University of Technology, Delft, Netherlands. Their email addresses are provided. No conflicts with evaluated products exist since CodeFill is their own novel system."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from the EU Horizon 2020 program (FASTEN project) and NWO (Dutch Research Council). These are public research funding bodies with no financial stake in whether CodeFill outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While the risk of conflict appears low (academic researchers), the absence of a declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses models trained from scratch on collected datasets, but does not state when the GHTorrent data snapshot was taken for the PY1690K dataset. The ETH 150K dataset collection date is also not specified. Without knowing when the data was collected, one cannot assess whether test files may have appeared in similar training distributions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses potential overlap: PY1690K is used exclusively for pre-training and PY117K for fine-tuning/evaluation (Section 4.2). It explicitly states 'PY1690K and PY117K do not have any common files.' Deduplication using Allamanis [2] is applied to both datasets to prevent file-level overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the established deduplication method from Allamanis [2] and explicitly ensures no overlap between training and evaluation datasets. Section 4.2 states datasets were deduplicated and have no common files. The models are trained from scratch (not pre-trained on external data), limiting contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a benchmark evaluation of code completion models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports inference latency in milliseconds for all models (CodeFill: 73ms). Section 5.4 discusses the practical constraint that latency must be under 100ms for IDE use."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports training time in hours for all models (CodeFill: 24h). Section 4.3 specifies the hardware: two GeForce GTX 1080 Ti GPUs, Intel Xeon CPU E5-2690 v4, 128G RAM. The ablation study (Table 7) also reports training times for each configuration."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeFill outperforms all baselines in single token prediction (MRR: 81.7% vs. TravTrans+ 79.4% and GPT-C 80.0%)",
       "evidence": "Table 3 (TLP-A) shows CodeFill achieves 80.6% accuracy and 81.7% MRR, surpassing TravTrans+ (78.9%/79.4%) and GPT-C (79.8%/80.0%).",
       "supported": "moderate"
     },
     {
       "claim": "CodeFill significantly outperforms the state of the art for multi-token prediction (ROUGE-L: 63.7% vs. 52.4% for TravTrans+ at n=4)",
       "evidence": "Figure 6 shows ROUGE-L scores for statement-level prediction across n=2 to n=8. At n=4, CodeFill achieves approximately 63.7% vs. TravTrans+ 52.4%. The margin grows wider as n increases.",
       "supported": "moderate"
     },
     {
       "claim": "Each component of the CodeFill model (MTL, soft-parameter sharing, statement completion task) contributes to performance",
       "evidence": "Table 7 ablation study shows incremental improvements: GPT-2 (78.2 MRR) → MTL HP (79.6) → MTL SP (79.5) → MTL SP with statement completion (81.7).",
       "supported": "moderate"
     },
     {
       "claim": "CodeFill's cardinal point evaluation is more representative of real-world autocompletion use",
       "evidence": "Section 4.1.1 describes the method for identifying cardinal points and cites Hellendoorn et al. [18] arguing standard evaluation is unrealistic. Table 6 shows CodeFill outperforms baselines on cardinal points (MRR: 70.9% vs. 66.2%).",
       "supported": "moderate"
     },
     {
       "claim": "All models feature average latency under 100ms, making them practical for IDE use",
       "evidence": "Table 8 reports latencies: CodeFill 73ms, all baselines between 53-79ms. Section 5.4 states this is within the 'golden standard in the industry.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeFill, a multi-task learning approach using parallel GPT-2 Transformers that jointly learns from token values, token types, and statement completion, outperforms six baselines on Python code completion. On single-token prediction, CodeFill achieves 81.7% MRR (vs. 79.4% for the best baseline TravTrans+), and on statement-level prediction with 4 tokens, it achieves 63.7% ROUGE-L (vs. 52.4% for TravTrans+). The ablation study confirms each component (MTL, soft-parameter sharing, statement completion task) contributes to performance. The paper also introduces a cardinal-point evaluation methodology that better reflects real-world autocompletion trigger points.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons between CodeFill and baselines are based on raw point estimates without any significance tests. The claimed improvements (e.g., 81.7% vs. 79.4% MRR) could potentially be within noise range, but no tests are performed to rule this out."
     },
     {
       "flag": "No variance or multi-run reporting",
       "detail": "Results appear to be from single training runs with no reported standard deviation or variance across seeds. Without knowing result variability, the reader cannot assess whether the improvements are stable or an artifact of a particular random initialization."
     },
     {
       "flag": "Model size confound in ablation",
       "detail": "CodeFill has 258M parameters compared to 119-177M for baselines (Table 8). The ablation study does not control for model size — it is unclear whether performance gains come from the architectural innovations or simply from having more parameters. The paper does not discuss this confound."
     },
     {
       "flag": "No failure analysis",
       "detail": "Despite claiming practical benefits for developers, the paper presents no qualitative analysis of failure cases — what kinds of completions CodeFill gets wrong, or where the baselines outperform it on specific examples."
     }
   ],
   "cited_papers": [
     {
       "title": "IntelliCode Compose: Code Generation Using Transformer",
       "authors": ["Alexey Svyatkovskiy", "Shao Kun Deng", "Shengyu Fu", "Neel Sundaresan"],
       "year": 2020,
       "relevance": "GPT-2 based multilingual code completion deployed in VS Code IDE, a direct baseline for code completion research."
     },
     {
       "title": "Code Prediction by Feeding Trees to Transformers",
       "authors": ["Seohyun Kim", "Jinman Zhao", "Yuchi Tian", "Satish Chandra"],
       "year": 2021,
       "doi": "10.1109/ICSE43902.2021.00026",
       "relevance": "State-of-the-art tree-based Transformer for code completion (TravTrans+), the primary comparison baseline."
     },
     {
       "title": "When Code Completion Fails: A Case Study on Real-World Completions",
       "authors": ["Vincent J Hellendoorn", "Sebastian Proksch", "Harald C Gall", "Alberto Bacchelli"],
       "year": 2019,
       "relevance": "Identified fundamental issues with code completion evaluation methodology, motivating CodeFill's cardinal point evaluation approach."
     },
     {
       "title": "Big Code != Big Vocabulary: Open-Vocabulary Models for Source Code",
       "authors": ["Rafael-Michael Karampatsis", "Hlib Babii", "Romain Robbes", "Charles Sutton", "Andrea Janes"],
       "year": 2020,
       "relevance": "BPE-based open-vocabulary code model addressing the OOV problem in code completion, a baseline in this study."
     },
     {
       "title": "A Self-Attentional Neural Architecture for Code Completion with Multi-Task Learning",
       "authors": ["Fang Liu", "Ge Li", "Bolin Wei", "Xin Xia", "Zhiyi Fu", "Zhi Jin"],
       "year": 2020,
       "relevance": "MTL-based code completion using Transformer-XL and Bi-LSTM, a direct predecessor and baseline."
     },
     {
       "title": "Pythia: AI-Assisted Code Completion System",
       "authors": ["Alexey Svyatkovskiy", "Ying Zhao", "Shengyu Fu", "Neel Sundaresan"],
       "year": 2019,
       "relevance": "Early LSTM-based code completion integrated into VS Code IDE, demonstrating practical deployment of ML-based completion."
     },
     {
       "title": "The Adverse Effects of Code Duplication in Machine Learning Models of Code",
       "authors": ["Miltiadis Allamanis"],
       "year": 2019,
       "relevance": "Proposed deduplication method used in CodeFill's dataset preparation, addressing data quality in code ML research."
     },
     {
       "title": "Are Deep Neural Networks the Best Choice for Modeling Source Code?",
       "authors": ["Vincent J Hellendoorn", "Premkumar Devanbu"],
       "year": 2017,
       "relevance": "Foundational study comparing n-gram and neural approaches for source code modeling, a baseline in this evaluation."
     },
     {
       "title": "On the Naturalness of Software",
       "authors": ["Abram Hindle", "Earl T Barr", "Zhendong Su", "Mark Gabel", "Premkumar Devanbu"],
       "year": 2012,
       "relevance": "Established the naturalness hypothesis for software, foundational to treating code completion as a language modeling task."
     },
     {
       "title": "Sequence Model Design for Code Completion in the Modern IDE",
       "authors": ["Gareth Ari Aye", "Gail E Kaiser"],
       "year": 2020,
       "relevance": "Study on designing sequence models specifically for IDE code completion, relevant to practical code completion systems."
     }
   ]
 }