ai-research-survey

scan.json (20687B)

---

 {
   "paper": {
     "title": "A Combined Approach of Program Analysis and Deep Learning for Code Completion",
     "authors": ["Yi Liu"],
     "year": 2024,
     "venue": "Scientific Journal of Technology",
     "doi": "10.54691/hkyc3a89"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'public datasets for Python and JavaScript' but does not name the specific datasets, provide download links, or cite the data sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, requirements files, or hardware details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are mentioned. The methodology section describes the approach conceptually but does not provide enough detail to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., '82.16% accuracy') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'our method significantly outperforms existing approaches' in the abstract but provides no statistical significance tests. The comparison in Table 1 appears to show two columns per language but no baselines are clearly labeled or tested for significance."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Results are presented as raw accuracy and MRR numbers without baseline context showing the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the dataset sizes. The paper says datasets were randomly split 70/15/15 but never states the total number of examples."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results across multiple runs are reported. Only single point estimates appear in Table 1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 appears to have two columns per language but the columns are not labeled with baseline names. The abstract claims the method 'significantly outperforms existing approaches' but no explicit baseline comparisons with named prior methods are presented."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are clearly identified. The references cite work from 2009-2022 but no specific prior methods are compared against in the experiments."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (program analysis, GGNN, Transformer, MLP) but no ablation study is conducted to determine the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both Accuracy and Mean Reciprocal Rank (MRR) for both Python and JavaScript datasets."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The paper claims the approach 'enhances development efficiency and code quality' but evaluation is entirely automated metrics (accuracy and MRR). Human evaluation of code completion quality would be relevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes a 70/15/15 split for training, validation, and test sets: 'Randomly select training, validation, and test sets from the Python and JavaScript datasets, with the training set comprising 70%, the validation set 15%, and the test set 15%.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate accuracy and MRR per language. No per-category breakdown (e.g., by code construct type, complexity level, or project type) is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses failure modes: 'the experiments observed a decline in model performance in more complex code structures, such as nested loops and deep function calls.' Section 5 also acknowledges 'limitations in the current model's performance when dealing with highly nested code and multilevel function calls.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper mentions performance declines in complex structures qualitatively but does not show any quantitative negative results, failed approaches, or ablations that hurt performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'our method significantly outperforms existing approaches in terms of code completion accuracy and Mean Reciprocal Rank.' However, no clear baseline comparisons are presented in the results — Table 1 shows numbers but without labeled baselines, so the claim of outperformance is not verifiably supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'by integrating program analysis techniques and deep learning methods, our code completion system can effectively identify and complete code snippets, thereby enhancing development efficiency and code quality.' Without ablation studies isolating the contribution of program analysis vs. deep learning, these causal attributions are not justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title broadly claims 'A Combined Approach of Program Analysis and Deep Learning for Code Completion' without bounding to the tested setting. While the paper tests on Python and JavaScript, the conclusions in Section 5 speak generally about 'enhancing the accuracy and efficiency of code completion' without qualifying that results are limited to these two languages and specific unnamed datasets."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether improvements might be due to dataset characteristics, increased model capacity, or other confounding factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specific model versions, library versions, or implementation details are provided. The paper describes GGNN and Transformer architecturally but does not specify which implementations or versions were used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It trains a neural network model (GGNN + Transformer) from scratch rather than prompting an LLM."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only learning rate (0.001) and batch size (128) are mentioned along with the Adam optimizer. Critical hyperparameters like embedding dimension D, number of GGNN layers, number of Transformer heads/layers, dropout rate, number of training epochs, and L2 regularization coefficient are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard neural network training and inference pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'code is collected from open-source community websites like GitHub and Stack Overflow' and 'after cleansing' but does not describe the cleansing procedure, filtering criteria, how many examples were collected, or any details of the preprocessing pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Limitations are briefly mentioned in Section 5 (Conclusion) as a single sentence about nested code, but this does not constitute a substantive limitations section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The brief mention of performance decline on complex structures in Section 4 is an observation about results, not a discussion of threats to the study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is made available. The paper references 'public datasets' but does not name them, provide links, or release processed data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Data collection is described only vaguely: 'code is collected from open-source community websites like GitHub and Stack Overflow.' No specifics about time period, selection criteria, number of repositories, or filtering are provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from public code repositories, which is a standard benchmark-style data source."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The data pipeline from raw code to training examples is not documented. The paper jumps from 'code is collected' to 'program analysis is performed' without detailing intermediate steps, filtering counts, or transformation procedures."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work is supported by the Research Project of Hunan Provincial Education Department [Grant No.22C0600].'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is clearly stated: 'School of Information, Hunan University of Humanities Science and Technology, Loudi, Hunan, 417000, China.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Hunan Provincial Education Department, a government education body with no apparent financial interest in the outcome of code completion research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own model from scratch on code datasets rather than evaluating a pre-trained model's capability on a benchmark. Contamination in the LLM sense does not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper trains a custom model with a described train/test split, not evaluating a pre-trained LLM on a benchmark. Standard train/test overlap is addressed by the 70/15/15 split."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model is evaluated on a public benchmark. The model is trained and tested on the paper's own data splits."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-prediction are reported despite the method being proposed for use in IDEs where latency is critical."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training time, GPU hours, hardware specifications, or computational budget are stated."
       }
     }
   },
   "claims": [
     {
       "claim": "The combined program analysis and deep learning method significantly outperforms existing approaches in code completion accuracy and MRR.",
       "evidence": "Abstract and Section 3 (Table 1) report accuracy of 82.16% and MRR of 87.02% for Python, and accuracy of 81.37% and MRR of 86.45% for JavaScript. However, no baseline methods are clearly identified or compared against.",
       "supported": "unsupported"
     },
     {
       "claim": "The approach enhances development efficiency and code quality by providing contextually relevant code snippets.",
       "evidence": "Section 3 states the results 'demonstrate that by integrating program analysis techniques and deep learning methods, our code completion system can effectively identify and complete code snippets, thereby enhancing development efficiency and code quality.' No user studies or efficiency measurements are provided.",
       "supported": "unsupported"
     },
     {
       "claim": "The system performs slightly better with Python code than with JavaScript.",
       "evidence": "Section 4 discusses this observation based on Table 1 numbers, where Python accuracy (82.16%) slightly exceeds JavaScript accuracy (81.37%). However, without statistical tests or variance measures, this small difference cannot be confirmed as meaningful.",
       "supported": "weak"
     },
     {
       "claim": "Model performance declines in more complex code structures such as nested loops and deep function calls.",
       "evidence": "Section 4 states: 'the experiments observed a decline in model performance in more complex code structures, such as nested loops and deep function calls.' No quantitative evidence for this observation is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes combining program analysis (inter-procedural backward slicing to construct program graphs) with GGNN and Transformer models for code completion. On unnamed Python and JavaScript datasets, it reports accuracy of ~82% and ~81% and MRR of ~87% and ~86% respectively. However, the lack of clearly identified baselines, named datasets, ablation studies, and statistical analysis makes the results difficult to interpret or verify.",
   "red_flags": [
     {
       "flag": "No identifiable baselines",
       "detail": "The abstract claims the method 'significantly outperforms existing approaches' but Table 1 does not clearly label any baseline methods. The two columns per language may represent before/after but this is never explained, making the claim of outperformance unverifiable."
     },
     {
       "flag": "Unnamed datasets",
       "detail": "The paper refers to 'public datasets for Python and JavaScript' without naming the specific datasets, providing download links, or citing them. This makes the results completely unreproducible."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single point estimates with no error bars, confidence intervals, variance across runs, or significance tests despite claims of significant outperformance."
     },
     {
       "flag": "Missing critical experimental details",
       "detail": "Key details like dataset sizes, model architecture dimensions (embedding size, number of layers/heads), training epochs, and hardware are absent, making reproduction impossible."
     },
     {
       "flag": "Possible table formatting error",
       "detail": "Table 1 shows '8645%' for JavaScript MRR which appears to be a formatting error (likely 86.45%). This raises questions about the care taken in preparing the results."
     },
     {
       "flag": "Venue quality concern",
       "detail": "Published in 'Scientific Journal of Technology' (ISSN: 2688-8645), which does not appear to be a well-known peer-reviewed venue in software engineering or machine learning. The paper is very short (~5 pages) and lacks the methodological depth expected for the claims made."
     }
   ],
   "cited_papers": [
     {
       "title": "On the naturalness of software",
       "authors": ["A. Hindle", "E. T. Barr", "M. Gabel"],
       "year": 2016,
       "relevance": "Foundational work on treating code as natural language, which motivates code completion using language models."
     },
     {
       "title": "The hidden cost of code completion: Understanding the impact of the recommendation-list length on its efficiency",
       "authors": ["X. Jin", "F. Servant"],
       "year": 2018,
       "relevance": "Studies the practical impact of code completion tools on developer productivity."
     },
     {
       "title": "Learning from examples to improve code completion systems",
       "authors": ["M. Bruch", "M. Monperrus", "M. Mezini"],
       "year": 2009,
       "relevance": "Early work on ML-based code completion systems, relevant as a baseline approach."
     },
     {
       "title": "A unified multi-task learning model for ast-level and token-level code completion",
       "authors": ["F. Liu", "G. Li", "B. Wei", "X. Xia", "Z. Fu", "Z. Jin"],
       "year": 2022,
       "relevance": "Recent work on AST-based code completion using multi-task learning, directly comparable to this paper's approach."
     },
     {
       "title": "Probabilistic model for code with decision trees",
       "authors": ["V. Raychev", "P. Bielik", "M. Vechev"],
       "year": 2016,
       "relevance": "Probabilistic approach to code modeling that represents a baseline category for code completion."
     }
   ]
 }