ai-research-survey

scan.json (25920B)

---

 {
   "paper": {
     "title": "CCTEST: Testing and Repairing Code Completion Systems",
     "authors": [
       "Zongjie Li",
       "Chaozheng Wang",
       "Zhibo Liu",
       "Haoxuan Wang",
       "Dong Chen",
       "Shuai Wang",
       "Cuiyun Gao"
     ],
     "year": 2022,
     "venue": "ICSE 2023",
     "arxiv_id": "2208.08289",
     "doi": "10.1109/ICSE48619.2023.00110"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We have released CCTEST to facilitate further research' and provides a URL: https://sites.google.com/view/cctest-info (reference [12])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses two publicly available datasets: LeetCode solutions from a public GitHub repository (reference [6]) and the CodeSearchNet test split (reference [39]). The artifact page also references released data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Intel Xeon Platinum 8276 CPU, 256 GB memory, and 4 NVIDIA A100 GPUs' and that CCTEST is 'implemented in Python, with about 5k LOC' using tree-sitter for parsing, but no requirements.txt, Dockerfile, or detailed library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The artifact site is referenced but the paper itself does not contain a 'Reproducing Results' section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables V, VI, VIII, and IX are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CCTEST improves accuracy by 40% and 67% and compares systems, but no statistical significance tests (p-values, t-tests, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'the accuracy of code completion systems is notably increased by 40.25% and 67.43% with respect to BLEU score and Levenshtein edit similarity' (Table VIII provides per-system improvement ratios). The human study reports the average score change from 2.3188 to 3.1565 (36.12% gain)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 2,910 seed programs, 4,000 manually inspected samples, 60 samples for human evaluation, or 12 human evaluators. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as aggregate counts and percentages. No standard deviations, variance, or spread measures across seeds, models, or runs are reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares the original (unenhanced) code completion outputs as the baseline against CCTEST-enhanced outputs. Table VIII reports improvement ratios relative to the baseline performance for all 8 systems."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 8 code completion systems tested (Copilot, CodeParrot, GPT-Neo, GPT-J, CodeGen) were contemporary and actively used at the time of publication (2022-2023). The paper notes these 'represent the best systems available to the public.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table VII and Table IX provide per-PSC-transformation breakdowns showing the contribution of each of the 9 mutation schemes to both outlier detection (RQ2) and enhancement (RQ3). This functions as an ablation showing which components contribute to the results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: precision, recall, F1 for outlier detection (Table VI); BLEU score and Levenshtein edit similarity for enhancement (Table VIII); and a 1-5 human rating scale for the human study."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C includes a human study with 12 experts (4 industrial developers and 8 academic researchers) who rated 60 samples on a 1-5 scale, with sanity checks and Fleiss' Kappa reported (0.94)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The CodeSearchNet test split is explicitly used for evaluation (Sec. IV). LeetCode programs are separate from any training data. No model tuning is done on the evaluation data since CCTEST is a black-box testing framework with no training phase."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-system (8 systems) and per-dataset (LeetCode vs CodeSearchNet) in Tables V and VIII. Per-PSC-transformation breakdowns are in Tables VII and IX."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B discusses false positives in detail: 59.46% are from highly robust completion outputs, 26.13% from vague prompts with no mainstream results. False negatives are also measured and reported."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that IRR and RTF mutations have limited application scope and produce fewer outliers (Table VII). It also reports that Copilot has the most 'no response' failures (41 of 58). The discussion of false positives and false negatives constitutes negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '33,540 inputs that can trigger erroneous cases (with a true positive rate of 86%)' which matches Table V (T=9: 5912+27628=33540) and Table VI (precision 0.861). The '40% and 67%' improvement claims match Table VIII averages (40.20% BLEU, ~67% edit sim)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that PSC mutations 'trigger' erroneous outputs. The experimental design supports this: each mutation is a controlled single-variable manipulation of the prompt, and the consistency testing oracle cross-compares outputs from structure-consistent variants. This constitutes adequate controlled manipulation for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states CCTEST 'currently focuses on mutating Python code' (Sec. IV), acknowledges the scope is limited to Python, and notes 'we leave supporting other programming languages as one future work' (Sec. III). The discussion section addresses migration to other languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses multiple alternative explanations: that mutations may not be 'natural-looking', that the consistency oracle cannot catch aligned-yet-erroneous patterns, and that cross-system differential testing would be an alternative. The paper discusses why syntactic comparison and functionality checking are less appropriate oracles."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table III lists model names and parameter counts (e.g., 'CodeParrot-small 110M', 'GPT-Neo-125M', 'GPT-J 6B', 'Codegen-2B') but no specific version numbers, snapshot dates, or commit hashes. Copilot is marked with '?' for all details. Models are stated to be from Hugging Face but exact model card versions are not given."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "CCTEST does not use prompting in the LLM prompting sense (system prompts, instructions). It feeds code snippets directly to code completion systems as inputs. The actual code inputs are generated programmatically from LeetCode/CodeSearchNet and the mutation algorithms are fully described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we follow Copilot's interface to only choose the completion result with the highest confidence score' and 'For other tested models, we disable their sampling strategy' (Sec. IV). The threshold T is discussed with empirical evaluation across T=1,3,5,7,9. Token length bounds (32-2048) are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "CCTEST is not an agentic system with scaffolding. It is a testing framework that applies mutations and collects outputs from code completion APIs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV documents the preprocessing pipeline: programs are selected with token length between 32 and 2048, for CodeSearchNet only test split is used and duplicates by 'path' attribute are removed, programs are parsed with tree-sitter, and sanity checks determine which PSC transformations are feasible. Table II provides full statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI begins with 'Limitations and Threats to Validity' and provides substantive discussion covering construct validity, external validity, and multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: (1) the consistency oracle cannot guarantee correctness, only detect inconsistencies; (2) defects can be missed when all outputs share aligned-yet-erroneous patterns; (3) mutations may not be natural-looking (e.g., LocalVar1); (4) the framework is tested only on Python. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section III explicitly states 'We primarily target the erroneous code completion outputs, denoting stealthy logic bugs' and that 'CCTEST cannot guarantee that the repaired outputs become semantically correct.' The study scope is limited to Python, and migration to other languages is left as future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact is released at the referenced URL (reference [12]), and the seed datasets (LeetCode solutions, CodeSearchNet) are publicly available. This allows independent verification of the generated test inputs and outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes data collection in detail: 613 LeetCode programs from a specific GitHub repository (reference [6]), 2,297 CodeSearchNet programs from the test split, with token length filtering (32-2048) and deduplication by path attribute."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation, the paper states 'We invite twelve experts, including four industrial developers and eight academy researchers with expertise in LLMs' but does not describe how these specific participants were recruited, whether they were convenience-sampled, or whether the recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: seed programs collected and filtered by token length, parsed with tree-sitter, sanity checks for PSC applicability, mutation to generate 19,898 variants from 2,910 seeds, code completion output collection, then outlier detection and enhancement. Table II provides statistics at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states 'HKUST authors are supported in part by a RGC ECS grant under the contract 26206520.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: The Hong Kong University of Science and Technology, Harbin Institute of Technology (Shenzhen), and Swiss Federal Institute of Technology Lausanne. None of the authors are affiliated with the companies whose products are tested (GitHub/Microsoft, Salesforce, EleutherAI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Hong Kong Research Grants Council (RGC), a government research funding agency with no commercial interest in the performance of code completion systems."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models. It notes that the systems 'are advertised as being trained with millions or even billions of lines of code' but provides no temporal cutoff information."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LeetCode solutions and CodeSearchNet as test inputs, both of which are publicly available and could have been in the training data of the evaluated models. No discussion of potential train/test overlap is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode solutions and CodeSearchNet were published before the models were trained. The paper does not address whether these test programs appeared in the training data, which could affect the interpretation of 'consistency' since the models may have memorized these specific programs."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation study with 12 experts is not pre-registered. No link to any pre-registration is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The paper involves 12 human participants in an evaluation study but makes no mention of IRB or ethics board approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participants are described only as 'four industrial developers and eight academy researchers with expertise in LLMs.' No further demographics (experience level, years of experience, geographic distribution, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for participants are stated beyond 'expertise in LLMs.' It is unclear what qualified someone as having this expertise."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states '60 samples' were 'randomly select[ed]', 'five sanity-check test items randomly [inserted] into the questionnaire', and 'We evenly assigned 30 real samples with five SC to each participant, and ensured that each selected sample was examined by six participants.'"
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We provide two completion outputs for each seed program without specifying whether they are the original or the CCTEST's enhanced outputs,' indicating participants were blinded to the condition."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All participants passed the sanity check,' indicating no attrition from the initial 12 participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: 'it takes 35 GPU seconds to finish completing one prompt on average,' '2.7 CPU seconds to generate nine mutated prompts,' 'about 82 GPU seconds for code completion systems to infer and obtain the results in parallel,' and 'about 1.1 CPU seconds' for outlier detection and enhancement."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The hardware is specified: 'Intel Xeon Platinum 8276 CPU, 256 GB memory, and 4 NVIDIA A100 GPUs.' Per-prompt timing is provided, allowing total compute to be estimated from the 182,464 test inputs."
       }
     }
   },
   "claims": [
     {
       "claim": "CCTEST detected 33,540 inputs that trigger erroneous cases from eight LLM-based code completion systems with a true positive rate of 86%.",
       "evidence": "Table V reports 5,912 + 27,628 = 33,540 outliers at T=9. Table VI shows precision of 0.861 at T=9 based on manual inspection of 800 randomly sampled positive findings.",
       "supported": "strong"
     },
     {
       "claim": "CCTEST's enhancement improves code completion accuracy by 40.25% (BLEU) and 67.43% (edit similarity) on average.",
       "evidence": "Table VIII reports per-system improvement ratios. The average enhancement column shows 40.20% BLEU and approximately 62-72% edit similarity improvements across 8 systems and 2 datasets. The abstract rounds these numbers.",
       "supported": "moderate"
     },
     {
       "claim": "Human evaluators rated CCTEST-enhanced outputs 36.12% higher on average than original outputs.",
       "evidence": "Section V-C reports average human scores of 2.3188 (original) vs 3.1565 (enhanced) from 12 experts on 60 samples, with Fleiss' Kappa of 0.94.",
       "supported": "moderate"
     },
     {
       "claim": "All nine PSC transformation schemes contribute meaningfully to both outlier detection and enhancement.",
       "evidence": "Tables VII and IX show per-transformation distributions. All schemes have non-zero contributions, though IRR and RTF have smaller scope. This is presented across both datasets.",
       "supported": "strong"
     },
     {
       "claim": "97.8% of enhanced outputs were rated equal to or better than original outputs by human evaluators.",
       "evidence": "Section V-C states 'Respondents believe that for 2.2% of the cases, the enhanced completion outputs look worse than the original outputs, while the remaining 97.8% treat the enhanced outputs equal to or better.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CCTEST introduces a metamorphic testing framework for code completion systems using program structure-consistent (PSC) mutations. Testing 8 code completion systems with ~18K seed inputs, it found 33,540 defect-triggering inputs with 86% precision at T=9. The enhancement strategy, which selects the output closest to the average appearance, improved BLEU scores by ~40% and edit similarity by ~67% on average. A human evaluation with 12 experts confirmed that 97.8% of enhanced outputs were rated equal to or better than originals, with high inter-rater agreement (Fleiss' Kappa = 0.94).",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The paper uses LeetCode solutions and CodeSearchNet as test inputs without discussing whether these appeared in the training data of the evaluated models. Since these are widely-used public datasets and the models were trained on open-source code, memorization could affect the consistency metrics. A model might produce inconsistent outputs on memorized programs specifically because mutations break memorization patterns."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Enhancement improvements of 40% (BLEU) and 67% (edit similarity) are reported without any significance testing. The comparison between original and enhanced outputs lacks statistical validation despite being the core claim."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "The human evaluation uses only 60 samples (30 per participant) and 12 evaluators with no power analysis. While Fleiss' Kappa is high, the sample size is small relative to the 33,540 detected defects and 182,464 total test cases."
     },
     {
       "flag": "No exact model versions",
       "detail": "Models are identified by name and parameter count only (e.g., 'GPT-J 6B', 'Codegen-2B-mono'). No Hugging Face model card version hashes or download dates are provided, making exact replication difficult as model weights on Hugging Face can be updated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code completion evaluation and Codex model, directly relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "A conversational paradigm for program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Introduces CodeGen models evaluated in this paper, relevant to understanding LLM-based code completion systems."
     },
     {
       "title": "An empirical evaluation of GitHub Copilot's code suggestions",
       "authors": ["Nhan Nguyen", "Sarah Nadi"],
       "year": 2022,
       "relevance": "Empirically evaluates Copilot using LeetCode programs, directly relevant to assessing code completion system quality."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David H. Choi"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Demonstrates LLM code generation at competition level, relevant to understanding the capabilities and evaluation of code generation models."
     },
     {
       "title": "Is GitHub Copilot a substitute for human pair-programming? An empirical study",
       "authors": ["Saki Imai"],
       "year": 2022,
       "relevance": "Empirical study on Copilot's effectiveness as a pair programming substitute, relevant to AI-assisted programming productivity."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Foundational pre-trained model for code understanding, relevant to the broader landscape of neural code models."
     },
     {
       "title": "Can OpenAI Codex and other large language models help us fix security bugs?",
       "authors": ["Hammond Pearce", "Benjamin Tan"],
       "year": 2021,
       "arxiv_id": "2112.02125",
       "relevance": "Evaluates LLMs for security bug fixing, relevant to AI-assisted programming and LLM code quality assessment."
     },
     {
       "title": "Adversarial robustness for code",
       "authors": ["Pavol Bielik", "Martin Vechev"],
       "year": 2020,
       "relevance": "Addresses adversarial robustness of code models, directly relevant to testing and evaluating neural code systems."
     },
     {
       "title": "Semantic robustness of models of source code",
       "authors": ["Jordan Henke", "Goutham Ramakrishnan"],
       "year": 2022,
       "relevance": "Studies semantic robustness of source code models under transformations, directly relevant to testing code intelligence systems."
     },
     {
       "title": "CodeSearchNet challenge: Evaluating the state of semantic code search",
       "authors": ["Hamel Husain", "Ho-Hsiang Wu"],
       "year": 2019,
       "arxiv_id": "1909.09436",
       "relevance": "Introduces the CodeSearchNet dataset used as a test corpus in this paper, relevant as a standard benchmark for code understanding."
     },
     {
       "title": "Automatic testing and improvement of machine translation",
       "authors": ["Zeyu Sun", "Jie M Zhang", "Mark Harman"],
       "year": 2020,
       "relevance": "Applies metamorphic testing to machine translation, directly inspiring CCTEST's approach to testing code completion."
     }
   ]
 }