ai-research-survey

scan.json (19783B)

---

 {
   "paper": {
     "title": "Evaluating the Code Quality of AI-Assisted Code Generation Tools: An Empirical Study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT",
     "authors": ["Burak Yetiştiren", "Işık Özsoy", "Miray Ayerdem", "Eray Tüzün"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2304.10778"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Repository URL provided: https://github.com/mirayayerdem/Github-Copilot-Amazon-Whisperer-ChatGPT. Also references a reproduction package with experiment results spreadsheet."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available HumanEval dataset and provide their experiment results in the GitHub repository. The reproduction package spreadsheet is linked in Section 3.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "They mention Python 3.10.10 interpreter and SonarQube but do not provide requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental workflow is described at a high level (Section 3.2) but no README with commands or scripts to replicate experiments is mentioned."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates are reported (e.g., 65.2% correctness for ChatGPT). No confidence intervals or error bars on any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares three tools and claims ChatGPT is 'the most successful' but no statistical significance tests are used to support comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw percentages and differences are reported but no formal effect sizes (Cohen's d, odds ratios, etc.) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample is the 164 HumanEval problems. No justification is given for why this sample size is adequate for the claims made, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results only. The paper acknowledges nondeterministic behavior of the tools (Section 6.2) but does not report variance across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three tools are compared against each other (GitHub Copilot, Amazon CodeWhisperer, ChatGPT), and older versions serve as baselines for RQ4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The three tools evaluated were all current at the time of the study (January 2023 versions)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 removes docstrings and RQ3 replaces function names with 'foo', effectively ablating input components to measure their contribution to code generation quality."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: Code Validity, Code Correctness, Code Security, Code Reliability, and Code Maintainability (Section 3.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of code quality. All evaluation is automated via unit tests and SonarQube. Human judgment of readability or usefulness of generated code is absent."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates code generation tools on a fixed benchmark, not training a model with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per tool, per experiment type (original, only function names, dummy names), and partially correct generations are further categorized into correctness score intervals (Figures 7, 9, 11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple code listings show specific failure cases (Listings 2-6 for invalid code, Listing 9 for complex prompts where Copilot outputs 'pass'). Section 5.1 discusses causes of invalid code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 reports that 17 of 47 correct answers from the old Copilot version became incorrect in the new version. Also reports regressions in CodeWhisperer's new version (16 of 40 correct answers became partial/incorrect)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (ChatGPT 65.2%, Copilot 46.3%, CodeWhisperer 31.1% correctness; improvement rates; technical debt figures) are all supported by results in Sections 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation experiments (RQ2 removing docstrings, RQ3 using dummy names) use controlled single-variable manipulation to support claims about the effect of input quality on code generation. The design is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Evaluating the Code Quality of AI-Assisted Code Generation Tools' broadly, but results are limited to Python on HumanEval's 164 problems. While Section 6.4 notes this limitation, the title and conclusions do not sufficiently bound the claims to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 6) discusses methodological issues but does not consider alternative explanations for observed results (e.g., whether HumanEval problems are in training data could explain ChatGPT's high performance)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GitHub Copilot versions are given (v1.7.4421 and v1.70.8099) and ChatGPT is identified as '9 Jan 23 Version', but the underlying model versions (which GPT model, which Codex version) are not specified. For CodeWhisperer, only the date is given as AWS does not provide version numbers. No API model identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompts are the HumanEval function signatures and docstrings, which are shown in Figure 1. For ChatGPT, they state the instruction: 'Generate code using the prompts I will provide' (Section 6.2). The full HumanEval dataset is publicly available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the three tools. The tools were used with default settings but this is not explicitly stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party tools (Copilot, CodeWhisperer, ChatGPT) as black boxes. The authors cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Figure 3 describe the pipeline: extracting problems from HumanEval, saving prompts and tests as separate files, generating solutions, then running assessments. The process is documented step by step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' is a dedicated section with four subsections (conclusion, internal, construct, external validity)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Multiple specific threats are discussed: nondeterministic code generation (Section 6.2), different code generation methods (auto vs key combination), unequal time intervals between version comparisons (13 months vs 2 months), and IDE dependency."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly states limitations: dependency on HumanEval dataset, limitation to Python, IDE dependency, and problem coverage limited to 164 problems."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw experiment results are available in the reproduction package spreadsheet linked in Section 3.2 and the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes code generation was done manually using VS Code for Copilot/CodeWhisperer and browser for ChatGPT. The process for each tool is detailed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is the standard HumanEval benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides a step-by-step workflow from problem extraction through code generation to assessment. Each stage is documented in Section 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section mentioning grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Bilkent University, clearly listed. They are not affiliated with any of the companies whose tools are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding disclosure is not the same as absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes an explicit statement: 'Conflict of Interests: The authors declare no conflicts of interest in relation to this article.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models. ChatGPT's training is vaguely described as 'completed its training process early in 2022' but no cutoff date for Copilot's or CodeWhisperer's models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. All three tools were likely trained on data including HumanEval problems, but no discussion of potential train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has been publicly available since July 2021 and the models were trained after that date. The paper does not address benchmark contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time for the code generation process is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of computational resources, time spent, or total budget for running the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT generates correct code 65.2% of the time, GitHub Copilot 46.3%, and Amazon CodeWhisperer 31.1% on HumanEval.",
       "evidence": "Section 4.2, Figures 5, 6, 8, 10. Results based on single-run evaluation of 164 HumanEval problems.",
       "supported": "moderate"
     },
     {
       "claim": "Removing docstrings significantly reduces code correctness for all three tools (drops of 26.2%, 16.5%, and 43.2% respectively).",
       "evidence": "Section 4.4, Table 4. Controlled experiment removing docstrings from HumanEval prompts.",
       "supported": "moderate"
     },
     {
       "claim": "Using dummy function names ('foo') causes smaller correctness drops (4.2%, 3.7%, 3.6%) than removing docstrings.",
       "evidence": "Section 4.5, Table 4. Controlled experiment replacing function names with 'foo'.",
       "supported": "moderate"
     },
     {
       "claim": "GitHub Copilot's new version showed 62% improvement in passed unit tests over the old version; CodeWhisperer showed 28% improvement.",
       "evidence": "Section 4.6, Figures 23-26. Comparison of Copilot v1.7.4421 vs v1.70.8099, and CodeWhisperer Nov '22 vs Jan '23.",
       "supported": "weak"
     },
     {
       "claim": "All code generation tools generate valid code approximately 90% of the time.",
       "evidence": "Section 4.1, Figure 4. Copilot 91.5%, CodeWhisperer 90.2%, ChatGPT 93.3%.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ChatGPT outperforms GitHub Copilot and Amazon CodeWhisperer on HumanEval with 65.2% code correctness vs 46.3% and 31.1% respectively. Providing docstrings in prompts is more important than meaningful function names for code generation quality. Both GitHub Copilot and CodeWhisperer showed improvements across versions. All three tools generate valid (syntactically correct) code approximately 90% of the time, but code maintainability issues (code smells) are common across all generators.",
   "red_flags": [
     {
       "flag": "No statistical tests",
       "detail": "Comparative claims between three tools are made based solely on comparing raw percentages without any significance tests, despite acknowledged nondeterminism in the tools."
     },
     {
       "flag": "Single-run evaluation with nondeterministic tools",
       "detail": "Each tool was run once per problem despite the paper acknowledging nondeterministic behavior (Section 6.2). No variance or confidence intervals are reported, making it impossible to know if observed differences are meaningful."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval was published in July 2021 and all three models were trained on data from after that date. ChatGPT's superior performance could partly be explained by memorization of HumanEval solutions, but this is not discussed."
     },
     {
       "flag": "Unequal comparison intervals for RQ4",
       "detail": "GitHub Copilot versions are 13 months apart while CodeWhisperer versions are only 2 months apart, making improvement rate comparisons misleading."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "doi": "10.48550/ARXIV.2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex model, foundational to this study's evaluation methodology."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce"],
       "year": 2021,
       "relevance": "Evaluates security vulnerabilities in Copilot-generated code, finding 40% of programs were vulnerable."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["Priyan Vaithilingam"],
       "year": 2022,
       "relevance": "Within-subjects user study of GitHub Copilot finding no significant improvement in speed/success rate."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Nguyen", "Nadi"],
       "year": 2022,
       "relevance": "Evaluates Copilot on LeetCode problems across four programming languages for code correctness and understandability."
     },
     {
       "title": "Robustness of AI-assisted code generation: a study on GitHub Copilot",
       "authors": ["Mastropaolo"],
       "year": 2023,
       "relevance": "Studies effect of semantic-preserving input changes on Copilot's code generation, finding 46% result in different recommendations."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of ChatGPT",
       "authors": ["Sobania"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's bug fixing capability on QuixBugs benchmark, comparing to Codex and standard APR approaches."
     },
     {
       "title": "ChatGPT and software testing education: Promises & perils",
       "authors": ["Jalil", "Rafi", "LaToza", "Moran", "Lam"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT on software testing curriculum questions, finding it would fail a software testing course."
     },
     {
       "title": "An empirical evaluation of GitHub Copilot's code suggestions",
       "authors": ["Sobania"],
       "year": 2022,
       "relevance": "Evaluates Copilot code correctness and compares to genetic programming-based code generators."
     }
   ]
 }