ai-research-survey

scan.json (25845B)

---

 {
   "paper": {
     "title": "A case study on the transformative potential of AI in software engineering on LeetCode and ChatGPT",
     "authors": ["Manuel Merkel", "Jens Dörpinghaus"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.03639",
     "doi": "10.48550/arXiv.2501.03639"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Zenodo repository with scripts and data: https://doi.org/10.5281/zenodo.13881451. This is stated in both the 'Code availability' and 'Data availability' sections."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released via the same Zenodo repository (https://doi.org/10.5281/zenodo.13881451). The 'Data availability' section explicitly states this."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python scripts, the requests library, Selenium, SonarQube, PostgreSQL, and various other tools, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the Zenodo repository contains code and data, the paper itself does not provide step-by-step reproduction instructions. The methodology is described in prose but there are no specific commands or README instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports p-values and effect sizes but does not report confidence intervals or error bars for its main results. Tables show means and medians without uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Mann-Whitney U tests (H1, H2) and Wilcoxon tests (H3, H4) for hypothesis testing, with p-values reported for each hypothesis. It also applies Bonferroni correction for multiple comparisons (adjusted significance level of 1.25 × 10^-2)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d is reported for each accepted hypothesis: d=0.65 (medium) for code quality, d=0.14 (small) for understandability, d=0.44 (small) for runtime. This provides context for the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The sample size is determined by what was available on LeetCode (2,321 problems, 57,238 user solutions) rather than by any a priori calculation."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Box plots showing IQR are provided for all four hypotheses (Figures 5-8). Medians and means are reported in tables. The distribution characteristics are visually documented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares GPT-4o generated code against human-written code from LeetCode users across all four metrics, which serves as the baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is between GPT-4o (the latest OpenAI model at the time, May 2024) and current LeetCode user solutions. The related work section also contextualizes against recent 2024 studies."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The study compares human vs. AI code quality; there is no multi-component system to ablate. The study measures four independent metrics rather than testing components of a proposed system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: code smells per kLOC (code quality), cognitive complexity per kLOC (understandability), runtime rank (time behaviour), and memory usage rank (resource utilisation)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of code quality is included. All evaluation is automated through SonarQube metrics and LeetCode platform metrics. Given that claims are about code quality and understandability, human evaluation of these subjective qualities would be relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training/evaluation study. The study analyzes all available LeetCode problems; there is no model tuning involved that would require train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (Easy, Medium, Hard) in Tables 4-7, and also by before/after the training cutoff date. Additional correlation analyses by problem age and difficulty are reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses GPT-4o's failure to solve 235 problems (10.12% invalid), the drop in performance for after-problems (51.94% vs 93.38%), and the failure of the memory usage hypothesis (H3). The heatmap in Figure 4 visualizes failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "H3 (resource utilisation) is reported as not accepted -- GPT-4o does not outperform humans on memory usage. The paper also reports that for after-problems, the code understandability difference was not significant (p=0.43). These are genuine negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that GPT-4o produces 'significantly lower values across all three metrics' (quality, understandability, runtime) vs. humans, that memory usage showed no significant improvement, and that GPT-4o struggled with problems outside training data. All of these are supported by the hypothesis tests in Section 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about training data influencing GPT-4o's quality, and the title implies AI has 'transformative potential.' The study design is observational/correlational -- it compares two groups (human vs. AI code) without controlling for confounds such as problem selection bias, developer experience levels, or the competitive incentive structure of LeetCode."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broad 'transformative potential of AI in software engineering' but results are limited to Python solutions on LeetCode coding problems using GPT-4o. The paper does not adequately bound generalizations: LeetCode problems are algorithmic puzzles, not representative of real-world software engineering. Section 6 discusses future work extending to other languages and LLMs, implicitly acknowledging limited scope, but the title and abstract overreach."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses several alternative explanations: LeetCode users may be inexperienced developers (Section 5.1), the competitive nature of LeetCode may bias memory usage results (Section 5.2), training data may prioritize runtime over memory (Section 5.2), and problem difficulty correlates with developer expertise (Section 5.1 with Spearman correlation test)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-4o-2024-05-13' as the exact model version used (Section 3.1.3, Appendix B), with the knowledge cutoff date of October 2023 stated explicitly."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts are shown in Figure 3a (first prompt) and Figure 3b (error-fixing prompt). The prompt structure is described in detail in Appendix B with all four components. While not every problem-specific prompt is shown, the template with its structure and an example with actual fill values is provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: temperature = 1 (default), n = 1 output, maximum tokens = 4,096. These API parameters are explicitly documented."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-round fixing approach (up to 5 retry attempts) is described in detail in Section 3.1.3 and Appendix B, including how error information from LeetCode is incorporated into subsequent prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is extensively documented: problem filtering criteria (Appendix A), code extraction from Markdown posts (Appendix C), import handling, code validation via AST parsing, and removal of non-Python/invalid solutions. Counts at each stage are provided (278,397 posts -> 70,261 Python -> 57,238 valid)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The discussion (Section 5) touches on some limitations embedded within the discussion of results, and the conclusion mentions future work, but there is no dedicated section for limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Though not in a dedicated section, the paper discusses specific threats: LeetCode metrics for runtime may not closely align with local times (Section 3.2.3, citing Choudhuri et al.), developer experience levels on LeetCode may skew results (Section 5.1), the small after-problems sample (107 cases, Section 5.1), and the competitive incentive structure of LeetCode affecting memory optimization (Section 5.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It mentions future work (other languages, other LLMs) in Section 6 but does not clearly delineate that results apply only to Python, only to LeetCode algorithmic problems, only to GPT-4o, and should not be generalized to real-world software engineering."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw data is available via the Zenodo repository (https://doi.org/10.5281/zenodo.13881451), as stated in the Data availability section."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described in extensive detail across Sections 3.1-3.2 and Appendices A-C, including web scraping via GraphQL queries, LeetCode API interaction, SonarQube analysis, and the time period (March-August 2024)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study mines existing LeetCode user posts and generates code via API. There are no human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Figure 1 and throughout Section 3 and Appendices: 2,992 problems -> 2,321 after filtering -> code generation (2,086 valid) and user scraping (278,397 posts -> 70,261 Python -> 57,238 valid). Each stage has explicit counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Declarations section states: 'This research received no external funding. This article was funded by the Open Access Publication Fund of the Federal Institute for Vocational Education and Training (BIBB), Bonn.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Stuttgart, BIBB, University of Koblenz, and Linnaeus University. Neither author is affiliated with OpenAI or any AI company whose product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (BIBB) funded only open access publication, not the research itself. BIBB is a vocational education institute with no financial stake in GPT-4o performance outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Declarations section includes 'Conflict of interest/Competing interests: The authors declare no conflict of interest.' This is an explicit competing interests statement."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The training data cutoff for GPT-4o is stated as October 2023 (Section 3.1.3, Appendix B), with a specific cutoff date of 1 October 2023 used for analysis (Section 4.1)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses train/test overlap: problems are divided into 'before-problems' (in training data, N=2,115) and 'after-problems' (post-cutoff, N=206). Section 4.1 and Table 7 analyze this split. The heatmap in Figure 4 visualizes the performance drop near the cutoff."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5 and Appendix A explicitly discuss contamination: 'some of the GenAIs have undergone training on code sourced from GitHub, there is a possibility of data contamination.' The study deliberately includes both before and after problems rather than excluding contaminated ones, and reports differential performance (93.38% vs 51.94% solve rates)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines existing LeetCode posts and generates code via API."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Declarations section states 'Ethics approval and consent to participate: Not applicable.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. LeetCode users' posts are mined but the users are not participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria for LeetCode problems and posts are documented, but this is data filtering, not human subject selection."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions to randomize."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report the cost of calling the GPT-4o API for 3,676 prompts. Token counts are mentioned briefly (e.g., 451 tokens for the Two Sum example) but total API cost is not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total compute budget, API costs, or hardware used for the SonarQube analysis of 59,324 files. The study required significant compute (SonarQube with PostgreSQL, Selenium automation, OpenAI API) but this is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o produces code with significantly fewer code smells per kLOC than human developers on LeetCode (median 76.92 vs 104.17).",
       "evidence": "Mann-Whitney U test, p = 2.13 × 10^-89, Cohen's d = 0.65 (medium effect). Based on 2,082 problems with 2,082 generated and 55,392 user solutions (Section 4.2, Table 8).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o produces code with significantly lower cognitive complexity per kLOC than human developers (median 375 vs 405.29).",
       "evidence": "Mann-Whitney U test, p = 4.01 × 10^-6, Cohen's d = 0.14 (small effect). However, for after-problems the p-value was 0.43 (not significant) (Section 4.3, Table 9).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o generated code runs significantly faster than human code on LeetCode (median runtime rank 57.18).",
       "evidence": "Wilcoxon test, p = 1.17 × 10^-22, Cohen's d = 0.44 (small effect). Based on 2,086 problems (Section 4.5, Table 12).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o does not produce code that uses less memory than human developers on LeetCode.",
       "evidence": "Wilcoxon test, p = 5.85 × 10^-2, above adjusted significance level. Memory rank median was 48.16, below 50th percentile (Section 4.4, Table 10).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o has a limited capacity to generalise to problems not in its training data, solving only 51.94% of after-problems vs 93.38% of before-problems.",
       "evidence": "Table 7 shows 107/206 after-problems solved vs 1,975/2,115 before-problems. The heatmap in Figure 4 visualizes the performance drop near the cutoff.",
       "supported": "strong"
     },
     {
       "claim": "The overall valid solution rate for GPT-4o across all LeetCode problems was 89.88%.",
       "evidence": "2,086 valid solutions out of 2,321 problems within 5 attempts (Section 3.1.3).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "GPT-4o generates Python code with significantly fewer code smells (medium effect, d=0.65) and lower cognitive complexity (small effect, d=0.14) than human LeetCode solutions. GPT-4o code also runs faster (small effect, d=0.44) but does not use less memory than human code. GPT-4o's problem-solving ability drops sharply for problems outside its training data (51.94% vs 93.38% success rate), demonstrating limited generalization.",
   "red_flags": [
     {
       "flag": "Title overstates scope",
       "detail": "The title claims 'transformative potential of AI in software engineering' but the study only evaluates GPT-4o on Python LeetCode algorithmic puzzles, which are not representative of real-world software engineering tasks."
     },
     {
       "flag": "Non-representative baseline population",
       "detail": "LeetCode users include many beginners and interview preparers, making them a potentially weak baseline. The paper acknowledges this (Section 5.1) but still draws conclusions about 'humans' vs 'AI' code quality. A comparison against professional production code would be more meaningful."
     },
     {
       "flag": "LeetCode metrics unreliability acknowledged but used anyway",
       "detail": "The paper cites Choudhuri et al. noting that LeetCode runtime/memory metrics 'do not consistently align closely with the times recorded locally' and have 'a higher degree of variance' (Section 3.2.3), yet these metrics are central to two of the four hypotheses."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "Despite being a study with multiple threats to validity (non-representative sample, unreliable metrics, potential contamination), there is no dedicated limitations or threats-to-validity section."
     },
     {
       "flag": "After-problems sample too small for strong claims",
       "detail": "Only 107 after-problems were solved by GPT-4o (out of 206), yet the paper draws conclusions about generalization capacity. The code understandability result was not significant for after-problems (p=0.43), suggesting the main findings may not hold for uncontaminated problems."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark widely used for evaluating LLM code generation capabilities."
     },
     {
       "title": "No need to lift a finger anymore? Assessing the quality of code generation by ChatGPT",
       "authors": ["Zhijie Liu", "Yutian Tang", "Xiapu Luo", "Yuming Zhou", "Liang Feng Zhang"],
       "year": 2024,
       "doi": "10.1109/TSE.2024.3392499",
       "relevance": "Large-scale study of ChatGPT code quality on LeetCode, examines correctness, complexity and security with multi-round fixing."
     },
     {
       "title": "An empirical study of code smells in transformer-based code generation techniques",
       "authors": ["Mohammed Latif Siddiq"],
       "year": 2022,
       "doi": "10.1109/SCAM55253.2022.00014",
       "relevance": "Analyzes code quality issues in LLM-generated code using static analysis, finding smells propagate from training data."
     },
     {
       "title": "An empirical evaluation of GitHub Copilot's code suggestions",
       "authors": ["Nhan Nguyen", "Sarah Nadi"],
       "year": 2022,
       "doi": "10.1145/3524842.3528470",
       "relevance": "Evaluates Copilot code correctness and understandability on LeetCode problems across four programming languages."
     },
     {
       "title": "Refining ChatGPT-generated code: Characterizing and mitigating code quality issues",
       "authors": ["Yue Liu"],
       "year": 2024,
       "doi": "10.1145/3643674",
       "relevance": "Studies ChatGPT code quality issues on 2,033 LeetCode tasks and evaluates self-repair capabilities."
     },
     {
       "title": "A performance study of LLM-generated code on LeetCode",
       "authors": ["Tristan Coignion", "Clément Quinton", "Romain Rouvoy"],
       "year": 2024,
       "doi": "10.1145/3661167.3661221",
       "relevance": "Compares runtime and memory efficiency of 18 LLMs on LeetCode, finding LLM code is on average more efficient than human code."
     },
     {
       "title": "Where Are Large Language Models for Code Generation on GitHub?",
       "authors": ["Xiang Yu"],
       "year": 2024,
       "arxiv_id": "2406.19544",
       "relevance": "Large-scale study comparing LLM-generated code found on GitHub with human-written code using SonarQube metrics."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad"],
       "year": 2022,
       "doi": "10.1109/SP46214.2022.9833571",
       "relevance": "Early study evaluating security vulnerabilities in AI-generated code from GitHub Copilot."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2023,
       "relevance": "Evaluates 26 LLMs on HumanEval+ with GPT-4 achieving 76.2%, relevant to LLM code generation capability assessment."
     },
     {
       "title": "Programming with ChatGPT: How far can we go?",
       "authors": ["Alessio Bucaioni"],
       "year": 2024,
       "doi": "10.1016/j.mlwa.2024.100526",
       "relevance": "Studies ChatGPT code generation for Java and C++ on LeetCode, finding opposite efficiency results to some other studies."
     },
     {
       "title": "Stop Uploading Test Data in Plain Text: Practical Strategies for Mitigating Data Contamination by Evaluation Benchmarks",
       "authors": ["Alon Jacovi"],
       "year": 2023,
       "arxiv_id": "2305.10160",
       "relevance": "Addresses benchmark contamination strategies, directly relevant to evaluation methodology for LLM code generation."
     },
     {
       "title": "Program code generation with generative AIs",
       "authors": ["Bulat Idrisov", "Tim Schlippe"],
       "year": 2024,
       "doi": "10.3390/a17020062",
       "relevance": "Compares 6 GenAIs on LeetCode problems created after training cutoff, finding Copilot most effective at 50%."
     }
   ]
 }