ai-research-survey

scan.json (28549B)

---

 {
   "paper": {
     "title": "An evaluation of LLM code generation capabilities through graded exercises",
     "authors": ["Álvaro Barbero Jiménez"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.16292",
     "doi": "10.48550/arXiv.2410.16292"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Evaluating GPT-4o-mini on ~14,346 Codewars coding challenges across 8 languages and 8 difficulty levels, the study finds the LLM outperforms humans on easy tasks but fails completely on the hardest exercises (kyu 1-2). A surrogate SHAP analysis attributes 46.6% of performance to task difficulty, 37.4% to solution leakage, and 16% to programming language. Performance drops dramatically for unpopular and legacy languages (especially COBOL), and newer challenges are harder for the LLM, suggesting current benchmarks overestimate LLM coding capability due to data contamination.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 (Limitations) explicitly states: 'we have decided to release neither the code of the developed botnet, nor the database of solutions proposed by the LLM' due to Codewars terms of use concerns."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The solutions database and evaluation results are explicitly withheld (Section 5). While Codewars katas are publicly available, the paper's collected dataset of LLM solutions and outcomes is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python, Selenium, and the OpenAI API but provides no requirements.txt, library versions, or environment specification beyond naming the tools."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The code is explicitly withheld and no alternative reproduction path is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (pass rates, Codewars scores). No confidence intervals or error bars appear in any figure or table."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (LLM vs humans, across languages, across kyu levels) but no statistical significance tests are performed. Differences are stated from visual inspection of charts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The SHAP analysis decomposes performance into difficulty (46.6%), leakage (37.4%), and language (16%) with full context. Per-language Codewars scores are reported (e.g., Python 36.1) against reference levels (87 for solving all kyu ≥4). Magnitudes are interpretable throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study uses ~14,346 katas but never justifies this sample size or discusses whether it is adequate for the claims made, particularly for rare kata-language combinations."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each kata appears to be attempted once with a single LLM call. No variance across runs, seeds, or repeated attempts is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human performance from Codewars community statistics serves as a baseline throughout. Figure 7 directly compares LLM vs human success rates by difficulty level, and Figure 11 compares Codewars scores over time."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The human baseline comes from current Codewars community statistics at the time of the study, making it contemporary and representative of human developer performance."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system under evaluation is a single LLM (GPT-4o-mini) with a fixed prompt. There are no components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses both raw pass rates (percentage of katas solved, Figures 7-8) and a weighted Codewars score metric (Equation 1, Figure 9) that emphasizes harder tasks. Timeout rates are also reported separately."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via Codewars unit tests (pass/fail). No human evaluation of generated code quality, readability, or correctness beyond test passing is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Codewars uses a two-tier testing system: public unit tests visible to the solver, plus hidden tests used for final verification (Figure 1). The LLM only sees public tests; final evaluation uses hidden tests, functioning as a held-out set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns by difficulty rank (8 levels, Figures 7-8), programming language (8 languages, Figures 8-9), and publication date (Figure 11) are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses COBOL's 'catastrophic results' with syntax errors, the complete failure on kyu 1-2 exercises, and timeout failures for JavaScript/Python on harder problems (yellow bars in Figure 8)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the LLM fails completely on kyu 1-2 exercises (0% success), performs poorly on legacy languages, and that performance degrades significantly on newer challenges. These are central findings, not hidden."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about correlation with task difficulty, language popularity, publication date, and the 46.6%/37.4%/16% decomposition are all directly supported by results in Sections 4.1-4.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states '37.4% seems to be related to leakage' and uses 'attributed to' language. The surrogate model (Section 4.5) provides correlational evidence only — SHAP values on a 74.88%-accurate linear SVM do not establish causation. The study design is observational with no causal identification strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only GPT-4o-mini on Codewars katas but generalizes broadly. The conclusions state 'current evaluations in the literature of the performance of state-of-the-art LLMs are, quite probably, overestimates of their real skill' — a claim about all LLMs based on one model on one platform."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly considers two competing hypotheses for the age effect: changed difficulty criteria vs. solution leakage. Section 4.5 addresses multicollinearity and acknowledges that features encode overlapping factors (e.g., easy katas have more completions AND more solutions available)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Limitations) explicitly distinguishes between what was measured (solving programming exercises) and what broader claims this supports: 'actual work as a software developer involves not just solving programming challenges, but also writing documentation, finding and reporting bugs...' The gap between proxy and outcome is clearly acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 specifies 'GPT-4o-mini (version 2024-07-18)' — an exact version with snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides the full system prompt and user prompt with all text. The placeholders (%LANGUAGE%, %DESCRIPTION%, %SOLUTION_TEMPLATE%, %UNIT_TESTS%) are filled from publicly available Codewars kata content, making the prompts fully reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM inference hyperparameters are reported — temperature, top-p, max tokens, and other API parameters that affect output quality are not mentioned anywhere."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLM receives a single prompt and produces a single response. The Selenium bot infrastructure is evaluation tooling, not LLM scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the data collection from Codewars, the 8 language selection rationale, that 57 katas (0.4%) were discarded due to regex/Selenium incompatibilities, and the resulting distribution across languages and difficulty levels (Figure 5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and contains substantive discussion of four specific limitations spanning multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: coverage limited to programming exercises (not full dev work), reproducibility constraints from Codewars ToS, absence of human-in-the-loop refinement, and surrogate model accuracy of only 74.88% introducing noise in the explanatory analysis."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states the evaluation 'only covers a small spectrum of this wide range of tasks' and enumerates what is excluded: documentation, bug finding, system design, requirements gathering, team collaboration, and roadmap definition."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (kata solutions, pass/fail results) is explicitly withheld per Section 5: 'we have decided to release neither the code of the developed botnet, nor the database of solutions proposed by the LLM.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Figure 3 describe the Selenium bot system, interaction with the Codewars API and OpenAI API, the account creation/deletion procedure, and responsible crawling practices."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is Codewars, a public coding challenge platform. The kata selection is the full set available for 8 languages."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 diagrams the full pipeline: kata downloader bot → solution generator bot (via OpenAI API) → kata attempter bot → result verification. Exclusion criteria (57 regex-incompatible katas) are documented with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section thanks the Codewars community and IIC colleagues but mentions no grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation with Instituto de Ingeniería del Conocimiento and Universidad Autónoma de Madrid is clearly stated on the title page. The author evaluates a third-party product (OpenAI's GPT-4o-mini), not their own."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The work appears to be unfunded academic research from a solo university-affiliated researcher."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The model version is specified (GPT-4o-mini 2024-07-18) but the training data cutoff date is not stated. Without this, temporal contamination cannot be precisely assessed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is a central theme. Section 4.4 discusses how ~38,500 public GitHub repos contain Codewars solutions, and the SHAP analysis (Section 4.5) attributes 37.4% of performance to leakage-related features. The paper also cites similar concerns with APPS and SWE-bench."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is the paper's core contribution. Section 4.4 discusses solution availability in public repos, Section 4.5 quantifies contamination impact via surrogate model analysis, and Section 6 proposes temporal evaluation as mitigation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is fully automated LLM-on-benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates an LLM on coding challenges."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Human performance data comes from aggregate Codewars community statistics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes GPT-4o-mini is 'approximately 30 times cheaper' than GPT-4o (Section 3) but never reports the actual API cost for the ~14,346 kata attempts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is reported — neither total API spend, wall-clock time for the full evaluation, nor hardware used for the bot infrastructure."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each kata appears to be attempted a single time. No multiple seeds or repeated sampling is performed, despite LLM outputs being stochastic."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper never explicitly states how many attempts per kata were made. It appears to be a single attempt but this is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For the LLM evaluation, API parameters (temperature, sampling) are not reported, let alone searched. The surrogate model hyperparameters are reported (Table 1) but the main LLM evaluation hyperparameters are absent."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For the surrogate model, 5-fold cross-validation is used to select the best configuration (Section 4.5, Table 1). The linear SVM is chosen over XGBoost because it produces equivalent accuracy (74.88% vs 74.91%) with greater interpretability."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so correction for multiple comparisons is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors build the evaluation framework and surrogate model but do not discuss how their design choices (feature selection, model choice, clustering of SHAP features into difficulty/leakage/language groups) could bias the results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only GPT-4o-mini, choosing it over GPT-4o for cost (30x cheaper). No performance-compute comparison is made, despite this being directly relevant to interpreting the results."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "This is a central theme. The paper extensively discusses whether coding benchmarks (Codewars, HumanEval, APPS, SWE-bench) actually measure LLM coding capability vs. memorization. Section 1.4 reviews construct validity issues and Section 6 argues current evaluations overestimate real skill."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The LLM receives a single prompt and produces a single response; no agentic workflow or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 analyzes how kata publication date correlates with LLM success, finding older katas with more time for solutions to spread are easier for the LLM. The SHAP analysis includes 'days since publication' as a leakage-related feature."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup (public unit tests provided in the prompt) leaks information about solutions that wouldn't be available in a real coding scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Codewars katas share structural similarities, whether solutions to similar katas in training data could transfer, or whether kata families create non-independence in the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The surrogate model with SHAP analysis (Section 4.5) provides a concrete, quantitative method for estimating leakage impact. Features encoding leakage likelihood (completions, publication date) are systematically analyzed via non-interventional SHAP with 100K sampling points."
       }
     }
   },
   "claims": [
     {
       "claim": "The LLM outperforms humans on easy tasks (kyu 7-8) but fails completely on the hardest exercises (kyu 1-2).",
       "evidence": "Figure 7 shows LLM success rates exceeding human rates at kyu 8 and 7, similar performance at kyu 6-4, and 0% LLM success at kyu 1-2 (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "LLM performance varies significantly across programming languages, with legacy languages (COBOL, Fortran) performing dramatically worse.",
       "evidence": "Figures 8-9 show Python leading (score 36.1) and COBOL near zero. Section 4.2 notes COBOL shows 'catastrophic results' with frequent syntax errors.",
       "supported": "strong"
     },
     {
       "claim": "Approximately 37.4% of the LLM's performance can be attributed to leakage of challenge solutions into the training set.",
       "evidence": "SHAP analysis of a linear SVM surrogate model (Section 4.5, Figure 13) groups features into difficulty (46.6%), leakage (37.4%), and language (16%). The surrogate model has 74.88% cross-validation accuracy.",
       "supported": "moderate"
     },
     {
       "claim": "The publication date of challenges negatively correlates with LLM success, with newer challenges being harder to solve.",
       "evidence": "Figure 11 shows LLM Codewars score declining for more recently published katas, both overall and when filtering by difficulty level.",
       "supported": "moderate"
     },
     {
       "claim": "Current evaluation methodologies are overestimating the actual skill of LLMs for generating functional code.",
       "evidence": "Based on the 37.4% leakage attribution and analysis of temporal effects (Sections 4.4-4.5, 6). However, this is based on one model (GPT-4o-mini) on one platform (Codewars), and the surrogate model has limited accuracy (74.88%).",
       "supported": "moderate"
     },
     {
       "claim": "Language popularity (measured by GitHub pushes) correlates with LLM coding performance.",
       "evidence": "Figure 10 shows a clear visual correlation between GitHub language popularity (pushes) and LLM Codewars score across 8 languages.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model generalized to all LLMs",
       "detail": "Only GPT-4o-mini is tested, but conclusions are framed in terms of 'Large Language Models' generally. The abstract and conclusions make claims about 'current evaluation methodologies' and LLM skill broadly."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Multiple comparative claims (LLM vs human, across languages, across difficulty levels) are made without any significance tests, confidence intervals, or error bars."
     },
     {
       "flag": "Single attempt per challenge",
       "detail": "Each kata appears to be attempted once despite LLM outputs being stochastic. The pass@k metric (which the paper reviews in Section 1.4) was designed to address this, yet is not used in the paper's own evaluation."
     },
     {
       "flag": "Code and data deliberately withheld",
       "detail": "Both the evaluation code and results database are withheld (Section 5), making independent verification impossible. While the Codewars ToS concern is acknowledged, alternative approaches (e.g., releasing anonymized results) are not explored."
     },
     {
       "flag": "Surrogate model accuracy limits conclusions",
       "detail": "The 46.6%/37.4%/16% decomposition comes from a linear SVM with only 74.88% accuracy. The remaining 25% unexplained variance means the decomposition is approximate, yet it is presented prominently in the abstract without this caveat."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex, foundational LLM code evaluation benchmark; noted public repos contain solutions to other datasets."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Code challenge benchmark with difficulty levels; found LLMs fail on competition-level problems, directly relevant to difficulty-performance analysis."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for basic Python programs; found LLMs struggle with multi-subproblem tasks and training data modifications."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world software engineering benchmark; discussed for contamination risk since tasks come from public GitHub repos."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "EvalPlus benchmark showing augmented tests reveal LLM code quality is lower than originally reported; directly supports the overestimation thesis."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Comprehensive code generation benchmark showing even best models achieve only 60% on well-structured tasks, 50% on natural language instructions."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for a major LLM family; relevant as the parent model family of the GPT-4o-mini model evaluated in this paper."
     },
     {
       "title": "Open LLM Leaderboard v2",
       "authors": ["Clémentine Fourrier"],
       "year": 2024,
       "relevance": "Updated general LLM leaderboard addressing dataset contamination issues; directly relevant to the paper's contamination argument."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Code-specialized LLM trained on open source code; relevant to understanding code generation training data composition."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Human preference-based LLM evaluation platform used to justify model selection in this study; alternative evaluation methodology."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-specialized LLM trained from scratch on code; relevant to understanding code LLM training and capabilities."
     },
     {
       "title": "CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X",
       "authors": ["Qinkai Zheng"],
       "year": 2023,
       "relevance": "Multilingual code generation benchmark (HumanEval-X); found models perform better on popular languages, consistent with this paper's findings."
     }
   ]
 }