ai-research-survey

scan.json (23735B)

---

 {
   "paper": {
     "title": "Performance Review on LLM for solving leetcode problems",
     "authors": [
       "Lun Wang",
       "Chuanqi Shi",
       "Shaoshuai Du",
       "Yiyi Tao",
       "Yixian Shen",
       "Hang Zhang",
       "Yanxin Shen",
       "Xinyu Qiu"
     ],
     "year": 2024,
     "venue": "2024 4th International Symposium on Artificial Intelligence and Intelligent Manufacturing (AIIM)",
     "arxiv_id": "2502.15770",
     "doi": "10.1109/AIIM64537.2024.10934280"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper evaluates 22 LLMs on LeetCode problems using pass@k and runtime metrics. GPT-4-omni achieves the highest non-canonical pass@1 (43.36%) and pass@10 (61.95%), while most open-source models score below 5% pass@1. Solutions from o1-mini achieve a mean LeetCode runtime percentile rank of 63%, suggesting LLM-generated code is faster than most human submissions. However, the paper has significant internal inconsistencies (204 vs ~2,100 problems, 18 vs 22 models) and its reference list is dominated by unrelated self-citations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub link is provided in the results section: https://github.com/DHUer/LLMevaluationresults. The paper states 'All the experiment code and dataset is published' there."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the dataset is published at the GitHub link alongside the experiment code. LeetCode problems are also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper mentions Python and pytest-benchmark but provides no requirements.txt, dependency versions, or setup instructions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the general methodology but gives no concrete commands or procedures to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table I reports only point estimates for pass@1 and pass@10 with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Model performance differences are compared by raw number comparison only. Claims like 'top performers' are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper presents raw pass@k percentages and a runtime percentile rank (63%) but no formal effect size measures such as Cohen's d or structured percentage improvement with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 204 (or ~2,100) problems were used, why 10 solutions per temperature were generated, or why 5 temperature settings were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. The paper generates multiple solutions but only reports aggregated pass@k metrics without spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I includes 'Canonical Solutions' (human-written, 97.94% pass@1) as a baseline, and 22 models are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes contemporary models such as GPT-4-omni, GPT-4-turbo, and o1-mini alongside older models like CodeGen and SantaCoder, providing a range of baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing LLMs as black boxes with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@10, runtime, memory usage, and LeetCode runtime percentile rank."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of code quality is performed. The comparison with humans is through automated LeetCode percentile rankings, not manual code review."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The best pass@k is selected across all temperature settings on the same test problems. There is no validation/test split — the temperature hyperparameter is optimized on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Although the dataset spans Easy/Medium/Hard difficulty levels (ratio ~11:50:10), no per-difficulty breakdowns of results are presented. Only aggregate pass@k numbers are shown in Table I."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The methodology section mentions 'Error Analysis: Identification of common errors or misconceptions' but the results section contains no actual error analysis or failure case discussion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "While many models show very low scores (e.g., InCoder-1B at 0.10% pass@1), these are presented as observations rather than analyzed as negative results. No discussion of what went wrong or unexpected findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'comprehensive performance evaluation' and promises insights into 'strengths and limitations.' The results section does provide pass@k data and runtime analysis supporting these general claims, though 'comprehensive' is generous given the lack of per-category breakdown."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to 'observe the impact of the temperature parameter on the correctness and efficiency of the generated code' but presents no per-temperature analysis to support this claim. Temperature is varied but its effect is not isolated or reported."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims insights into LLMs' 'potential applications and areas for improvement in automated programming assistance,' generalizing beyond LeetCode algorithmic problems to broader programming assistance without justification."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed performance differences between models or for the 63% runtime percentile rank finding."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@k on LeetCode and frames this as evaluating LLMs' capabilities in 'code generation and problem-solving tasks' and 'automated programming assistance' without discussing that LeetCode algorithmic problems are a narrow proxy for real-world programming ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-4', 'GPT-3.5-turbo', 'GPT-4-omni', 'o1-mini', 'Copilot'. No specific API versions, snapshot dates, or exact model identifiers (e.g., 'gpt-4-0613') are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes providing 'problem statement, code comments, and code framework' as input but never shows the actual prompt text sent to the models. The prompt structure is described in natural language only."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are explicitly reported (0.2, 0.4, 0.6, 0.8, 1.0) along with the number of solutions generated (10 per temperature per model per problem)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are queried directly via API or IDE integration."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes preprocessing: 'We standardized the problem data by removing any extraneous information such as solution discussions, hints, or previously submitted solutions' and extracting problem statements, function signatures, and code comments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations, threats to validity, or discussion section addressing the study's shortcomings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Critical threats such as benchmark contamination, LeetCode's noisy runtime measurements, and the narrow scope of algorithmic problems are unaddressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show or which populations/settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository link (https://github.com/DHUer/LLMevaluationresults) claimed to contain the experiment code and dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes crawling LeetCode to collect problems, extracting problem statements, function signatures, and code comments, and standardizing the data by removing extraneous information."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the public LeetCode platform."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from collection to analysis has unexplained inconsistencies. The contributions section claims '204 Leetcode problems' while the dataset analysis section claims 'approximately 2,100 LeetCode problems.' Table I shows 22 model entries but the abstract claims '18 LLMs.' These discrepancies suggest the pipeline is incompletely documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The paper contains an unfilled template placeholder: 'Identify applicable funding agency here. If none, delete this.' No funding information is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All eight authors' university affiliations are listed (Duke, UCSD, UvA, JHU, SFU, Northeastern). None appear affiliated with the model providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed — the template placeholder was left unfilled. Independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models evaluated. This is critical because LeetCode problems and solutions are widely available online and almost certainly in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluated models may have been trained on LeetCode problems and solutions, which are publicly available and widely scraped."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems have been publicly available for years and solutions are extensively discussed online. All evaluated models likely trained on this data. This fundamental contamination risk is never mentioned."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported despite generating thousands of API calls across multiple models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, hardware used, or total API spend is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. While 10 solutions per temperature are generated, results are only reported as aggregated pass@k without examining variance across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states '10 distinct solutions per model for each problem' at each of 5 temperature settings."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The temperature search space is explicitly stated: 5 values (0.2, 0.4, 0.6, 0.8, 1.0) with 10 solutions at each, constituting 50 generations per model per problem."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The best pass@k value observed across all temperatures was then considered the final pass@k metric' — this selects the best temperature on the test set with no separate validation split, inflating reported performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "22 models are compared with no statistical tests at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential bias in the evaluation methodology. The authors control the prompt construction, temperature selection, and submission process, which could introduce systematic advantages or disadvantages."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 350M parameters (CodeGen-350M) to GPT-4 are compared on the same metrics without any normalization or discussion of compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "While contribution 3 claims to 'assess the usability of Leetcode as a public repository,' the results section contains no actual analysis of construct validity — whether LeetCode performance measures what the paper claims about LLM programming capability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GitHub Copilot is accessed through VS Code integration while other models use the OpenAI API. These are fundamentally different interfaces that may provide different context. This confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. LeetCode problems have been available online for years, predating the training of all evaluated models. No temporal analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. No discussion of whether providing function signatures and code comments provides information that wouldn't be available in real-world usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. LeetCode problems within the same topic category may share structural similarities, and many problems have publicly available solutions that could be in training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, temporal splits, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4-omni is the top-performing LLM with pass@1 of 43.36% and pass@10 of 61.95%",
       "evidence": "Table I shows pass@1 and pass@10 for 22 models, with GPT-4-omni ranking highest among LLMs (Canonical Solutions excluded).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-generated solutions (o1-mini) achieve a mean runtime percentile rank of 63%, faster than 63% of human submissions",
       "evidence": "Section III.B and Figure 3 report this finding based on LeetCode's runtime percentile rankings.",
       "supported": "weak"
     },
     {
       "claim": "The paper evaluates 18 LLMs on 204 LeetCode problems",
       "evidence": "Contribution 1 states '18 LLMs on 204 Leetcode problems,' but Table I lists 22 entries and the dataset analysis section claims approximately 2,100 problems.",
       "supported": "weak"
     },
     {
       "claim": "Temperature parameter impacts correctness and efficiency of generated code",
       "evidence": "The methodology describes testing 5 temperature settings (0.2–1.0), but no per-temperature results are presented in the results section. Only best-across-temperature pass@k values are reported.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Internal inconsistencies in dataset and model counts",
       "detail": "The contributions section claims '18 LLMs on 204 Leetcode problems,' but Table I lists 22 model entries (including Canonical Solutions) and the dataset analysis section says 'approximately 2,100 LeetCode problems.' The abstract lists 5 authors while the paper header lists 8."
     },
     {
       "flag": "Unrelated self-citation padding in references",
       "detail": "The majority of numbered references [1]-[17] are unrelated to the paper's topic: PIQI quantized inference [1], data augmentation [3], fMRI analysis [4], few-shot learning in GANs [5], phishing attacks [6], robot obstacle avoidance [7], diabetes risk prediction [8], parameter-efficient fine-tuning [9], meta learning [10][11], explainability [12], vision-language pre-training [13], DNN frameworks [14], load balancing [15][16], multi-label learning [17]. Many are by the paper's own authors. Only [2] (Coignion et al.) is directly relevant."
     },
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "LeetCode problems and their solutions are among the most extensively indexed programming content online. Every model tested was almost certainly trained on LeetCode data. This fundamental validity threat is never mentioned."
     },
     {
       "flag": "Unfilled template placeholder left in paper",
       "detail": "The paper contains the text 'Identify applicable funding agency here. If none, delete this.' — an IEEE template instruction that was not addressed before submission."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "22 models are ranked by point estimates with no error bars, confidence intervals, or significance tests. Claims of 'top performers' and 'lower performers' are based solely on comparing raw numbers."
     },
     {
       "flag": "Best temperature selected on test set",
       "detail": "The methodology states 'The best pass@k value observed across all temperatures was then considered the final pass@k metric.' This optimizes hyperparameters on the test set with no validation split, inflating reported performance."
     },
     {
       "flag": "Claimed analysis not presented",
       "detail": "Several analyses described in the methodology (error analysis, per-temperature effects, LeetCode dataset assessment) are never presented in the results section."
     }
   ],
   "cited_papers": [
     {
       "title": "A performance study of LLM-generated code on Leetcode",
       "authors": ["Tristan Coignion", "Clément Quinton", "Romain Rouvoy"],
       "year": 2024,
       "relevance": "Directly relevant benchmark evaluation of LLM code generation on the same LeetCode platform."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex/HumanEval)",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Defines the pass@k metric and unbiased estimator used as the primary evaluation methodology in this paper."
     }
   ]
 }