ai-research-survey

scan.json (24248B)

---

 {
   "paper": {
     "title": "Holistic Evaluation of State-of-the-Art LLMs for Code Generation",
     "authors": ["Le Zhang", "Suresh Kothari"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.18131"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DeepSeek-R1 and GPT-4.1 consistently outperform other models across 944 LeetCode problems in 5 languages on Pass@1, compile-time errors, runtime errors, functional failures, and algorithmic suboptimality. Llama-3.3 performs worst across nearly all metrics. Adding an optimization prompt hint reduces algorithmic suboptimality for most models, with DeepSeek-R1 showing strongest responsiveness. Python and JavaScript yield fewest compile-time errors while Go has the most.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.7 provides a Figshare link: https://figshare.com/s/26448e92798aab34e407 containing datasets, code, and evaluation results."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package on Figshare includes datasets and evaluation results (Section 3.7). The underlying LeetCode problems are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper mentions a replication package but does not describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., Pass@1 percentages in Table 4) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models 'outperform' others based solely on comparing raw percentages. No statistical significance tests are applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4-6 report absolute performance metrics with baselines for context. E.g., Pass@1 ranging from 51.38% (Llama-3.3 Go) to 89.30% (DeepSeek-R1 Python), providing magnitude context for comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 944 problems (large dataset) and 202 (small dataset) with no justification for why these sizes are adequate or how they were determined."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results only. Each problem is submitted once per model per language (Section 3.5). No variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six models are compared against each other across multiple metrics, serving as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All six models are from 2024-2025: DeepSeek-R1, DeepSeek-V3, GPT-4.1, Claude-3.7, Qwen2.5-Coder, Llama-3.3. These are state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark comparison study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: Pass@1, Compile-time Error, Runtime Error, Functional Failure, and Algorithmic Suboptimality (Sections 3.6.1-3.6.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. All evaluation is automated through LeetCode's test suite."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LeetCode's built-in test suites serve as the evaluation, and these are not used for any model tuning in this study. The models are evaluated as-is via API."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model, programming language, and difficulty level (Tables 2, 4, 5). The large vs. small dataset also provides category-level analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure scenarios including syntax errors, logical flaws, and suboptimal algorithms. Section 4 analyzes specific error types (CE, RE, FF, AS) with language-specific patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Llama-3.3's poor performance is consistently reported. Section 4.6 notes Claude-3.7 shows minimal/no improvement from optimization hints, and some models show higher AS with hints."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims DeepSeek-R1 and GPT-4.1 consistently outperform others, which is supported by Tables 4-5 across all metrics."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about prompt engineering improving results (Section 4.6, 'the inclusion of the optimization hint leads to a reduction in AS values') based on a single before/after comparison with no controls for stochastic variation. Single-run design means differences could be noise."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title says 'Holistic Evaluation' and abstract claims 'comprehensive empirical evaluation' but tests only LeetCode algorithmic problems (greedy, sorting, binary search, trees). Does not bound claims to this narrow domain — real-world software development involves far more than algorithmic puzzles."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance differences. Model size, training data composition, reasoning capabilities, and other confounds are not systematically analyzed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames LeetCode Pass@1 as evidence for 'reliable code generation in real-world software development tasks' (abstract) without acknowledging that LeetCode algorithmic puzzles are a narrow proxy for real-world coding capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 specifies exact model names: DeepSeek-R1, DeepSeek-V3-0324, Qwen2.5-Coder-32B-Instruct, Llama-3.3-70B-Instruct, GPT-4.1, Claude-3.7-Sonnet with sizes and release years."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 provides the complete prompt structure with an actual example including role specification, problem description, constraints, code snippet, test cases, and additional instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports temperature=0.1 and top-p=0.95 for all models, with justification for these choices."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are called directly via API with single-shot prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the dataset construction: 944 manually picked LeetCode problems focusing on four topics, with 202 selected for complexity sensitivity. Problem distribution by difficulty and test case count is documented in Tables 2-3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper has Conclusions, Recommendations, and Future Work sections but no substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion. The contamination concern in Section 3.2 is the only acknowledgment of a methodological limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Future Work (Section 6) mentions areas not covered but does not frame them as scope boundaries of the current findings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.7 states all artifacts including datasets, code, and evaluation results are available at a Figshare link."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data collection: 944 manually picked LeetCode problems covering greedy algorithms, sorting, binary search, and tree-based problems, with difficulty distribution."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is LeetCode problems (a standard public benchmark)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes the pipeline: LLMs generate code → submitted via custom LeetCode API → one submission per model per language → results collected in JSON format."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are from the Department of Computer Science, Iowa State University. No affiliation with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the six models. This is critical since LeetCode problems and solutions are widely available online."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 acknowledges that 'some problems or their corresponding solutions may have appeared in the pretraining corpora' and describes mitigation: selecting recent problems, avoiding discussion content/solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3.2 acknowledges contamination risk and uses recent problems as mitigation, no concrete detection method is applied. The paper admits 'complete elimination of contamination cannot be guaranteed' but does not analyze which models' training data may include LeetCode solutions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports API pricing per million tokens and total costs for each model. Section 4.7.2 discusses cost analysis including DeepSeek's off-peak pricing."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports total token usage (input/output in millions) and total USD costs for all models. Total costs range from $2.39 (Llama-3.3) to $108.65 (Claude-3.7)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results only. Temperature is set to 0.1 but no seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 explicitly states 'one submission per model and per programming language' — i.e., exactly 1 run per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature=0.1 and top-p=0.95 are chosen based on prior literature but no search was performed. Only one configuration is tested per model."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 justifies parameter choices by citing prior work showing top-p=0.95 and temperature=0.1 are effective for one-shot code generation."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction. Comparisons across 6 models × 5 languages × 5 metrics are made without any statistical framework."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors are not evaluating their own system. They are comparing third-party LLMs."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 analyzes token usage and cost across models, noting DeepSeek-R1 uses substantially more tokens. Table 6 provides full cost breakdown alongside performance results."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LeetCode algorithmic problems are a valid proxy for 'real-world software development tasks' as claimed. The paper uses LeetCode without questioning its construct validity for the broad claims being made."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive the same direct API prompt."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states they 'intentionally selected relatively recent problems that are less likely to be included in existing training corpora' as a mitigation strategy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompt structure or LeetCode problem format provides hints not available in real-world usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LeetCode problems share structural similarities that could inflate performance (e.g., similar problem templates, shared solution patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. Mitigation is limited to selecting recent problems — no canary strings, membership inference, or n-gram overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-R1 and GPT-4.1 consistently outperform other models in correctness, efficiency, and robustness across all programming languages.",
       "evidence": "Table 4 shows DeepSeek-R1 achieves highest Pass@1 (86.97-89.30%) and lowest FF/AS metrics across all 5 languages. GPT-4.1 is second-best on most metrics.",
       "supported": "moderate"
     },
     {
       "claim": "Llama-3.3 shows the lowest performance across most metrics.",
       "evidence": "Table 4: Llama-3.3 has Pass@1 of 51-55%, highest FF (30-34%), and highest AS (7.6-10.5%) across languages.",
       "supported": "strong"
     },
     {
       "claim": "Python3 and JavaScript produce zero compile-time errors across all models.",
       "evidence": "Table 4 shows CE=0.00% for both Python3 and JavaScript across all six models.",
       "supported": "strong"
     },
     {
       "claim": "Optimization prompt hints reduce algorithmic suboptimality for most models.",
       "evidence": "Table 5 and Figure 7 show AS reduction with hints for DeepSeek-R1, GPT-4.1, DeepSeek-V3, and Qwen2.5-Coder. Claude-3.7 shows minimal/no improvement.",
       "supported": "weak"
     },
     {
       "claim": "DeepSeek-R1 maintains moderate costs despite high token consumption due to pricing structure.",
       "evidence": "Table 6 shows DeepSeek-R1 uses 36.52M output tokens (vs 4-6M for others) but costs $20.52 at off-peak vs Claude-3.7's $108.65.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run evaluation",
       "detail": "Each problem is submitted exactly once per model. With temperature=0.1, outputs are near-deterministic but not fully deterministic. No variance or reproducibility assessment is possible from single runs."
     },
     {
       "flag": "No statistical tests for claimed differences",
       "detail": "All comparative claims ('outperforms', 'consistently better') are based on raw percentage comparisons with no significance testing. Differences between some models may not be meaningful."
     },
     {
       "flag": "LeetCode contamination risk understated",
       "detail": "LeetCode problems and solutions are extensively available online. Models trained on web data almost certainly saw many of these problems. The mitigation of using 'recent problems' is not validated — no cutoff dates are provided for any model's training data."
     },
     {
       "flag": "Overly broad claims from narrow benchmark",
       "detail": "The title claims 'Holistic Evaluation' and recommendations target 'real-world software development tasks' but the benchmark covers only algorithmic problems (greedy, sorting, binary search, trees) — no API usage, debugging, refactoring, or real-world software engineering tasks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations or threats to validity, which is a significant methodological omission for an empirical evaluation."
     },
     {
       "flag": "Prompt engineering claim from uncontrolled comparison",
       "detail": "The optimization hint experiment (Section 4.6) compares two single runs without controlling for stochastic variation. Observed differences may be noise rather than signal."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex evaluation (HumanEval benchmark) that established LLM code generation evaluation methodology."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2025,
       "relevance": "Evaluates security vulnerabilities in LLM-generated code, directly relevant to code quality assessment."
     },
     {
       "title": "A performance study of LLM-generated code on LeetCode",
       "authors": ["T. Coignion", "C. Quinton", "R. Rouvoy"],
       "year": 2024,
       "relevance": "Prior LeetCode-based LLM evaluation study examining hyperparameter impact and prompt design."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["D. Guo", "D. Yang", "H. Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Technical report for one of the top-performing models evaluated in this study."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C.S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorously evaluates LLM code generation correctness, questioning optimistic early assessments."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp", "B. Pang", "H. Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Demonstrates structured prompting improving code generation robustness across languages."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen", "S. Kim", "S. Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Comprehensive survey of LLM code generation covering architectures and evaluation approaches."
     },
     {
       "title": "LeetCodeDataset: A temporal dataset for robust evaluation and efficient training of code LLMs",
       "authors": ["Y. Xia", "W. Shen", "Y. Wang"],
       "year": 2025,
       "arxiv_id": "2504.14655",
       "relevance": "Addresses temporal aspects of LeetCode evaluation and contamination concerns for code LLMs."
     },
     {
       "title": "Proving the coding interview: A benchmark for formally verified code generation",
       "authors": ["Q. Dougherty", "R. Mehta"],
       "year": 2025,
       "relevance": "Evaluates LLMs on deductively verifiable code generation, complementary evaluation approach."
     }
   ]
 }