ai-research-survey

scan.json (27064B)

---

 {
   "paper": {
     "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
     "authors": ["Naman Jain", "King Han", "Alex Gu", "Wen-Ding Li", "Fanjia Yan", "Tianjun Zhang", "Sida I. Wang", "Armando Solar-Lezama", "Koushik Sen", "Ion Stoica"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2403.07974"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LiveCodeBench introduces a continuously updated, contamination-free benchmark for evaluating LLMs on code across four scenarios (generation, self-repair, execution, test output prediction). The benchmark reveals likely contamination in DeepSeek and GPT-4-O models on older LeetCode problems, and shows that many fine-tuned open models that perform well on HumanEval do not generalize to LiveCodeBench, suggesting overfitting. GPT-4-Turbo and Claude-3-Opus lead across all scenarios, with closed models maintaining a large gap over open models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a website (https://livecodebench.github.io/) and states 'We will release all prompts and model completions for further community analysis, along with a general toolkit for adding new scenarios and models' (Abstract). The website serves as the release platform."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark problems (511 problems) are released through the LiveCodeBench platform. The paper states problems are collected from public contest platforms and the benchmark is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. They mention using vLLM for inference but do not provide a full environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While prompts are provided in the appendix and the toolkit is mentioned, no step-by-step reproduction instructions are included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (Pass@1) with no confidence intervals or error bars. The paper acknowledges '1-1.5% performance variations' from bootstrapping (Section 7) but does not report CIs in the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite numerous comparative claims (e.g., 'GPT-4-Turbo outperforms all other models'). Differences are assessed by comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage differences with baseline context, e.g., 'DS-Ins-33B is merely 4.3 point behind GPT-4-Turbo on HumanEval+ but 16.2 points (69%) on LCB' (Section 5.2), and 'GPT-4-Turbo leads DS-Ins-33B by 96% and 134%' on other scenarios."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not justify why 511 problems were collected or perform a power analysis. The Limitations section acknowledges the sample size issue but provides no formal justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Pass@1 is computed from 10 samples per problem but no variance across runs or seeds is reported. Only point estimates appear in all results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against HumanEval/HumanEval+ as existing benchmarks and evaluates 52 models as mutual baselines across four scenarios."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4-Turbo, Claude-3-Opus, Gemini-1.5-Pro, LLama-3-70B, and other contemporary models as of 2024."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@1 across four distinct scenarios (code generation, self-repair, code execution, test output prediction), providing multiple complementary evaluation dimensions. Per-difficulty breakdowns (Easy/Medium/Hard) are also provided."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is performed. Evaluation is entirely automated via test case execution. The paper mentions 'manual inspection' of code execution filters (Appendix A.3) but this is for data curation, not output evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses hidden test cases (avg 17 per problem) not shown to models. The temporal split mechanism ensures models are evaluated on problems released after their training cutoff."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty (Easy/Medium/Hard), platform (LeetCode/AtCoder/CodeForces), time window, and scenario. Tables 3-6 provide detailed per-category breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides qualitative examples of code execution failures by GPT-4 with CoT. Section 5.2 discusses where models struggle (Hard problems, open vs closed gap)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CoT prompting sometimes hurts open models on code execution ('open models perform even worse in comparison to the direct code execution baseline', Section 7). Contamination is reported as a negative finding for specific models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about contamination detection, holistic evaluation, overfitting, and model comparisons are all supported by specific results in Sections 5.1 and 5.2 with figures and tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about contamination ('models are likely trained on the older LeetCode problems') based on temporal performance drops, but this is observational evidence — the drop could have other explanations (e.g., problem difficulty variation over time). Similarly, 'overfitting to HumanEval' is inferred from correlation patterns without controlled experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Holistic and Contamination Free Evaluation of Large Language Models for Code' but the benchmark only covers Python and competitive programming problems. The Limitations section acknowledges this but the title and framing are broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses alternative explanations: performance drops could be due to problem difficulty variation, prompt sensitivity, or domain specificity. The paper acknowledges 'different domains might have individual requirements' and that competition problems may not represent general coding."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clear that it measures Pass@1 on competitive programming tasks and does not frame this as measuring general 'coding ability'. It explicitly states LiveCodeBench 'might not be representative of the most general notion of LLM programming capabilities' (Section 7)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 (Appendix C.1) lists exact model IDs for all 52 models, including specific versions like 'gpt-4-0613', 'gpt-4-1106-preview', 'claude-3-opus-20240229', and HuggingFace model paths."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix C.2-C.5 for all four scenarios: code generation, self-repair, code execution (with and without CoT), and test output prediction. Figures 6-7 provide input generator prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'We use nucleus sampling with temperature 0.2 and top p 0.95' and 'generate 10 candidate answers for each problem.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: HTML scraping, formula parsing, image exclusion, difficulty filtering thresholds, test generation procedures. Appendix A.3 details filtering criteria for code execution (character length 100-500, runtime filters, step count limits)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is a dedicated section covering benchmark size, Python-only focus, prompt robustness, and problem domain limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific threats: '1-1.5% performance variations' from bootstrapping, prompt sensitivity particularly for open models on CoT, Python-only limitation, and competition-only domain. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'LiveCodeBench currently only focuses on Python', 'focuses on competition problems sourced from three platforms', and 'might not be representative of the most general notion of LLM programming capabilities.' Recommends using it as 'a starting point.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states they will release 'all prompts and model completions for further community analysis' (Abstract), and the benchmark data is available through the website."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: HTML scrapers for LeetCode, AtCoder, CodeForces; contest dates from May 2023 to May 2024; what metadata is collected (problem statements, test cases, ground truth solutions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data is sourced from public competitive programming platforms (standard benchmark construction)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: scraping → parsing/filtering → difficulty classification → test generation (GPT-4-Turbo generators, Appendix A.2) → validation on correct programs → scenario construction. Filtering criteria and statistics are provided in Table 1 and Appendix A.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists NSF grants (CCF:1900968, CCF:1908870), SKY Lab industrial sponsors (Google, IBM, Intel, Microsoft, etc.), and NSF Graduate Research Fellowship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC Berkeley, MIT, Cornell. No authors are affiliated with the model providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Industrial sponsors include Google, IBM, Intel, Microsoft — companies whose models are evaluated. While the sponsors are for the SKY Lab generally (not this specific paper), the funders have a stake in LLM benchmark outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 (Appendix C.1) lists 'Approximate Cutoff Date' for all 52 models. Section 5.1 discusses specific cutoff dates for DeepSeek (Aug 2023) and GPT-4-O (Nov 2023)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 extensively analyzes potential contamination through temporal performance analysis, showing performance drops after cutoff dates for DeepSeek and GPT-4-O models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination avoidance is a core contribution. The temporal split mechanism (evaluating only on problems released after model cutoff) directly addresses this. Section 5.1 provides empirical evidence of contamination in existing benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time for evaluations are reported despite evaluating 52 models × 10 samples × 511 problems across four scenarios."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API spend) is stated. The paper mentions using vLLM for open models but does not quantify compute."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from sampling (temperature 0.2, 10 samples) but no seed sensitivity analysis is performed. No variance across seeds or runs is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'we generate 10 candidate answers for each problem' and Pass@1 is computed from these 10 samples."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper uses temperature 0.2 and top-p 0.95 without justifying these choices or reporting alternatives tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Prompt and sampling parameters appear chosen without systematic justification. Section 7 acknowledges 'we either do not tune prompts across models or make minor adjustments' but does not explain the selection process."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone multiple comparison corrections, despite comparing 52 models across multiple scenarios."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors construct the benchmark and evaluate models on it without acknowledging potential biases in their benchmark construction choices (e.g., platform selection, difficulty thresholds)."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models via inference only; it does not propose a new method with compute tradeoffs against baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses construct validity: 'Programming is a vast domain' and LiveCodeBench 'currently focuses on competition problems' which 'might not be representative of the most general notion of LLM programming capabilities.' The comparison with HumanEval (Figure 5) also examines whether benchmarks measure the same construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are evaluated via direct prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is a core focus. The paper tags problems with release dates and evaluates models only on problems released after their training cutoff dates (Section 1, Section 5.1)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through prompt formatting, example inputs, or other contextual cues."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether problems across platforms share structural similarities, or whether similar problem types appear in both training corpora and the benchmark."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses temporal performance analysis as a concrete detection method: measuring Pass@1 across monthly time windows and identifying performance drops after cutoff dates (Figures 1, 10-12). This is an empirical leakage detection approach."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek and GPT-4-O show likely contamination on older LeetCode problems, with stark performance drops after their training cutoff dates.",
       "evidence": "Figure 1 shows DS-Ins-33B performance drops after Aug 2023 (its release date) and GPT-4-O drops after Nov 2023 (its cutoff date). DS-Base-33B drops from Pass@1 ~60 to ~0 on Sep vs May LeetCode problems (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Many fine-tuned open models overfit to HumanEval and their performance does not generalize to LiveCodeBench.",
       "evidence": "Figure 5 shows two clusters: base/closed models where HumanEval+ and LCB-Easy performance align, and fine-tuned open models that perform well on HumanEval+ but poorly on LCB-Easy. DS-Ins-1.3B achieves 59.8% on HumanEval+ but only 26.3% on LCB-Easy (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Closed API models consistently outperform open models across all four scenarios, with the gap amplifying on harder tasks.",
       "evidence": "Tables 3-6 and Figure 2 show GPT-4-Turbo and Claude-3-Opus leading across scenarios. Only L3-Ins-70B, Mixtral, and DS-Ins-33B among open models approach closed model performance (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Model performances are correlated across the four scenarios but relative differences vary, highlighting the need for holistic evaluation.",
       "evidence": "Figure 13 shows correlations >0.88 across all scenario pairs, with 0.98 for generation-repair and 0.96 for test output-execution. Claude-3-Opus outperforms GPT-4-Turbo on test output prediction despite trailing on code generation (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "LiveCodeBench better highlights the gap between SoTA and open models compared to HumanEval.",
       "evidence": "DS-Ins-33B is 4.3 points behind GPT-4-Turbo on HumanEval+ but 16.2 points (69%) behind on LCB code generation. This gap amplifies on other scenarios: 96% and 134% on test output prediction and code execution (Section 5.2).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes extensive comparative claims ('outperforms', 'significantly worse') across 52 models based solely on raw Pass@1 numbers without any significance tests, confidence intervals, or uncertainty quantification. The acknowledged 1-1.5% bootstrap variation suggests some close comparisons may not be meaningful."
     },
     {
       "flag": "Industrial sponsors with stake in outcomes",
       "detail": "SKY Lab sponsors include Google, Microsoft, Intel, and Samsung, whose models and products are evaluated. While the authors are academic, the funding relationship creates a potential conflict that is not explicitly acknowledged."
     },
     {
       "flag": "Contamination claims based on correlational evidence",
       "detail": "The contamination claims for DeepSeek and GPT-4-O are based on temporal performance drops, but the paper does not rule out that later problems could simply be harder. The relatively stable AtCoder performance (Figure 11) provides partial control, but LeetCode-specific difficulty trends are not analyzed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "et al."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex model — the primary baseline benchmark LiveCodeBench critiques for contamination."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Released HumanEval+ with more tests revealing up to 8% performance drops, demonstrating test insufficiency in existing benchmarks."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "et al."],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world software engineering benchmark complementary to LiveCodeBench's competitive programming focus."
     },
     {
       "title": "CRUXEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "et al."],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Inspired LiveCodeBench's code execution scenario; evaluates code comprehension through output prediction."
     },
     {
       "title": "Demystifying gpt self-repair for code generation",
       "authors": ["Theo X Olausson", "et al."],
       "year": 2023,
       "arxiv_id": "2306.09896",
       "relevance": "Foundational work on LLM self-repair capabilities that inspired LiveCodeBench's self-repair scenario."
     },
     {
       "title": "Code generation with alphacodium: From prompt engineering to flow engineering",
       "authors": ["Tal Ridnik", "Dedy Kredo", "Itamar Friedman"],
       "year": 2024,
       "arxiv_id": "2401.08500",
       "relevance": "Demonstrates that multi-step LLM pipelines (reasoning, test generation, self-repair) outperform naive generation, motivating holistic evaluation."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "relevance": "Uses edit distance and AST-based similarity to detect code benchmark contamination — directly relevant to contamination methodology."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "et al."],
       "year": 2022,
       "relevance": "Competitive programming benchmark and evaluation methodology for code generation at scale."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "et al."],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark — one of the key existing benchmarks LiveCodeBench aims to improve upon."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "et al."],
       "year": 2023,
       "relevance": "Self-repair and iterative refinement approach relevant to agentic coding workflows and LiveCodeBench's self-repair scenario."
     }
   ]
 }