ai-research-survey

scan.json (26241B)

---

 {
   "paper": {
     "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
     "authors": [
       "Xueying Du",
       "Mingwei Liu",
       "Kaixin Wang",
       "Hanlin Wang",
       "Junwei Liu",
       "Yixuan Chen",
       "Jiayi Feng",
       "Chaofeng Sha",
       "Xin Peng",
       "Yiling Lou"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2308.01861"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is publicly available at https://github.com/FudanSELab/ClassEval, referenced in the abstract and Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (100 class-level coding tasks with test suites and canonical solutions) is released via the GitHub repository. The paper states 'Our benchmark is available at https://github.com/FudanSELab/ClassEval'."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.5 mentions 'eight A800-80G GPUs' and that open-source LLMs were run 'based on the documentation', but no requirements.txt, Dockerfile, or detailed environment specification with library versions is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the experimental setup in Sections 4.3-4.5 but does not provide step-by-step reproduction instructions, README with commands, or scripts to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 7 and Figure 5 report only point estimates (e.g., '37.6% Pass@1') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., 'GPT-4 and GPT-3.5 still exhibit dominate superior', 'WizardCoder outperforms Instruct-CodeGen by 27.5%') but uses no statistical significance tests to support any of these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point differences with baseline context throughout. For example, Section 5.1 states 'GPT-4 and GPT-3.5 achieve 85.4%/68.9% correctness on method-level tasks in HumanEval, but only 37.0%/27.0% correctness on class-level tasks in ClassEval', providing both absolute values and magnitudes of difference."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 100 tasks and n=5 samples per task, but no justification is given for why these sizes are sufficient for the comparative claims made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the unbiased Pass@k estimator (Eq. 1, following Chen et al.), but does not report standard deviations or variance across runs. Only point estimates are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 11 LLMs against each other and also compares ClassEval performance against HumanEval performance for each model (Figure 5, Table 7). Multiple models serve as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 11 studied LLMs include models released in 2022-2023 (e.g., GPT-4, WizardCoder, StarCoder), which were state-of-the-art at the time of the study (July 2023). The paper explicitly states they 'focus on recent models released since 2022' (Section 4.1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "While not a traditional ablation, the comparison of three generation strategies (holistic, incremental, compositional) in Section 5.2/RQ2 effectively ablates the impact of generation approach on each model, isolating the contribution of different strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@1, Pass@3, Pass@5 at both class-level and method-level granularity (Table 7, Figure 5), plus DEP(F) and DEP(M) metrics for dependency generation (Section 4.4, Figure 7)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper where correctness is determined by automated test suites (unit tests with pass/fail outcomes). Human evaluation of generated code quality beyond test-passing is not relevant to the core claims about Pass@k correctness."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation, not a training study. There is no tuning on the benchmark data; models are evaluated directly. The concept of held-out vs. dev sets does not apply in this context."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides breakdowns by generation strategy (Figure 6), by dependency type (Figure 7, Table 5), by error type (Figure 9), and by number of method dependencies (Figure 8). It also compares class-level vs. method-level Pass@k separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 (RQ4: Bad Case Analysis) explicitly analyzes error types (AttributeError, TypeError, KeyError, etc.) in Figure 9, and provides a concrete code example of a KeyError caused by field dependency misinterpretation (Figure 10)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that holistic generation is worse for most models (Section 5.2), that incremental strategy can negatively affect some models like Instruct-StarCoder and WizardCoder (Section 5.2), and that all models show 'much worse performance on class-level code generation' (Finding 1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's four main claims are all supported: (1) worse class-level vs. method-level performance (Figure 5), (2) GPT-4/GPT-3.5 dominance (Table 7, Figure 5), (3) holistic strategy best only for GPT models (Figure 6), (4) limited dependent code generation (Figure 7). Each is backed by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-like claims about why certain strategies work better, e.g., 'most models exhibit rather limited capability of utilizing long input contexts' (Section 5.2), citing Liu et al. [47] as support but without directly testing this explanation. The claim that performance drops are 'attributed to the complexity of generating code that depends on other context' (Section 5.1) is offered as an explanation without controlled experimentation isolating that factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper titles itself 'Evaluating LLMs on Class-level Code Generation' broadly, but results are limited to Python only and 100 tasks. While Section 6 acknowledges 'limited size and programming languages', the title, abstract, and findings (e.g., Finding 1: 'all existing LLMs show much worse performance on class-level code generation') generalize beyond the tested setting without consistent qualification."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses alternative explanations for why holistic generation works better for GPT models (better long-context understanding vs. others' limited capability), and considers both the 'helpful hints' and 'misleading information' effects of incremental generation. Section 6 also acknowledges that prompt design choices might impact findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.5 states 'we use the OpenAI API interface, specifically the \"gpt-4\" and \"gpt-3.5-turbo\" model interface, in July 2023' but does not provide exact snapshot versions (e.g., 'gpt-4-0613'). For open-source models, Table 6 lists model names and sizes but not exact checkpoint versions or commit hashes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides the full prompt templates for all three generation strategies for both IF and non-IF models, including system prompt, Instruction-H, Instruction-I, and Instruction-C with their variable placeholders. Since the class skeleton format is fully defined (Table 2, Figure 2), the prompts are reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports: temperature of 0.2 for nucleus sampling, default top_p, do_sample=false for greedy decoding (temperature 0), maximum window length of 2,048 tokens, n=5 samples for nucleus sampling."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are prompted directly without tool use, feedback loops, or multi-step orchestration beyond the generation strategies themselves."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the full benchmark construction pipeline: task selection (three sources with exclusion criteria in Section 3.2.1), class skeleton construction with design principles (Section 3.2.2), test construction methodology (Section 3.2.3), and canonical solution construction (Section 3.2.4). Each step describes the process and quality controls."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated discussion of threats in both benchmark construction and the empirical study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: (1) potential data leakage mitigated by manual construction, (2) limited benchmark size and Python-only scope affecting generalizability, (3) prompt sensitivity addressed by a pilot study on prompt candidates, (4) randomness addressed by greedy decoding results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6 mentions 'limited size and programming languages' and plans to extend, the paper does not explicitly state what the results do NOT show (e.g., does not claim results apply only to Python, does not bound conclusions to the specific 100 tasks). The findings are stated broadly (e.g., 'all existing LLMs show much worse performance on class-level code generation') without scope qualifiers."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (class skeletons, test suites, canonical solutions) is publicly available at the GitHub repository. Generated code samples and raw evaluation logs are not mentioned as being released, but the benchmark itself enables independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 provides detailed description of the data collection/construction process: task selection from three sources (existing benchmarks, PyPI, brainstorming), explicit exclusion criteria, and the four-step construction pipeline."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for a study. The benchmark constructors are the paper's authors. This is a benchmark construction + automated evaluation paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 illustrates the full construction pipeline. Each step is documented: task selection (100 tasks from 3 sources with exclusion criteria), class skeleton construction (5 participants, double-checking), test construction (method-level + class-level tests, coverage metrics), and canonical solution construction (4 separate participants, double-checking)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is visible in the paper text. There is no mention of grants, sponsors, or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with their affiliation at Fudan University, Shanghai, China, along with their email addresses. Since they are evaluating third-party models (GPT-4, WizardCoder, etc.) and not their own products, there is no affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence of the funder cannot be assessed. The absence of disclosure is not the same as absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Table 6 lists model release dates, the actual training data cutoff dates for the models are not stated. The paper does not specify when training data collection ended for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly addresses this: 'To avoid the coding tasks being seen by LLMs during their training, our benchmark is constructed completely manually, so as to mitigate potential data leakages from existing code sources.' This is the central contamination mitigation strategy."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's entire benchmark design philosophy addresses contamination. Section 3.2 states the benchmark was 'constructed completely manually' with '500 person-hours' specifically to avoid overlap with training data. Section 6 also lists data leakage as a threat and the manual construction as mitigation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The paper constructs a benchmark and evaluates LLMs automatically."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The benchmark constructors are the authors themselves."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The paper notes benchmark constructors have '2-8 years of Python development experience' but this characterizes workers, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses OpenAI APIs (GPT-4, GPT-3.5) and runs 11 models with nucleus sampling (5 samples per task) and greedy sampling across 100 tasks and 3 strategies, but reports no API costs, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.5 mentions 'eight A800-80G GPUs' as infrastructure but does not report total GPU hours, total API spend, or training/inference time."
       }
     }
   },
   "claims": [
     {
       "claim": "All existing LLMs show much worse performance on class-level code generation compared to standalone method-level code generation benchmarks like HumanEval.",
       "evidence": "Figure 5 shows GPT-4 drops from 85.4% (HumanEval) to 37.0% (ClassEval class-level), GPT-3.5 from 68.9% to 27.0%, WizardCoder from 59.8% to 11.0%, with consistent drops across all 11 models. Method-level Pass@1 on ClassEval also lower than HumanEval (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 and GPT-3.5 dominate other LLMs on class-level code generation, with Instruct-StarCoder, Instruct-CodeGen, and WizardCoder forming a similar second tier.",
       "evidence": "Table 7 and Figure 5: GPT-4 achieves 37.6% class-level Pass@1 (nucleus), GPT-3.5 at 29.6%, while third-ranked WizardCoder is at 12.2%. Second-tier models (WizardCoder, Instruct-StarCoder, Instruct-CodeGen) range from 10.0%-11.1% in greedy Pass@1 (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Holistic generation is the best strategy only for GPT-4 and GPT-3.5, while method-by-method generation is better for other models.",
       "evidence": "Figure 6 shows holistic generation achieves 6%-9% higher class-level Pass@5 for GPT-4 and 4%-14% for GPT-3.5 compared to other strategies. For other models like CodeGeeX and SantaCoder, incremental strategy produces 9% and 7% more correct classes than holistic (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "The disparity in generation strategy effectiveness stems from models' limited capability of understanding long instructions and utilizing middle information.",
       "evidence": "Section 5.2 cites Liu et al. [47] showing LLMs become less effective with increasing input length and tend to use beginning/end information better. This is offered as a potential explanation, not directly tested.",
       "supported": "weak"
     },
     {
       "claim": "It is easier for all models to generate field-accessing code than method-invoking code.",
       "evidence": "Figure 7 shows DEP(F) consistently higher than DEP(M) across all 11 models. GPT-4 leads with at least 12.6%/6.3% improvement over other models in DEP(F)/DEP(M) (Section 5.3).",
       "supported": "strong"
     },
     {
       "claim": "Method-level coding ability cannot equivalently reflect class-level coding ability among LLMs.",
       "evidence": "Section 5.1: WizardCoder and Instruct-StarCoder have much higher HumanEval scores (59.8% and 34.1%) than SantaCoder (14.6%), but all three perform similarly on ClassEval (around 10-11% Pass@1). Rankings do not transfer.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ClassEval is a manually-crafted benchmark of 100 class-level Python code generation tasks requiring ~500 person-hours to construct. All 11 evaluated LLMs perform substantially worse on class-level code generation than on method-level benchmarks like HumanEval, with GPT-4 dropping from 85.4% to 37.0% Pass@1. The optimal generation strategy varies by model: holistic generation works best for GPT-4/GPT-3.5, while method-by-method generation is better for weaker models. Models consistently find it easier to generate field-dependent code than method-dependent code.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper makes numerous comparative claims about model performance differences (e.g., 'GPT-4 and GPT-3.5 still exhibit dominate superior') without any statistical significance testing. With only 100 tasks and n=5 samples, the differences between closely-ranked models (e.g., WizardCoder at 11.0% vs. Instruct-StarCoder at 10.0%) may not be statistically significant."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Despite using nucleus sampling with temperature=0.2, no standard deviations, confidence intervals, or error bars are reported. The Pass@k estimator is unbiased but its variance is not quantified, making it impossible to assess the reliability of the reported numbers."
     },
     {
       "flag": "Causal explanations without causal evidence",
       "detail": "The paper attributes performance differences to 'limited capability of understanding long instructions and utilizing the middle information' (Section 5.2) and 'complexity of generating code that depends on other context' (Section 5.1) but these are post-hoc explanations citing related work, not experimentally validated causal claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and the Pass@k metric, which is the primary baseline benchmark and evaluation metric used in this study."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark for code generation evaluation, one of the key baselines for comparing benchmark complexity."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces HumanEval+ with enhanced test suites, relevant to benchmark quality and rigor in code generation evaluation."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Z. Luo", "C. Xu", "P. Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "One of the evaluated code LLMs; represents instruction-tuned code generation models."
     },
     {
       "title": "StarCoder: May the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Foundation model for several evaluated LLMs (WizardCoder, Instruct-StarCoder); large-scale open code model."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["H. Yu", "B. Shen", "D. Ran"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Related benchmark for non-standalone code generation that motivates ClassEval's focus on method dependencies."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["N. F. Liu", "K. Lin", "J. Hewitt"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Cited to explain why holistic generation strategy fails for most models -- relevant to understanding LLM context utilization limitations."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "The top-performing model in the ClassEval evaluation; foundational reference for frontier LLM capabilities."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Y. Chang", "X. Wang", "J. Wang"],
       "year": 2023,
       "arxiv_id": "2307.03109",
       "relevance": "Comprehensive survey of LLM evaluation approaches, contextualizing ClassEval within the broader evaluation landscape."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. H. Choi", "J. Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Competitive programming code generation approach; represents a different type of challenging code generation benchmark."
     },
     {
       "title": "Large language models meet NL2Code: A survey",
       "authors": ["D. Zan", "B. Chen", "F. Zhang"],
       "year": 2023,
       "relevance": "Survey of code generation with LLMs, providing broader context for the ClassEval evaluation."
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "authors": ["B. Athiwaratkun", "S. K. Gouda", "Z. Wang"],
       "year": 2023,
       "relevance": "Multi-lingual code generation benchmark (MBXP, Multi-HumanEval); extends HumanEval-style evaluation to multiple languages."
     }
   ]
 }