ai-research-survey

scan.json (27092B)

---

 {
   "paper": {
     "title": "Beyond Correctness: Benchmarking Multi-dimensional Code Generation for Large Language Models",
     "authors": [
       "Jiasheng Zheng",
       "Boxi Cao",
       "Zhengzhao Ma",
       "Ruotong Pan",
       "Hongyu Lin",
       "Yaojie Lu",
       "Xianpei Han",
       "Le Sun"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.11470"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release our benchmark and source code at https://github.com/jszheng21/RACE and leaderboard at https://huggingface.co/spaces/jszheng/RACE_leaderboard' in the abstract (footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released via the GitHub repository. The paper also uses publicly available datasets (HumanEval+, MBPP+, ClassEval, LeetCode) and the benchmark itself is released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is mentioned in the paper. The paper does not specify library versions or dependencies needed to reproduce the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While source code is released, the paper itself does not contain step-by-step reproduction instructions or reference a README with commands to run. The experimental setup describes models and hyperparameters but not how to execute the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, 5, 6, and 7 report only point estimates (percentages and scalars). No confidence intervals, error bars, or ± notation are provided anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GPT-4o-mini outperforms it by at least 5 percentage points') based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap tests, etc.) are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage differences and scores with baseline context throughout. For example, Table 3 reports deltas (e.g., '+18.9', '+75.0') showing the magnitude of contamination impact. Section 4.2 states specific scores like 'o1-mini achieves only a score of 60.3 in time complexity' with context across models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not justify why the specific number of evaluation cases per factor was chosen (e.g., 923 for correctness, 492 for code length, 101 for efficiency). No power analysis or justification for these sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses greedy decoding (temperature 0, Section 4.1), producing deterministic single-run results. No variance, standard deviation, or multi-run spread is reported. While greedy decoding makes runs deterministic for a given model, there is no discussion of variance across problem instances or other sources of variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 28 LLMs against each other across all dimensions. For the contamination experiment (Section 4.3), a clean vs. contaminated comparison using starcoderbase-7b as baseline is conducted."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include state-of-the-art models as of 2024: o1-mini-2024-09-12, Claude-3.5-Sonnet, GPT-4o, Qwen2.5-72B, DeepSeek-V2.5. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like experiments: Section 4.4 tests complex instruction following with varying numbers of requirements (2-5), and Section 4.5 examines preference bias by varying specific instructions (camel vs. snake case, different length constraints, for vs. while loops). The contamination experiment (Section 4.3) ablates data contamination levels."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics across dimensions: accuracy (Acc), instruction-following rate (IF rate), Maintainability Index (MI), Normalized Index for Time (NIT), Normalized Index for Space (NIS), and an overall RACE score."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates code quality dimensions like readability and maintainability entirely through automated metrics (AST analysis, MI formula, runtime monitoring). No human evaluation of the generated code quality or the benchmark itself is conducted, despite readability and maintainability being inherently subjective dimensions where human judgment would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses established test sets (HumanEval+, MBPP+, ClassEval, LeetCode) that are not used for model development by the authors. The contamination experiment (Section 4.3) explicitly separates clean and contaminated training from evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: Tables 2, 5, 6, and 7 show results broken down by individual factors (naming convention, length, comments, MI, modularity, time/space efficiency) for all 28 models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses how models fail with complex instructions (performance drops to near 0 with 5 requirements). Section 4.5 discusses preference bias failures (e.g., 'CodeLlama and WizardCoder almost fail to comprehend and fulfill' camel-case requirements, with IF rates below 30%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: models struggle significantly with complex multi-requirement instructions (Section 4.4), most models have strong preference biases that hinder instruction following (Section 4.5), and adding length requirements tends to decrease code accuracy (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's four claims are all supported: (1) RACE reveals multi-dimensional defects (Table 2, Section 4.2), (2) RACE resists data contamination (Table 3, Section 4.3), (3) LLMs struggle with complex instructions (Figure 5, Section 4.4), (4) LLMs exhibit coding style preferences (Figure 6, Section 4.5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'we hypothesize that this improvement may be attributed to comments facilitating an implicit chain-of-thought' (Section 4.2) and 'data contamination significantly impairs the model's instruction-following ability' (Section 4.3). While the contamination experiment uses a controlled setup, the chain-of-thought hypothesis is speculative. The claim about preference bias ('primarily learn the inherent patterns of token prediction from examples, lacking a comprehensive understanding of code logic') is a causal explanation without adequate evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims 'Multi-dimensional Code Generation for Large Language Models' broadly, but experiments are limited to Python only. The paper mentions in Section 4.5 that preference bias 'could be even more pronounced in programming languages like Perl, JavaScript, and PHP' but does not test them. The generalization from Python-only results to 'code generation' broadly is not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the claim that models have 'inherent preference bias' could alternatively be explained by dataset composition or fine-tuning distributions. The improvement from comments could have explanations beyond chain-of-thought. No threats-to-validity section or alternative explanation discussion is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 (Appendix B.1) lists specific model IDs: 'gpt-4o-2024-05-13', 'gpt-3.5-turbo-0125', 'o1-mini-2024-09-12', 'claude-3.5-sonnet', 'deepseek-coder-6.7b-instruct', 'CodeLlama-7b-Python-hf', etc. Most closed-source models include snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 9 and 10 (Appendix B.2) for all factors across all four dimensions. These include the complete text with placeholders like {problem} and {starter_code} where the fill values come from standard benchmark problems."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'we use a greedy strategy and set the temperature to 0.' For the contamination experiment (Section 4.3), LoRA training details are provided: 'batch size of 32 and a learning rate of 1e-3', trained for 10 epochs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct prompt-to-completion generation with no tool use, retry logic, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data preprocessing: 'we exclude such information from the datasets' to prevent bias. Appendix B.2 states 'we modify the original prompt format by extracting the core task descriptions to serve as the final prompts' to prevent conflicts with readability requirements. Table 1 provides exact case counts per factor."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section in the paper. The conclusion (Section 5) mentions future work directions but does not substantively discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no discussion of potential issues such as the limitation to Python, the reliance on automated metrics for subjective quality dimensions, or the representativeness of the selected benchmark tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to Python, to the specific models tested, or acknowledge that automated readability/maintainability metrics may not align with human judgment."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data and source code are released at https://github.com/jszheng21/RACE. The leaderboard is available at https://huggingface.co/spaces/jszheng/RACE_leaderboard, suggesting raw evaluation data is accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection in detail: correctness data from HumanEval+, MBPP+, ClassEval, and LeetCode; readability evaluated on HumanEval+; maintainability on ClassEval and LeetCode; efficiency on 101 self-constructed LeetCode-based cases. Table 1 provides exact counts per factor."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented through Section 3 and Appendix B.2: selection of source datasets, extraction and modification of prompts, incorporation of customized requirements, and evaluation through AST analysis and runtime monitoring. Figure 2 illustrates the overall evaluation pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. The authors are affiliated with the Chinese Academy of Sciences, which suggests institutional funding, but no explicit disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese Information Processing Laboratory, State Key Laboratory of Computer Science, Institute of Software, Chinese Academy of Sciences, and University of Chinese Academy of Sciences."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors do not appear to have a direct conflict with any of the evaluated models, but the absence of any funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 28 LLMs on benchmarks but does not state the training data cutoff dates for the models being evaluated. For the contamination experiment, it notes that starcoderbase-7b was 'carefully curated to exclude data from HumanEval and MBPP' and that 'ClassEval and LeetCode are not within the temporal coverage of its training data,' but no specific cutoff dates are given for the other 27 models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 is entirely dedicated to discussing train/test overlap (data contamination). The paper explicitly investigates the impact of contamination on benchmark results and demonstrates that RACE is more robust to contamination than traditional benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 directly addresses benchmark contamination as a core contribution. The paper argues that traditional correctness-only benchmarks are susceptible to contamination while RACE provides more stable results under contamination. However, this is done through a controlled experiment with starcoderbase-7b rather than analyzing contamination risk for all 28 evaluated models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 28 LLMs across thousands of evaluation cases but does not report inference costs, API costs, tokens consumed, or wall-clock time for the evaluations. The o1-mini results used 30% sampling 'to optimize budget efficiency' (Table 2 footnote), implying significant cost, but actual costs are not stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware specifications are reported. The contamination experiment involved LoRA training for 10 epochs but does not mention the GPU type or training time."
       }
     }
   },
   "claims": [
     {
       "claim": "Current correctness-centric benchmarks fail to capture the multifaceted requirements of code in real-world scenarios, while RACE provides comprehensive evaluation revealing defects across multiple dimensions.",
       "evidence": "Table 2 and Section 4.2 show that even the best model (o1-mini) scores only 60.3 on time efficiency, and most models score below 45% on modularity. Models like Qwen2.5-Coder-7B-Ins have competitive correctness but lag significantly on other dimensions compared to GPT-4o-mini.",
       "supported": "strong"
     },
     {
       "claim": "The RACE benchmark serves as an effective tool for resisting the risk of data contamination.",
       "evidence": "Table 3 (Section 4.3) shows that while traditional benchmarks reach 85%+ accuracy under severe contamination (8 epochs), the RACE overall score increases much more slowly (from ~20% to ~43%), and the instruction-following rate consistently remains below 10% compared to the clean model.",
       "supported": "strong"
     },
     {
       "claim": "Even the most advanced code LLMs struggle with complex instructions involving multiple customization requirements, with performance deteriorating significantly as requirements increase.",
       "evidence": "Figure 5 (Section 4.4) shows that when requirements increase from 2 to 5, the Acc.IF rate approaches 0 for almost all models, including Claude-3.5-Sonnet and GPT-4o.",
       "supported": "strong"
     },
     {
       "claim": "Most LLMs exhibit an inherent preference for specific coding styles, making it difficult to follow user instructions inconsistent with their preference.",
       "evidence": "Figure 6 (Section 4.5) shows that models like CodeLlama and WizardCoder have IF rates below 30% for camel-case naming despite high rates for snake-case. Most instruct-type LLMs see ~15% IF rate drop under stricter length constraints.",
       "supported": "strong"
     },
     {
       "claim": "Comments serve as an implicit chain-of-thought mechanism, enhancing the accuracy of generated code.",
       "evidence": "Figure 8 (Appendix A) shows that requiring function-level or line-level comments improves correctness for some models (e.g., GPT-4o, DS-V2.5-236B). However, this is presented as a hypothesis without controlled testing.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The RACE benchmark evaluates code generation across four dimensions (readability, maintainability, correctness, efficiency) rather than correctness alone, revealing that even top LLMs like o1-mini and Claude-3.5-Sonnet have significant deficiencies in non-correctness dimensions. The benchmark demonstrates robustness against data contamination: while traditional benchmarks see accuracy inflate to 85%+ under contamination, RACE's multi-dimensional evaluation remains relatively stable. A notable finding is that LLMs exhibit strong preference biases toward specific coding styles (e.g., snake-case in Python) and performance degrades sharply when asked to satisfy multiple simultaneous requirements, with Acc.IF rates approaching zero at five concurrent requirements.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity discussion. For a benchmark paper proposing a new evaluation framework, discussing the limitations of automated metrics for subjective dimensions like readability and maintainability is essential."
     },
     {
       "flag": "Python-only evaluation with broad claims",
       "detail": "All experiments are conducted solely in Python, but the paper makes general claims about 'code generation for LLMs' without bounding the findings to Python. The title and abstract do not indicate this restriction."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Despite making numerous comparative claims across 28 models, no significance tests, confidence intervals, or variance measures are reported. All results are single-run point estimates using greedy decoding."
     },
     {
       "flag": "Automated metrics for inherently subjective dimensions",
       "detail": "Readability and maintainability are measured entirely through automated proxies (AST analysis, MI formula) without any human validation that these automated metrics correlate with actual human judgment of code quality."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, one of the primary correctness benchmarks used in RACE and a foundational code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark, another key correctness dataset used in RACE evaluation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Created HumanEval+ and MBPP+ with enhanced test suites used in RACE for more rigorous correctness evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses contamination in code benchmarks through temporal splits, directly relevant to RACE's contamination resistance claims."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "relevance": "Directly studies data contamination in code generation benchmarks, providing context for RACE's contamination mitigation contribution."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Provides class-level code generation benchmark used for maintainability evaluation in RACE."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major open-source code LLM evaluated in RACE, representing state-of-the-art in code generation models."
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "authors": ["Dong Huang", "Jie M Zhang", "Yuhao Qing", "Heming Cui"],
       "year": 2024,
       "arxiv_id": "2402.02037",
       "relevance": "Evaluates efficiency of generated code, directly related to the efficiency dimension in RACE."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Comprehensive survey on LLMs for software engineering, providing context for why multi-dimensional code evaluation matters."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2024,
       "relevance": "Represents agentic approaches to code generation that benefit from multi-dimensional evaluation."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major code LLM family evaluated in RACE, demonstrating strong preference biases in coding style."
     },
     {
       "title": "WizardCoder: Empowering code large language models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2024,
       "relevance": "Code LLM using instruction tuning, evaluated in RACE and showing significant coding style preference biases."
     }
   ]
 }