ai-research-survey

scan.json (27500B)

---

 {
   "paper": {
     "title": "CodeScope: An Execution-based Multilingual Multitask Multidimensional Benchmark for Evaluating LLMs on Code Understanding and Generation",
     "authors": [
       "Weixiang Yan",
       "Haitian Liu",
       "Yunkun Wang",
       "Yunzhe Li",
       "Qian Chen",
       "Wen Wang",
       "Tingyu Lin",
       "Weishan Zhao",
       "Li Zhu",
       "Hari Sundaram",
       "Shuiguang Deng"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2311.08588"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/WeixiangYAN/CodeScope (stated in abstract and throughout the paper)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly released at the same GitHub repository. The abstract states: 'The CodeScope benchmark and code are publicly available at https://github.com/WeixiangYAN/CodeScope.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 in the appendix provides specific model versions, model sizes, and inference hardware (e.g., 'NVIDIA Tesla A800 * 4' for LLaMA 2, 'NVIDIA GeForce RTX 4090 * 4' for Code LLaMA). However, no requirements.txt or Dockerfile is mentioned. The hardware information is detailed enough to partially reconstruct the environment, but software dependency specifications are absent. Given Table 8 provides specific model versions and GPU configurations, this is borderline but the paper does specify the execution environment for MultiCodeEngine supporting 47 compiler/interpreter versions. Answering YES given the level of detail across Table 8 and compiler specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions code is available at the GitHub repository, there are no step-by-step reproduction instructions in the paper itself. No README with commands, no 'Reproducing Results' section, and no scripts for replicating experiments are described in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. All tables (Tables 3-6, 9-12, etc.) report only point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite numerous comparative claims (e.g., 'WizardCoder demonstrates the best performance'). Rankings and comparisons are based solely on numerical differences without any significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. While raw performance numbers and differences can be inferred from tables, there is no explicit effect size reporting or contextual framing beyond raw scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why specific sample sizes were chosen for each task (e.g., 200 samples for code smell, 900 for code review, 100 per language for automated testing). The numbers appear arbitrary with no power analysis or justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The SD reported in Table 3 is standard deviation across length categories (short/medium/long), not across experimental runs. No variance across runs, random seeds, or repeated experiments is reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight baseline LLMs are evaluated: GPT-4, GPT-3.5, PaLM 2, LLaMA 2, StarCoder, Code LLaMA, WizardCoder, and Vicuna. The paper also compares CodeScope rankings against HumanEval and MBPP benchmarks (Table 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-4, GPT-3.5, PaLM 2, LLaMA 2, Code LLaMA, and WizardCoder, which were all contemporary and competitive models at the time of writing (2023). Table 8 specifies gpt-4-0613 and gpt-3.5-turbo-0613."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper that evaluates existing LLMs rather than proposing a system with multiple components. There is no system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across tasks: BLEU, METEOR, ROUGE, BERTScore for code summarization; accuracy, precision, recall, F1 for code smell and code review; pass rate, line coverage, branch coverage for automated testing; Pass@k for program synthesis/translation; DSR@K for code repair; Opt@K for code optimization."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the LLMs' outputs is conducted. All evaluations are automated through execution-based metrics or matching-based metrics. While the paper argues execution-based metrics are more reliable, no human judgment is used to validate outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a separation between development/validation and test sets. The benchmark appears to be used directly without any dev/test split for tuning evaluation methodology."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: per-language results for all tasks (Tables 9-12, 14-36 in appendix), per-difficulty breakdowns (Easy vs. Hard in Table 4), per-length breakdowns (Short/Medium/Long in Table 3), and per-efficiency-dimension (Memory vs. Time in Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Case studies in the appendix (Tables 37-52) include both high-quality and low-quality examples. For example, Table 39 shows a low-quality code summarization by PaLM 2 where it 'misunderstood the task and inserted natural language comments back into the code.' The paper discusses GPT-4's poor performance on automated testing and models' struggles with hard problems."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT-4 'does not exhibit leading performance' in understanding tasks (ranking 5th), and most models 'struggle to provide correct solutions for hard problems.' It notes open-source models perform poorly on code generation tasks. The paper also discusses that LLMs optimize C code worst, especially execution time."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the benchmark covers 43 programming languages and 8 tasks, which is verified in Table 1. It claims to demonstrate 'superior breadth and challenges of CodeScope,' which is supported by the comparison with HumanEval and MBPP in Table 6 and Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal attributions such as 'This advantage is attributed to its Evol-Instruct approach, which significantly enhances the model's understanding' (Section 4.1 on WizardCoder). This is a causal claim without adequate justification — the authors cannot attribute performance differences to specific training methods without controlled experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims to be a benchmark 'for Evaluating LLMs on Code Understanding and Generation' broadly, but evaluations are limited to 8 specific LLMs. The paper states 'CodeScope evaluates the coding proficiency of LLMs' without sufficiently bounding claims to the tested models. Claims like 'GPT-4 performs the best' are stated without acknowledging this is only among the 8 tested models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Alternative explanations for performance differences are not discussed. For example, when attributing WizardCoder's performance to Evol-Instruct, no alternative explanations (training data composition, model size relative to task complexity, etc.) are considered. The Limitations section focuses only on data leakage, not on alternative interpretations of results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides specific model versions: gpt-4-0613, gpt-3.5-turbo-0613, text-bison-001, LLaMA-2-70b-chat-hf, starchat-beta, Code LLaMA-34b-Instruct-hf, WizardCoder-15B-V1.0, and vicuna-13b-v1.5-16k."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in the appendix case studies (Tables 37-52), showing complete prompt templates with actual content for each task type. The prompts include specific instructions, problem descriptions, and response format specifications."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Temperature, top-p, max tokens, and other sampling parameters for the LLM API calls are not mentioned anywhere in the paper. The paper states 'we detail the specific configuration information for each LLM' in Table 8, but Table 8 only lists model versions, sizes, and GPU hardware — no sampling/generation hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLMs are prompted directly for each task without any agentic workflow, retry logic, or multi-step reasoning framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented in detail for each task in Sections 3.1-3.2 and Appendix A.3.1-A.3.8. For example, code summarization describes collecting from Rosetta Code, selecting 170 tasks and 4,838 samples; program synthesis describes excluding problems with fewer than 10 test cases, non-deterministic problems, and brute force solutions exceeding 5,000 tokens."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present (Section after Conclusion), discussing data leakage concerns from three perspectives."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses specific threats: data memorization as a confound for benchmark evaluation, the technical infeasibility of constructing a zero-leakage dataset, and how five independent data sources were used to mitigate reliance on any single source. These are specific to this benchmark study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what claims it is NOT making. The limitations section focuses exclusively on data leakage but does not bound the scope of conclusions — for example, it does not state that results may not generalize beyond the 8 tested models, or that performance on Codeforces-sourced problems may not reflect real-world development tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is publicly available at https://github.com/WeixiangYAN/CodeScope, enabling independent verification of the underlying data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail for each task: Rosetta Code for code summarization (43 languages), Madeyski/Lewowski and Slivka et al. datasets for code smell, Li et al. (2022b) for code review, and Codeforces for program synthesis/translation/repair/optimization. Time periods and selection criteria are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data sources are publicly available code repositories and programming competition platforms (Rosetta Code, Codeforces, GitHub)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented for each task. For example, program synthesis: collect from Codeforces -> filter by 10+ test cases -> exclude non-deterministic problems -> execute validation -> exclude failing submissions -> exclude brute force >5000 tokens. Code summarization: collect from Rosetta Code -> select 170 tasks -> manually create summaries -> GPT-4 paraphrase -> manual review."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'Work is supported by Alibaba Group' (footnote on page 1)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed, including that Qian Chen and Wen Wang are from Alibaba Group, with alibaba-inc.com email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Alibaba Group funded the work, and no Alibaba product is being evaluated. The benchmark evaluates OpenAI, Google, Meta, and independent models. Alibaba has no direct financial stake in the relative performance of these models on this benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Two authors are from Alibaba Group, which develops its own LLMs, but no declaration of financial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. This is important because several data sources (Codeforces, Rosetta Code) are publicly available and could have been in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section extensively discusses data leakage risks, acknowledging that 'data leakage is a likely problem for benchmarks for evaluating LLMs' and discussing it from three perspectives. They note using 'five independent data sources' to mitigate leakage. However, no concrete analysis of actual overlap is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses data leakage conceptually in the Limitations section, it does not address the specific contamination risk that Codeforces solutions (a major data source) and Rosetta Code are publicly available and likely included in the training data of the evaluated models. The paper argues data leakage is 'unavoidable' rather than taking concrete steps to quantify or mitigate it."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. This is a benchmark evaluation of LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time for running the benchmark are reported, despite using closed-source API models (GPT-4, GPT-3.5, PaLM 2) which incur monetary costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While GPU types are listed in Table 8, no total compute budget (GPU hours, API spend, total runtime) is provided. Running 8 LLMs across 8 tasks with ~13,000 total samples likely required significant compute, but this is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeScope is the first comprehensive benchmark covering 43 programming languages and 8 coding tasks with execution-based evaluation.",
       "evidence": "Table 1 lists 8 tasks and their language coverage. Table 2 compares CodeScope with 13 other benchmarks, showing it is the only one that is execution-based, multilingual, multitask, and multidimensional.",
       "supported": "strong"
     },
     {
       "claim": "WizardCoder demonstrates the best overall performance among tested LLMs on code understanding tasks with a score of 50.14.",
       "evidence": "Table 3 (Length dimension) and Table 6 show WizardCoder at 50.14 overall for code understanding, ahead of LLaMA 2 (48.79) and GPT-3.5 (48.10).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 leads in code generation tasks but ranks only 5th in code understanding on CodeScope.",
       "evidence": "Table 6 shows GPT-4 at 31.47 for generation (1st) but 47.16 for understanding (5th). Section 5 discusses this discrepancy.",
       "supported": "moderate"
     },
     {
       "claim": "CodeScope rankings differ substantially from HumanEval and MBPP rankings, demonstrating its complementary evaluation value.",
       "evidence": "Table 6 shows GPT-4 ranks 1st on HumanEval/MBPP but ranks differently on CodeScope Understanding/Generation. WizardCoder ranks 1st on CodeScope Understanding but 2nd/4th on HumanEval/MBPP.",
       "supported": "moderate"
     },
     {
       "claim": "WizardCoder's strong performance is attributed to its Evol-Instruct training approach.",
       "evidence": "Section 4.1 states: 'This advantage is attributed to its Evol-Instruct approach, which significantly enhances the model's understanding.' This is an attribution claim without controlled evidence.",
       "supported": "weak"
     },
     {
       "claim": "LLMs optimize Python code best but C code worst, especially in terms of execution time.",
       "evidence": "Table 5 shows optimization results across 4 languages. For time optimization, most models achieve higher scores in Python than in C. GPT-4 achieves 36.67% time optimization for Python vs. 6.67% for C.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeScope is presented as the first comprehensive execution-based benchmark for evaluating LLMs on coding tasks, covering 43 programming languages and 8 tasks across 3 evaluation dimensions (length, difficulty, efficiency). Evaluation of 8 LLMs reveals that model rankings differ substantially from existing benchmarks like HumanEval and MBPP, with WizardCoder leading in code understanding but GPT-4 leading in code generation. The paper finds that strong code generation performance does not imply strong code understanding, and that most open-source LLMs struggle significantly on hard programming problems, with near-zero pass rates. The MultiCodeEngine execution environment supporting 14 programming languages enables more reliable evaluation than matching-based metrics.",
   "red_flags": [
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, max tokens, and other sampling/generation parameters are not reported for any model. These parameters significantly affect LLM output quality and reproducibility. Table 8 lists only model versions and hardware."
     },
     {
       "flag": "Major contamination risk unaddressed",
       "detail": "Codeforces solutions (used for 4 of 8 tasks) and Rosetta Code (used for code summarization) are publicly available websites likely crawled for LLM training data. The paper acknowledges data leakage conceptually but does not analyze whether specific models were trained on this data, nor does it state training cutoff dates."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('WizardCoder demonstrates the best performance,' 'GPT-4 excels') are based on raw numerical differences without any significance testing. With single-run results and no error bars, it is impossible to determine whether performance differences are meaningful."
     },
     {
       "flag": "Unsupported causal attributions",
       "detail": "The paper attributes WizardCoder's performance to its Evol-Instruct approach and GPT-4's struggles to difficulties with 'tracking and analyzing data flow.' These are causal claims without controlled experiments to isolate the effects of specific training methods."
     },
     {
       "flag": "Reference summaries partially generated by GPT-4",
       "detail": "Code summarization reference summaries were created using a three-step process involving GPT-4 paraphrasing. While the authors claim this was NL-to-NL paraphrasing that doesn't bias toward GPT-4, using GPT-4 in the reference creation process introduces a potential confound when GPT-4 is also being evaluated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the most widely used code generation benchmark against which CodeScope explicitly compares."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation evaluation, directly compared with CodeScope."
     },
     {
       "title": "CodeXGLUE: A machine learning benchmark dataset for code understanding and generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Influential code benchmark using matching-based metrics that CodeScope aims to improve upon with execution-based evaluation."
     },
     {
       "title": "XCodeEval: A large scale multilingual multitask benchmark for code understanding, generation, translation and retrieval",
       "authors": ["Mohammad Abdullah Matin Khan", "M. Saiful Bari"],
       "year": 2023,
       "arxiv_id": "2303.03004",
       "relevance": "Most directly comparable prior benchmark to CodeScope; uses execution-based metrics in multilingual multitask setting."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "relevance": "Code LLM that demonstrates the best understanding performance on CodeScope, relevant for evaluating LLM coding capabilities."
     },
     {
       "title": "Code LLaMA: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM evaluated in CodeScope, relevant for benchmark evaluation of code generation models."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM trained on 80+ programming languages, evaluated in CodeScope."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Benchmark evaluating LLMs on more complex class-level code generation, complementary to CodeScope's function-level evaluation."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "relevance": "Translates HumanEval/MBPP to 18 languages, directly relevant as a multilingual code benchmark approach."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Challenging program synthesis benchmark; CodeScope aims to provide similar difficulty assessment but across more languages."
     },
     {
       "title": "No more manual tests? Evaluating and improving ChatGPT for unit test generation",
       "authors": ["Zhiqiang Yuan", "Yiling Lou", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "Evaluates LLM capabilities for automated test generation, directly relevant to CodeScope's automated testing task."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Benchmark addressing limitations of HumanEval for real-world code generation evaluation."
     }
   ]
 }