ai-research-survey

scan.json (27783B)

---

 {
   "paper": {
     "title": "CodeBenchGen: Creating Scalable Execution-Based Code Generation Benchmarks",
     "authors": ["Yiqing Xie", "Alex Xie", "Divyanshu Sheth", "Pengfei Liu", "Daniel Fried", "Carolyn Rosé"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2404.00566"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/yiqingxyq/CodeBenchGen (footnote 1, Section 1: 'Code and dataset available at https://github.com/yiqingxyq/CodeBenchGen')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same footnote states 'Code and dataset available at https://github.com/yiqingxyq/CodeBenchGen.' The Exec-CSN dataset is released. The source data (CodeSearchNet) is also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section listing specific library versions is provided in the paper. The paper mentions using various models and libraries but does not specify a reproducible environment configuration."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology at a high level but does not include step-by-step reproduction instructions such as specific commands to run. The GitHub repository may contain these, but the paper itself does not provide them."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All Pass@k results in Table 4 are reported as point estimates without confidence intervals or error bars. The human study results in Figure 7 also lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses logistic regression analysis (Appendix A.6) with significance tests to assess which factors affect model performance. Pearson-r correlations with p-values are reported in Section 4.2 (e.g., 'Pearson-r correlation coefficient for number of variables is 0.73, with p-value < 0.001'). However, the main claim that humans outperform models after revision (Table 5) lacks a formal significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context throughout. For example, Pass@1 scores with baselines (GPT-4 at 37.21% vs. DeepSeek-Coder-33B at 34.00%), Pearson-r correlation coefficients (0.51, 0.58, 0.73), BLEU score of 0.5116 vs. random pair BLEU of 0.0052, and Jaccard similarity of 83%. These provide sufficient context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The human study uses only 64 examples evaluated by graduate student volunteers, but no justification is given for this sample size. No power analysis is discussed. The number of human participants is not even clearly stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Pass@k results are reported as single values without standard deviations across runs. The paper does not report variance or confidence intervals for any experimental results. Table 3 reports average and [min, max] for dataset statistics but no standard deviation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 12 models (10 open-source, 2 proprietary) against each other in Table 4. It also compares the CodeBenchGen framework against RepoEval and R2E approaches for benchmark creation (Figure 1, Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-4, Llama-3 (2024), CodeQwen1.5, and DeepSeek-Coder, which were contemporary at the time of writing. The benchmark construction comparison includes R2E (Jain et al., 2024) and SWE-BENCH (Jimenez et al., 2024), both concurrent/recent."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like analyses: (1) iterative debugging impact shown in Table 1 (examples increasing from 1,260 to 2,343 across 3 debugging iterations), (2) test augmentation study in Table 7 comparing original vs. augmented tests with different models, (3) comparison of instruction types (generated vs. docstring) in Figure 7."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pass@1, Pass@2, Pass@5, and Pass@10 (Table 4). It also uses BLEU score, Jaccard similarity, AST depth correlation, Pearson-r, line coverage rate, and Likert scale ratings for different aspects of the evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents a human study with computer science graduate students evaluating 64 examples for solvability, difficulty, instruction clarity, and test case quality. Section 5.3 also compares human vs. model performance on code generation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The Exec-CSN dataset is derived from the test split of CodeSearchNet ('We sample 5,000 Python examples from the test split of CodeSearchNet', Section 3). The benchmark itself serves as the test set and was not used to tune any of the evaluated models."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 provides detailed breakdowns by target length, context length, number of function calls, type of libraries, and specific external libraries. The logistic regression in Appendix A.6 further breaks down performance by multiple factors."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses the CodeLlama-70B failure mode where it refuses to generate code due to alignment training (~8% of cases). Appendix A.5 provides a detailed case study comparing human and GPT-4 revision strategies, showing GPT-4's 'stubborn' behavior of repeating the same wrong answer."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT-4 fails to improve beyond 57.81% even with 4 revisions while humans reach 81.25% (Table 5). It also reports that GPT-4 sometimes repeats the same wrong answer across revisions (Appendix A.5), and that smaller models generate worse tests (Table 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 1,931 examples from 367 repos (Table 1, Section 3), 81.3% solvable by humans (Figure 7), 61% rated as 'requires effort' (Figure 7 shows 44% medium + 17% hard = 61%), and GPT-4 achieves 37.21% Pass@1 (Table 4)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation-style analysis (e.g., 'models receive overall lower scores on examples with longer target lengths'), supported by controlled factor analysis in the logistic regression (Appendix A.6) and the performance breakdown in Figure 8. The debugging iteration analysis (Table 1) demonstrates causal impact of the iterative process."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Creating Scalable Execution-Based Code Generation Benchmarks' implies broad applicability, but the framework is only demonstrated on Python code from CodeSearchNet. The paper does not explicitly bound generalization to Python or to the specific types of code in CodeSearchNet. Claims about model capabilities on 'diverse real-world tasks' are based solely on Python function completion."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several alternative explanations: LLM self-bias in test generation (Appendix A.3, Table 8), the CodeLlama alignment training causing refusals rather than inability (Section 5.2), and that context length's positive effect on performance may be because 'longer context provides more information' rather than being inherently easier (Section 5.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 (Appendix A.4) provides full checkpoint names for all models: e.g., 'gpt-4-0125-preview', 'gpt-3.5-turbo-0125', 'Meta-Llama-3-8B-Instruct', 'DeepSeek-Coder-7B-instruct-v1.5'. These include version identifiers sufficient for reproduction."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.8 provides a full example of the code generation prompt used for evaluation, including the boilerplate instruction, context, and function header format. The prompt structure is fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states: 'we sample outputs with a temperature of 0.3 and top-p of 0.95.' For test augmentation (Appendix A.2): 'temperature of 0.3 and top-p of 0.7.' Number of samples per example is also specified (20 for open-source, 10 for closed-source)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the multi-step LLM scaffolding in detail across Section 2: sandboxing (Step 1), test generation (Step 2), iterative execution and debugging (Step 3), and post-processing (Step 4). Figure 4 provides a visual workflow. Retry logic and filtering criteria are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the data pipeline: starting from 5,000 Python examples from CodeSearchNet test split, filtering invalid file links and I/O operations (4,079 remaining), then the framework's multi-step processing (Table 1 shows counts at each step), post-filtering for safety (Table 6, Appendix A.2), resulting in 1,931 final examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section in the paper. Section 7 is 'Conclusions and Future Works' which briefly mentions future directions but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address issues such as the human study's small sample size (64 examples), potential LLM bias in the benchmark construction, or the representativeness of CodeSearchNet as a source."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to Python, to the specific types of code in CodeSearchNet, or acknowledge that the framework has only been demonstrated with GPT-4 as the sandbox LLM."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset (Exec-CSN) is released at the GitHub repository. The source data (CodeSearchNet) is publicly available. Multiple examples are shown in Appendix A.7 and A.9."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection: 5,000 Python examples sampled from CodeSearchNet test split, corresponding GitHub code files downloaded, filtering for valid file links and I/O operations. The framework's conversion process is described in Section 2."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human study mentions 'computer science graduate student volunteers' (Section 4.4) but does not describe how they were recruited, how many participants there were, from which institution, or whether recruitment could introduce bias. The number of participants is not explicitly stated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the pipeline with counts at each stage: 4,079 input examples -> 1,260 after sandboxing & test gen -> 1,973/2,155/2,343 after debugging iterations -> 1,931 after post-processing. Appendix A.2 provides additional filtering details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Acknowledgement) states: 'This work was supported in part by NSF grant DSES 2222762.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University (5 authors) and Shanghai Jiao Tong University (1 author). No author is affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is NSF (National Science Foundation), a government agency with no financial interest in the performance of any particular code generation model. NSF is independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple pre-trained models on the Exec-CSN benchmark but does not state training data cutoff dates for any of the models. This is relevant because CodeSearchNet data is publicly available and could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the CodeSearchNet data (which is public and widely used) could have been in the training data of the evaluated models. This is a significant concern since CodeSearchNet is a well-known dataset."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeSearchNet was published in 2020, and all evaluated models were trained after 2020. The adapted code maintains high similarity to the original (83% Jaccard similarity for variables), meaning models trained on CodeSearchNet could have memorized solutions. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human study involving graduate student participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participants are described only as 'computer science graduate student volunteers' (Section 4.4). No further demographics (experience level, years of programming, gender, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for participants. It is unclear how 'computer science graduate student volunteers' were screened or selected."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 64 examples were assigned to participants. It is unclear whether examples were randomly assigned or if participants chose examples."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not clearly applicable to this study design where participants solve coding problems. There is no comparison between conditions that would require blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 'obtained results on 64 examples in total' but does not report how many participants were initially recruited, whether any dropped out, or whether any examples were excluded from analysis."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report inference costs, API costs, or total tokens consumed for either the benchmark construction (which uses GPT-4 extensively) or the model evaluation experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper uses GPT-4 for benchmark construction (4,079 examples with multiple retries and debugging iterations) and runs 12 models for evaluation, but does not quantify the compute required."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeBenchGen successfully converts 47% of source code fragments into evaluation examples with executable test cases, covering 90% of input repositories.",
       "evidence": "Table 1 shows 1,931 examples from 4,079 inputs (47%). Section 3 states coverage of 367 out of 408 repositories (90%).",
       "supported": "strong"
     },
     {
       "claim": "81.3% of Exec-CSN examples are solvable by human computer science graduate students.",
       "evidence": "Figure 7 reports 81.3% solved rate from a human study on 64 examples. However, the study uses an unspecified number of participants on only 64/1,931 examples (3.3%).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 achieves the best Pass@1 score of 37.21% on Exec-CSN, indicating substantial room for improvement.",
       "evidence": "Table 4 shows GPT-4 at 37.21% Pass@1, the highest among all 12 evaluated models.",
       "supported": "strong"
     },
     {
       "claim": "Models perform worse on examples with longer target lengths, more function calls, or external libraries.",
       "evidence": "Figure 8 shows consistent performance degradation across these factors for both GPT-4 and DeepSeek-Coder-33B. Logistic regression in Appendix A.6 confirms TargetLength and NumberImports are significant predictors.",
       "supported": "strong"
     },
     {
       "claim": "Humans achieve significantly better scores than GPT-4 after several rounds of revision, despite GPT-4 having higher initial Pass@1.",
       "evidence": "Table 5 shows GPT-4 Pass@1 of 40.63% vs. human 34.38%, but after 4 revisions humans reach 68.75% vs. GPT-4's 57.81%. Humans ultimately reach 81.25%. However, this is based on only 64 examples with no statistical test.",
       "supported": "moderate"
     },
     {
       "claim": "Exec-CSN covers more diverse libraries, repository topics, and contributor distributions than existing benchmarks.",
       "evidence": "Table 2 and Figure 5 compare Exec-CSN (367 repos, 668 topics, 293 libraries) to RepoEval (8 repos, 23 topics, 75 libraries) and SWE-BENCH (12 repos, 60 topics, 214 libraries). Figure 6 shows contributor distribution is closer to natural distribution.",
       "supported": "strong"
     },
     {
       "claim": "The adapted code preserves high similarity to the input code, with average Jaccard similarity of variables at 83%.",
       "evidence": "Section 4.2 reports BLEU score of 0.5116, Jaccard similarity of 83% for variables, and Pearson-r correlations of 0.51-0.73 for code tokens, AST depth, and variables. Figure 10 visualizes the Jaccard similarity.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeBenchGen is a framework that uses LLMs to convert arbitrary code fragments into execution-based evaluation examples, achieving 47% conversion rate and 90% repository coverage from CodeSearchNet. The resulting Exec-CSN benchmark (1,931 examples, 367 repositories) is substantially more diverse than existing benchmarks like RepoEval and SWE-BENCH in terms of libraries, topics, and contributor diversity. GPT-4 achieves only 37.21% Pass@1, with performance degrading for longer targets and external library usage. Humans outperform GPT-4 after iterative revision (81.25% vs. 57.81%), demonstrating better error-message utilization.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CodeSearchNet is a widely-used public dataset (published 2020). All evaluated models were trained after 2020 and may have seen this data. The adapted code maintains 83% Jaccard similarity to the original, meaning memorized solutions could still apply. The paper does not discuss this contamination risk at all."
     },
     {
       "flag": "Small and undescribed human study sample",
       "detail": "The human study evaluates only 64 out of 1,931 examples (3.3%) with an unspecified number of 'graduate student volunteers.' No demographics, recruitment methods, inclusion criteria, randomization, or IRB approval are reported. The claim of 81.3% solvability is generalized from this small, poorly-characterized sample."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Key limitations such as Python-only evaluation, dependence on GPT-4 quality for benchmark construction, potential LLM bias in generated tests, and CodeSearchNet representativeness are not discussed."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "All Pass@k results in Table 4 are reported as point estimates without confidence intervals, error bars, or variance across runs. The human-vs-model comparison (Table 5) makes performance claims without any statistical test."
     },
     {
       "flag": "Circular LLM dependence in benchmark",
       "detail": "GPT-4 is used to construct the benchmark (sandboxing, test generation, debugging) and is also one of the evaluated models. Although GPT-3.5 is used for test augmentation to reduce self-bias, the fundamental benchmark structure is shaped by GPT-4's capabilities, which could systematically favor GPT-4 in evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational execution-based code generation benchmark and the Pass@k evaluation metric used throughout this paper."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "A major repository-level code generation benchmark that CodeBenchGen explicitly compares against for diversity and scalability."
     },
     {
       "title": "R2E: Turning any github repository into a programming agent environment",
       "authors": ["Naman Jain", "Manish Shetty", "Tianjun Zhang"],
       "year": 2024,
       "relevance": "A concurrent approach to repository-based benchmark creation that CodeBenchGen compares against for scalability."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Proposes EvalPlus with improved test coverage for code generation evaluation, directly relevant to benchmark quality methodology."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming -- the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "relevance": "One of the primary code generation models evaluated in this paper, representing state-of-the-art open-source code models."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li", "Yiming Wang", "Tianyi Zhang"],
       "year": 2023,
       "relevance": "A manually-created data science code generation benchmark, representing the category of execution-based benchmarks with limited scalability."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang"],
       "year": 2023,
       "relevance": "Introduces RepoEval benchmark for repository-level code completion, a key comparison point for dataset diversity."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Large-scale execution-based benchmark sourced from coding contests, representing hand-curated benchmark approaches."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2024,
       "relevance": "Describes the Code Llama family of models evaluated in this paper, including the alignment-related refusal behavior observed."
     },
     {
       "title": "Grounding data science code generation with input-output specifications",
       "authors": ["Yeming Wen", "Pengcheng Yin", "Kensen Shi"],
       "year": 2024,
       "relevance": "Demonstrates that I/O specifications improve model performance on code generation, directly motivating the instruction design in CodeBenchGen."
     },
     {
       "title": "ChatUniTest: A ChatGPT-based automated unit test generation tool",
       "authors": ["Zhuokui Xie", "Yinghao Chen", "Chen Zhi"],
       "year": 2023,
       "relevance": "LLM-based test generation approach that inspires the test generation strategy used in CodeBenchGen."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "relevance": "Extends execution-based code generation evaluation to multiple programming languages."
     }
   ]
 }