ai-research-survey

scan.json (27702B)

---

 {
   "paper": {
     "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
     "authors": [
       "Terry Yue Zhuo",
       "Minh Chien Vu",
       "Jenny Chim",
       "Han Hu",
       "Wenhao Yu",
       "Ratnadira Widyasari",
       "Imam Nur Bani Yusuf",
       "Haolan Zhan",
       "Junda He",
       "Indraneil Paul",
       "Simon Brunner",
       "Chen Gong",
       "Thong Hoang",
       "Armel Zebaze",
       "Xiaoheng Hong",
       "Wen-Ding Li",
       "Jean Kaddour",
       "Ming Xu",
       "Zhihan Zhang",
       "Prateek Yadav",
       "Naman Jain",
       "Alex Gu",
       "Zhoujun Cheng",
       "Jiawei Liu",
       "Qian Liu",
       "Zijian Wang",
       "Binyuan Hui",
       "Niklas Muennighoff",
       "David Lo",
       "Daniel Fried",
       "Xiaoning Du",
       "Harm de Vries",
       "Leandro von Werra"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2406.15877",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides multiple GitHub links for the evaluation framework (https://github.com/bigcode-project/bigcodebench), annotation framework (https://github.com/bigcode-project/bigcodebench-annotation), and Hugging Face dataset (https://huggingface.co/datasets/bigcode/bigcodebench). These are listed in Appendix H (Table 4)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is publicly available on Hugging Face (https://huggingface.co/datasets/bigcode/bigcodebench) and a Croissant endpoint is provided. Listed explicitly in Appendix H, Table 4."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I.3 provides a detailed requirements.txt with pinned library versions (e.g., pandas==2.0.3, scikit-learn==1.3.1, numpy==1.21.2, etc.). The evaluation framework is also available as a PyPI package (bigcodebench)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a PyPI-installable evaluation framework, prompt templates (Appendix K.3, Figures 14-17), execution hardware details (Appendix K.1-K.2), and the full dataset. The GitHub repositories contain documentation for reproducing the benchmark evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports Pass@1 and Pass@5 scores as point estimates. No confidence intervals or error bars are reported for the main results in Tables 6 and 7 or Figure 6."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'LLMs perform much worse on BigCodeBench-Instruct than BigCodeBench-Complete with an average decrease of 8.5%') but no statistical significance tests are applied to any of these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports performance differences with baseline context, e.g., 'the best performing LLM, GPT-4o, solves merely 60% of tasks on BigCodeBench-Complete and less than 50% on BigCodeBench-Instruct' compared to '97%' human performance. The 8.5% average decrease between Complete and Instruct is provided with both absolute scores visible in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 1,140 tasks, but there is no justification for why this number is sufficient. The human evaluation sample for quality validation uses only 33 tasks (Section 2.3) with no justification for this sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports greedy decoding results (single-run, deterministic) for the main experiments. For random sampling experiments (Appendix L), Pass@1 and Pass@5 are reported as single values with no standard deviation or variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively compares BigCodeBench against prior benchmarks (HumanEval, MBPP, DS-1000, ODEX, APPS) in Table 1 and evaluates 60 LLMs, including comparisons between instruction-tuned and base models, closed and open models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary state-of-the-art at time of writing: GPT-4o, Claude-3-Opus, Llama-3-70B, Qwen2-72B, DeepSeek-V2, and others from 2024. Benchmark comparisons include recent works like DS-1000 (2023) and SWE-bench (2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like analyses: comparing Complete vs. Instruct variants (Section 4.1), base vs. instruction-tuned models (Section 4.1), examining tool-level performance across domains (Section 4.2), and constructing BigCodeBench-Hard with specific filtering criteria (Appendix F)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pass@1 (greedy), calibrated Pass@1, Pass@1 (random sampling), and Pass@5 (random sampling) in Table 6. Domain-specific pass rates are also reported in Figure 7."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human annotators validated benchmark quality: 33 randomly sampled tasks were assigned to 11 annotators to write solutions, achieving 97% pass rate (Section 2.3). The benchmark construction involved 20 human annotators across three stages."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper, not a model training paper. The benchmark itself serves as a test set for models, and there is no train/validation/test split within BigCodeBench itself."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 provides per-domain breakdowns across 7 domains (General, Computation, System, Visualization, Time, Network, Cryptography). Section 4.2 and Table 2 provide tool-level analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses 'model laziness' where LLMs omit import statements. Section 4.2 and Table 2 analyze how models use different function calls and why this leads to failures. Appendix M provides qualitative failure examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: models fail on up to 40% of tasks, instruction-tuned LLMs omit essential details (Section 4.1), models perform worse on NL-oriented instructions (8.5% average decrease), and GPT-4 shows 'model laziness' with long prompts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'scores up to 60%' which matches GPT-4o's 0.602 Pass@1 in Table 6. The '97%' human performance claim is supported by Section 2.3 (32/33 tasks). The claim about 60 LLMs is supported by Table 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are generally well-supported through controlled comparisons: instruction tuning's effect is shown by comparing base vs. instruction-tuned versions of the same model (Section 4.1). The 'model laziness' claim is supported by the calibration mechanism that adds back missing imports."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'Code Generation' generally, but the benchmark is Python-only and covers only 139 libraries. While the Limitations section (Appendix G.1) acknowledges the Python-only limitation, the abstract and main text make broad claims about 'LLMs' capabilities without consistently scoping to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses alternative explanations for performance differences: lower BigCodeBench-Instruct scores could be due to ambiguity rather than capability gaps ('While it is possible that the lower verbosity may introduce more ambiguity'). The Saturation discussion in Appendix G.1 addresses why SWE-bench's low performance may be due to under-specified instructions rather than task difficulty."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H (Table 4) provides exact model versions for all 60 evaluated models: GPT-4o (gpt-4-turbo-2024-04-09), GPT-4 (gpt-4-0613), Claude-3-Opus (claude-3-opus-20240229), etc. HuggingFace model IDs are given for all open models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix K.3 provides the exact prompt templates used for each API provider (Figures 14-17). The data synthesis prompts are given in Appendix J.1. The benchmark tasks themselves serve as the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'greedy decoding for the main experiments in the zero-shot setting' and 'N=5 samples with a temperature of 0.8 and top-p of 0.95' for random sampling experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates LLMs in a direct code generation setting (zero-shot prompting) without agentic scaffolding. No retry logic, feedback mechanisms, or iterative refinement is used in the evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the three-stage construction pipeline in detail: data synthesis from ODEX seeds using GPT-4 (Section 2.1, starting with 4,718 samples), semi-automatic refactoring reducing to 1,223 tasks (Section 2.2), and human curation producing 1,140 final tasks (Section 2.3). Obfuscation and perturbation steps are also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G.1 contains a dedicated 'Limitations' subsection covering multilingualism, saturation, reliability, rigorousness, generalization, evolution, interaction, and diversity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this benchmark: 'BigCodeBench is Python-only' (multilingualism), 'some test cases are flaky' with 'uncontrollable changes of Pass@1 under 0.6%' (reliability), 'high test coverage for the ground-truth solutions... does not guarantee that any code generated by LLMs will be correctly assessed' (rigorousness). These are concrete, actionable threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G.1 explicitly states what BigCodeBench does NOT cover: other programming languages, private/emerging libraries, agentic interaction, out-of-distribution tools. The paper also explicitly positions against SWE-bench's repository-level scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full benchmark dataset with all 1,140 tasks, test cases, and ground-truth solutions is available on Hugging Face (https://huggingface.co/datasets/bigcode/bigcodebench). The annotation framework is also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the three-stage data collection in detail: seed examples from ODEX/StackOverflow, GPT-4-based synthesis with 2-shot in-context learning, human annotation by 20 authors over one year. The data synthesis prompt is provided in Appendix J.1."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides demographics of the 20 annotators (age distribution, Python experience, academic background). Section 2.2 states '13 authors as human annotators' and describes task assignment. Appendix A details each contributor's role."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is fully documented: 4,718 synthesized samples (Section 2.1) → program analysis filtering → 1,223 refactored tasks (Section 2.2) → human curation and cross-checking → 1,140 final tasks (Section 2.3). Each stage's filtering criteria and counts are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses: 'CSIRO's Data61 PhD Scholarships, the National Research Foundation... Investigatorship Grant (NRF-NRFI08-2022-0002), and Xiaoning Du's Google Research Scholar Program Award.' Computational resources from Sea AI Lab and MASSIVE are also acknowledged."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on the first page, including companies like Intel, Uber, Sea AI Lab, AWS AI Labs, Contextual AI, ServiceNow Research, and Hugging Face. One author has a specific disclaimer: '∗The work does not relate to the author's position at Amazon.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The disclosed funders (CSIRO Data61, National Research Foundation Singapore, Google Research Scholar) are research funding bodies without a direct stake in the benchmark results. The note that AWS AI Labs work 'does not relate to the author's position at Amazon' further demonstrates independence awareness."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Several authors are affiliated with companies whose models are evaluated (e.g., Hugging Face authors evaluating StarCoder2), but no formal competing interests declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the 60 evaluated models. While some model references point to papers with this information, the paper itself does not consolidate or discuss cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides a dedicated 'Data Contamination' section with N-gram overlap experiments (10-gram and 13-gram) against ODEX, Stack Overflow, and StarCoderData. Table 3 shows overlap percentages are below 2.5%. The paper also discusses hosting on Hugging Face rather than GitHub to mitigate contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes obfuscation (replacing function names with dummy names) and perturbation (back-translation of docstrings) to mitigate contamination. Appendix D provides quantitative contamination analysis. The paper discusses future contamination risks from API providers collecting inference data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human involvement is for benchmark construction (annotation), not a human subjects study. Pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The human involvement is expert annotation for benchmark construction, not a human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study is conducted. The annotator demographics in Appendix B (age, experience, education) describe benchmark creators, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study is conducted. Annotators are co-authors, not study participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report the cost of running inference on 60 models across 1,140 tasks. No API costs, token counts, or wall-clock times for the evaluation are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix K states 'A100 GPUs' for inference and 'Intel(R) Xeon(R) Gold 6150 CPU' for execution, no total GPU hours, API spend, or overall computational budget is reported. The paper acknowledges 'limited computational resources' (Section 4) but does not quantify them."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o achieves the highest Pass@1 of 60.2% on BigCodeBench-Complete (calibrated: 61.1%), significantly below human performance of 97%.",
       "evidence": "Table 6 shows GPT-4o at 0.602 original and 0.611 calibrated Pass@1. Section 2.3 reports 97% human performance (32/33 tasks passed).",
       "supported": "strong"
     },
     {
       "claim": "LLMs perform substantially worse on BigCodeBench-Instruct than BigCodeBench-Complete, with an average decrease of 8.5% on Pass@1.",
       "evidence": "Table 7 shows the delta between Instruct and Complete variants. Section 4.1 reports the 8.5% average decrease, with individual model deltas visible in the table.",
       "supported": "strong"
     },
     {
       "claim": "Instruction-tuned LLMs omit essential import statements due to 'model laziness' with long prompts.",
       "evidence": "Section 4.1 describes this phenomenon. Figure 6 shows calibrated vs. original Pass@1 differences, with GPT-4 showing the largest gap (0.484 original vs. 0.572 calibrated on Complete). The difference is less than 0.3% on Instruct (shorter prompts).",
       "supported": "strong"
     },
     {
       "claim": "Instruction tuning improves the capability of following complex programming constraints, with instruction-tuned LLMs averaging 40.7% vs. 35.7% for base LLMs.",
       "evidence": "Section 4.1 provides the comparison between instruction-tuned and base model means. Table 6 shows paired comparisons (e.g., CodeLlama-instruct-70B at 0.496 vs. CodeLlama-base-70B at 0.443).",
       "supported": "strong"
     },
     {
       "claim": "BigCodeBench is the first benchmark combining diverse function calls across 139 libraries and 7 domains with complex instructions for 1,140 tasks.",
       "evidence": "Table 1 compares BigCodeBench statistics with prior benchmarks. The benchmark covers 723 function calls from 139 libraries, more than any compared benchmark.",
       "supported": "strong"
     },
     {
       "claim": "BigCodeBench has minimal data contamination, with less than 2.5% overlap in N-gram analysis.",
       "evidence": "Appendix D, Table 3 shows 10-gram overlap of 0.09% with ODEX, 1.49% with Stack Overflow, and 2.49% with StarCoderData. 13-gram overlap is 0.00% with ODEX and 0.18% with Stack Overflow.",
       "supported": "moderate"
     },
     {
       "claim": "Model rankings show strong positive correlation between BigCodeBench-Complete and BigCodeBench-Instruct (Pearson's r = 0.982).",
       "evidence": "Section 4.1 reports the Pearson's r correlation. The model rankings in Figure 6 are visually consistent.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BigCodeBench is a benchmark of 1,140 Python programming tasks requiring diverse function calls across 139 libraries and 7 domains, constructed through human-LLM collaboration. Evaluation of 60 LLMs shows the best model (GPT-4o) achieves only 60% Pass@1, far below 97% human performance. LLMs perform worse on natural-language-oriented instructions (8.5% average decrease) and exhibit 'model laziness' by omitting essential code when given long prompts. The benchmark reveals that while models generally follow scaling laws, domain-specific performance varies significantly.",
   "red_flags": [
     {
       "flag": "No error bars or variance across evaluation runs",
       "detail": "Main results use greedy decoding (deterministic, single-run). The random sampling results (Pass@1 and Pass@5 with temperature=0.8) also report single point estimates without variance across seeds or runs, making it impossible to assess result stability."
     },
     {
       "flag": "Human performance estimate from small, non-independent sample",
       "detail": "The 97% human performance claim is based on only 33 tasks (out of 1,140) evaluated by the benchmark's own annotators (11 of the authors). This sample is small and the evaluators are not independent of the benchmark creators, which could inflate the human baseline."
     },
     {
       "flag": "Potential conflict of interest with evaluated models",
       "detail": "Several authors are affiliated with organizations whose models are evaluated: Hugging Face (StarCoder2), ServiceNow Research, and others. While affiliations are disclosed, no formal competing interests statement addresses this."
     },
     {
       "flag": "Benchmark constructed partially with GPT-4, which is also evaluated",
       "detail": "GPT-4 (gpt-4-0613) was used for data synthesis and program refactoring, and is also one of the evaluated models. While the paper applies obfuscation and perturbation to mitigate self-preference bias, the model may still have subtle advantages on tasks it helped construct."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, one of the most widely used code generation benchmarks that BigCodeBench aims to supersede."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "relevance": "Repository-level code generation benchmark that evaluates agent-level capabilities, complementary to BigCodeBench's function-level evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses data contamination in code benchmarks through temporal splits, directly relevant to benchmark methodology quality."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "EvalPlus framework for rigorous code evaluation, used as inspiration for BigCodeBench's evaluation approach."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li", "Yiming Wang"],
       "year": 2023,
       "relevance": "Domain-specific code generation benchmark for data science, one of BigCodeBench's key comparison baselines."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open code LLM family evaluated in BigCodeBench, relevant to understanding code model capabilities."
     },
     {
       "title": "Starcoder 2 and the stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open code LLM and training data used for contamination analysis in BigCodeBench."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Agent-based approach to software engineering tasks, representing the agentic paradigm that BigCodeBench's function-level evaluation complements."
     },
     {
       "title": "Magicoder: Empowering code generation with oss-instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2024,
       "relevance": "Instruction-tuned code model using synthetic data, evaluated on BigCodeBench and relevant to understanding training data effects on code generation."
     },
     {
       "title": "SciCode: A research coding benchmark curated by scientists",
       "authors": ["Minyang Tian", "Luyu Gao", "Dylan Zhang"],
       "year": 2024,
       "relevance": "Scientific code generation benchmark representing domain-specific evaluation that extends beyond BigCodeBench's scope."
     },
     {
       "title": "Tool learning with foundation models",
       "authors": ["Yujia Qin", "Shengding Hu", "Yankai Lin"],
       "year": 2023,
       "arxiv_id": "2304.08354",
       "relevance": "Survey on tool-use capabilities of LLMs, directly relevant to BigCodeBench's focus on function-call-as-tool evaluation."
     },
     {
       "title": "OctoPack: Instruction tuning code large language models",
       "authors": ["Niklas Muennighoff", "Qian Liu", "Armel Randy Zebaze"],
       "year": 2023,
       "relevance": "Instruction tuning approach for code models that created HumanEvalPack, a predecessor to BigCodeBench-Instruct's NL-oriented evaluation."
     }
   ]
 }