ai-research-survey

scan.json (27834B)

---

 {
   "paper": {
     "title": "DSCodeBench: A Realistic Benchmark for Data Science Code Generation",
     "authors": [
       "Shuyin Ouyang",
       "Dong Huang",
       "Jingwen Guo",
       "Zeyu Sun",
       "Qihao Zhu",
       "Jie M. Zhang"
     ],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2505.15621"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DSCodeBench is a 1,000-problem benchmark for data science code generation across 10 Python libraries, sourced from GitHub with extensive test suites (200 tests per problem by default). Evaluation of 10 LLMs shows GPT-4o achieves best pass@1 of 0.392, with clear scaling behavior within model families. DSCodeBench is substantially harder than DS-1000 and exhibits lower evaluation variance. Visualization libraries (Matplotlib, Seaborn) remain particularly challenging for all models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: 'The benchmark, code, and experiment results are available at https://github.com/ShuyinOuyang/DSCodeBench.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (1,000 problems with ground truth code, test case scripts, and problem descriptions) is released at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Intel Xeon Platinum 8336C, 8×A100, 2.0TiB memory) but no software environment specification is provided — no requirements.txt, Python version, or library versions listed in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but no reproduction guide or README is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across 3 runs in Tables 2 and 4 (e.g., '391.7±4.6' for GPT-4o correct outputs, '105.0±4.3' for partially correct)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'GPT-4o achieves the highest pass@1' and scaling behavior are based on raw number comparisons without p-values, t-tests, or any formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: 'GPT-4o obtains pass@1 and pass@3 of 0.451 and 0.545 on DS-1000 but only 0.392 and 0.438 on DSCodeBench.' Absolute performance values and cross-benchmark differences are provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 1,000 problems, 10 models, or 3 experimental runs. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across 3 identical runs are reported in Tables 2 and 4 (e.g., '391.7±4.6', '307.3±5.6')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "DSCodeBench is compared against 9 other benchmarks in Table 1 (HumanEval, MBPP, APPS, BigCodeBench, LiveCodeBench, DSP, DA-Code, DataSciBench, DS-1000). Model results are compared across both DSCodeBench and DS-1000 in Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include contemporary options: GPT-4o (2024), Qwen2.5-Coder (2024), DeepSeek-Coder-V2 (2024). Benchmark comparisons include recent work like BigCodeBench (2024) and LiveCodeBench (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction pipeline has multiple components (seed code collection, context reconstruction, filtering, test case generation, manual editing) but no ablation study examines which components contribute most to benchmark quality or difficulty."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics reported: pass@1, pass@3, average correct/partially correct/wrong counts, per-library breakdowns, test case coverage (97.8%), and text/AST similarity for leakage analysis."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of LLM-generated code is entirely automated via test suites. Human evaluation is used only for benchmark construction alignment (97.4% agreement among authors + LLM judges), not for evaluating model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are evaluated on the full 1,000-problem benchmark without any fine-tuning or tuning on DSCodeBench data. The entire benchmark functions as a held-out test set for pre-trained models."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-library pass@1 breakdowns across all 10 libraries for each model. Tables 5 and 6 provide similar breakdowns at temperature=0.6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Common failure modes discussed: 'data structure mismatches, such as incompatible shapes or types during transformation steps, particularly in NumPy, TensorFlow, and PyTorch tasks. Visualization libraries also exhibit frequent issues, with models often mis-specifying figures or plotting arguments.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: Matplotlib and Seaborn scores remain very low across all models, increasing temperature to 0.6 significantly degrades Qwen2.5-Coder-7B and 14B performance, and DeepSeek-Coder-33B performs worse than 6.7B on DS-1000."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 1,000 problems across 10 libraries (confirmed in statistics section), robust scaling behavior (Table 2 shows consistent within-family scaling), GPT-4o pass@1 of 0.392 (Table 2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'architectural and training optimizations play a critical role beyond model scaling alone' (comparing GPT-4o-mini vs GPT-3.5-turbo) but cannot isolate these factors. Scaling claims are based on observational comparisons within model families where training data, architecture, and scale all change simultaneously."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'A Realistic Benchmark for Data Science Code Generation' and abstract claim 'realistic data science code generation tasks' are broader than the tested scope of Python-only with 10 specific libraries. The Limitations section acknowledges Python-only scope, but the title and abstract do not bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for observed results. For example, why Matplotlib/Seaborn scores are low (training data distribution? API complexity?), or why scaling behavior is clear on DSCodeBench but not DS-1000, is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures functional correctness via test suites and explicitly acknowledges the proxy gap: 'DSCodeBench evaluates functional correctness based on unit tests, without explicitly assessing other important dimensions of code quality, such as computational efficiency, coding style, readability, or security.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models identified by marketing names only: 'GPT-3.5-turbo', 'GPT-4o-mini', 'GPT-4o' without API versions or snapshot dates. Open-source models list parameter sizes but no checkpoint version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided in the Appendix for all prompt types: Code Generation Prompt, Test Case Script Generation/Repair Prompts, Code Problem Description Generation Prompt, and LLM Judge Prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature set to 0.2 (and 0.6 for sensitivity): 'We set the temperature of all models to 0.2, while keeping all other parameters at their default values.' Random seed set to 42 for test case generation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly with code problem descriptions and generate solutions in a single pass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented in detail: seed code collection from DS-1000 and Stack Overflow, GitHub REST API search yielding 807,198 candidates, AST-based context reconstruction, two-stage filtering to 7,623, test case generation to 2,407, manual editing to final 1,000."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitation' section is present in the Appendix with substantive discussion of three specific limitations (Python-only, simplified error handling, functional correctness only)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: Python-only coverage, error-handling simplification ('error-raising code segments are either removed or replaced with default behaviors'), exclusion of multi-file codebases, and no efficiency/security/readability evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries stated: 'DSCodeBench focuses exclusively on Python and ten popular data science libraries,' does not assess 'runtime performance, security, and adherence to coding best practices,' and 'primarily targets single-function or single-file tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset including code problems, ground truth code, test case scripts, and problem descriptions is available at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: seed code from DS-1000 reference code and Stack Overflow top-voted answers (up to 500 per library), GitHub REST API search using line-level queries, deduplication, and metadata preservation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Code sample selection described: seed code from DS-1000 and Stack Overflow, GitHub search via REST API, filtering by star count (≥10), minimum API calls (≥3), and compilation success. However, potential selection bias toward popular repositories is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline with counts at each stage: 807,198 initial candidates → property filtering → functional filtering → 7,623 validated → test case generation → 2,407 retained → manual editing → 1,000 final problems."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed: ITEA Genius and ITEA GreenCode projects (InnovateUK), UKRI CDT in Safe and Trusted AI (EP/S023356/1), and NSFC (62402482)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: King's College London, NUS, Chinese Academy of Sciences, Peking University. None affiliated with companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government and academic bodies (InnovateUK, UKRI, NSFC) with no financial interest in the benchmark results or evaluated models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 10 evaluated models (GPT-3.5-turbo, GPT-4o-mini, GPT-4o, DeepSeek variants, Qwen variants)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Discussed extensively: 'using raw GitHub code directly poses a risk of data leakage, as large language models may have been pre-trained on the same public code.' Mitigation via code perturbation, context reconstruction, and similarity analysis (text <0.4, AST <0.5)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination addressed through systematic code perturbation ('modifying function signatures, adding or removing lines of code, restructuring control flows'), context reconstruction, and quantitative similarity analysis confirming low overlap."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The paper evaluates LLMs on a code generation benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-problem cost is reported. API costs for closed-source models and inference time for open-source models are not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Intel Xeon Platinum 8336C, 8×A100, 2.0TiB) but total GPU hours, wall-clock time, or API spend are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results reported across 3 runs with standard deviations. Random seed set to 42. The paper also verifies that different seeds in test case generation produce consistent results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'We also run all the models 3 times identically to mitigate the randomness that affects our experiment results.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature set to 0.2 without justification for this specific value. No search budget or method for selecting this hyperparameter is described. Temperature=0.6 is tested as sensitivity analysis but not as part of a systematic search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "A uniform configuration (temperature=0.2, default parameters) is applied across all models, avoiding cherry-picking. Additional temperature=0.6 results are provided for robustness in Table 4."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create the benchmark and evaluate models on it without acknowledging potential biases of self-evaluation, such as benchmark design choices that might inflate difficulty claims or favor certain model families."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is compared across models of vastly different sizes (1.3B to GPT-4o) without discussing or normalizing for compute budget. Scaling claims do not account for compute differences between models."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Construct validity is discussed through comparison with 9 other benchmarks (Table 1), test case coverage analysis (97.8%), alignment validation (97.4% agreement), and explicit acknowledgment that functional correctness doesn't capture efficiency, style, or security."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Models are prompted directly without any agentic framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses general data leakage risk from GitHub code but does not specifically address temporal leakage: no model training cutoff dates are stated, and the timing of GitHub code availability relative to model training is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the detailed problem descriptions (including step-by-step logic, function signatures, and examples) provide information beyond what would be available in realistic coding scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data share structural similarities. Problems are constructed from GitHub repositories that models may have trained on; while code is perturbed, structural patterns may overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Concrete detection methods applied: text similarity analysis (scores <0.4 across libraries), AST similarity analysis (scores <0.5), and semantic comparison via pass@k performance gap between model outputs and ground truth."
       }
     }
   },
   "claims": [
     {
       "claim": "DSCodeBench is substantially more challenging than DS-1000, with all models achieving lower scores.",
       "evidence": "Table 2: GPT-4o drops from 0.451 to 0.392 pass@1; all 10 models show lower scores on DSCodeBench vs DS-1000.",
       "supported": "strong"
     },
     {
       "claim": "DSCodeBench exhibits robust scaling behavior where larger models systematically outperform smaller ones.",
       "evidence": "Table 2: Within DeepSeek family, pass@1 increases 0.076→0.163→0.205→0.222 with model size. Within Qwen family, 0.116→0.213→0.229.",
       "supported": "strong"
     },
     {
       "claim": "The best LLM (GPT-4o) achieves pass@1 of 0.392, indicating significant room for improvement.",
       "evidence": "Table 2: GPT-4o pass@1 = 0.392 averaged across 3 runs (391.7±4.6 correct out of 1,000).",
       "supported": "strong"
     },
     {
       "claim": "DSCodeBench provides more reliable evaluation than DS-1000 with lower variance in results.",
       "evidence": "Observation in results that variance is smaller in DSCodeBench than DS-1000, supported by comparing ± values in Table 2. No formal statistical test of this claim.",
       "supported": "moderate"
     },
     {
       "claim": "Data leakage is effectively mitigated through code perturbation and context reconstruction.",
       "evidence": "Text similarity <0.4, AST similarity <0.5 between LLM-generated code and ground truth (Figures 3-4). Code perturbation and context reconstruction described in Manual Editing section.",
       "supported": "moderate"
     },
     {
       "claim": "Visualization libraries (Matplotlib, Seaborn) remain particularly challenging for all current LLMs.",
       "evidence": "Table 3: Matplotlib pass@1 ranges 0.010–0.311, Seaborn 0.052–0.177 across all models, consistently the lowest-scoring libraries.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All claims of model superiority and scaling behavior are based on raw number comparisons across 3 runs with no formal statistical testing. With only 3 runs and sometimes overlapping standard deviations, claims of 'systematic' differences are not statistically validated."
     },
     {
       "flag": "No model version specification",
       "detail": "Closed-source models are identified by marketing names only (GPT-4o, GPT-4o-mini, GPT-3.5-turbo) without snapshot dates or API versions. Results may not be reproducible as model behavior changes over time."
     },
     {
       "flag": "Overclaiming in title and abstract",
       "detail": "The title claims 'Data Science Code Generation' broadly but the benchmark covers only Python with 10 libraries. Error handling is simplified (removed or replaced with defaults), and only single-file tasks are included, limiting the 'realistic' claim."
     },
     {
       "flag": "No inference cost reporting",
       "detail": "Evaluating 10 models on 1,000 problems × 3 runs (plus temperature=0.6 experiments) involves substantial compute, especially for closed-source APIs. No cost information makes practical reproducibility unclear."
     }
   ],
   "cited_papers": [
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Y. Lai", "C. Li", "Y. Wang", "T. Zhang", "R. Zhong", "L. Zettlemoyer", "W. Yih", "D. Fried", "S. Wang", "T. Yu"],
       "year": 2023,
       "relevance": "Primary baseline benchmark for data science code generation being compared against and improved upon."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and pass@k metric, foundational for code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for Python code generation evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo", "M. C. Vu", "J. Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code generation benchmark with diverse function calls, compared in Table 1."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code generation benchmark addressing temporal leakage."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Landmark benchmark for realistic software engineering tasks from GitHub issues."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming—The Rise of Code Intelligence",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Open-source code generation model family evaluated in this study."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["B. Hui", "J. Yang", "Z. Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Open-source code generation model family evaluated in this study."
     },
     {
       "title": "DataSciBench: An LLM Agent Benchmark for Data Science",
       "authors": ["D. Zhang", "S. Zhoubian", "M. Cai"],
       "year": 2025,
       "arxiv_id": "2502.13897",
       "relevance": "Contemporary data science benchmark compared against DSCodeBench."
     },
     {
       "title": "DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models",
       "authors": ["Y. Huang", "J. Luo", "Y. Yu"],
       "year": 2024,
       "arxiv_id": "2410.07331",
       "relevance": "Data science code generation benchmark focusing on task diversity."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["D. Huang", "J. M. Zhang", "M. Luck"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent approach to code generation with iterative testing and refinement."
     },
     {
       "title": "An empirical study of the non-determinism of ChatGPT in code generation",
       "authors": ["S. Ouyang", "J. M. Zhang", "M. Harman", "M. Wang"],
       "year": 2025,
       "relevance": "Studies LLM non-determinism in code generation, motivating randomness control design in DSCodeBench."
     }
   ]
 }