ai-research-survey

scan-v5.json (21421B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DSCodeBench: A Realistic Benchmark for Data Science Code Generation",
     "authors": [
       "Shuyin Ouyang",
       "Dong Huang",
       "Jingwen Guo",
       "Zeyu Sun",
       "Qihao Zhu",
       "Jie M. Zhang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.15621",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All key abstract claims (1,000 problems, 10 libraries, GPT-4o pass@1 0.392, scaling behavior, comparison metrics vs DS-1000) are directly supported by Table 1, Table 2, and Figure 2.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims scaling behavior 'validates' the benchmark's discrimination ability, but this is a circular post-hoc argument; it does not rule out that task selection or perturbation methodology specifically favors larger models rather than reflecting inherent capability differences.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DSCodeBench is 'a rigorous and trustworthy foundation for advancing LLM-based data science programming' broadly, but only 10 models and 1,000 Python-only single-function tasks were tested; the framing does not adequately reflect this scope.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its main finding—that DSCodeBench shows scaling behavior while DS-1000 does not—such as whether perturbation design or task filtering biases the difficulty distribution toward features favored by larger models.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly acknowledges that pass@k measures functional correctness only and not efficiency, security, or readability, with the evaluation scope clearly bounded accordingly.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitation' section appears in the Appendix covering Python-only scope, single-function restriction, simplified error-handling, and functional-correctness-only evaluation.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: error-handling simplification (replacing raises with None/defaults), exclusion of multi-file and project-level tasks, SSIM-only visualization scoring, and restriction to Python.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what DSCodeBench does NOT cover: other languages (R), multi-file workflows, runtime efficiency, security, code style, and readability—both in limitations and future work sections.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgement section: ITEA Genius and GreenCode (InnovateUK), UKRI CDT (EP/S023356/1), and NSFC (62402482).",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header: King's College London, NUS, CAS, and Peking University—none are affiliated with the commercial LLM vendors evaluated.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (InnovateUK, UKRI, NSFC) are government/public research agencies unrelated to the commercial vendors (OpenAI, DeepSeek, Alibaba/Qwen) whose models are evaluated.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration appears; the paper contains only the funding acknowledgement and no 'no competing interests' statement.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "The key term 'realistic' is used as the central descriptor throughout the paper but is never formally defined; it is only operationalized implicitly via proxy metrics (code length, test count, GitHub source) without explicit definition.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly enumerated in bullet form: the DSCodeBench dataset, the construction pipeline, and the empirical evaluation of 10 state-of-the-art LLMs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides direct metric-by-metric comparison against 9 prior benchmarks, and the introduction explains specific limitations of each predecessor that motivate DSCodeBench's design choices.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper asserts GitHub-sourced code is more realistic than Stack Overflow snippets by definition, but provides no independent validation (no domain expert surveys, no analysis of what tasks data scientists actually perform) to confirm the construct maps to real-world data science work.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Per-library model performance variation is shown (Table 3, pass@1 range 0.029–0.591) but no explicit difficulty tiers are defined or characterized; difficulty is inferred post-hoc from model pass rates rather than measured a priori.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "The best model (GPT-4o) achieves 0.392 pass@1 and the weakest (DeepSeek-1.3B) achieves 0.076, confirming no ceiling or severe floor effects; the benchmark clearly discriminates across the evaluated model range.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is provided; the alignment analysis (97.4% agreement) tests whether a human can understand the problem description, not whether humans can solve the coding tasks.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Pass@k is adopted from Chen et al. 2021 without discussing alternatives; the SSIM > 0.5 threshold for visualization tasks is stated but not justified or ablated, and no edge-case scoring analysis is provided.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "The paper applies systematic code perturbations (signature changes, line insertions/removals, control flow restructuring) and validates decontamination via text similarity < 0.4 and AST similarity < 0.5 across all models and libraries (Figures 3–4).",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of how long the current 1,000 problems will remain undiscovered or ungamed; future work mentions expansions but does not address plans for benchmark updates or versioning as models improve.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Limitations discuss what the benchmark does not measure but do not analyze failure modes of the benchmark itself—e.g., whether high test-case coverage metrics can be gamed, or whether perturbation-based decontamination introduces systematic biases in task difficulty.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper states 'The benchmark, code, and experiment results are available at https://github.com/ShuyinOuyang/DSCodeBench', providing all components needed to reproduce reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "The paper provides detailed construction documentation covering all 5 pipeline stages, filtering criteria, per-library statistics (Table 1, Figure 2), alignment procedures, and test case generation methodology including prompts.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub URL is provided but no license is stated anywhere in the paper; it is unclear under what terms the benchmark can be used, modified, or redistributed by others.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The 'Broader Impact' section explicitly specifies intended uses (research, education, industry) and cautions that the benchmark should be used alongside diverse evaluation methods to avoid over-optimization.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DSCodeBench is more challenging than DS-1000, as GPT-4o achieves pass@1 of 0.392 on DSCodeBench vs 0.451 on DS-1000.",
       "evidence": "Table 2 shows direct head-to-head comparison on all 10 evaluated models; DSCodeBench consistently yields lower pass@1 and pass@3 than DS-1000 across all models.",
       "supported": "strong"
     },
     {
       "claim": "DSCodeBench exhibits robust scaling behavior where larger models systematically outperform smaller ones, while DS-1000 shows irregular scaling.",
       "evidence": "Table 2 shows consistent within-family ordering for DeepSeek (1.3B < 6.7B < 33B) and Qwen (7B < 14B < 32B) on DSCodeBench; DS-1000 shows reversals (e.g., DeepSeek-33B underperforms V2-Lite).",
       "supported": "strong"
     },
     {
       "claim": "Automatically generated test cases achieve 97.8% mean line coverage across all 1,000 benchmark problems.",
       "evidence": "Figure 5 reports per-library coverage from 95.5% (Pandas) to 99.7% (Matplotlib) with an overall mean of 97.8%.",
       "supported": "strong"
     },
     {
       "claim": "Contamination is effectively mitigated: LLM-generated code has text similarity < 0.4 and AST similarity < 0.5 to ground truth.",
       "evidence": "Figures 3 and 4 show similarity distributions across 10 models and 10 libraries all below stated thresholds, using text and AST similarity metrics.",
       "supported": "moderate"
     },
     {
       "claim": "DSCodeBench provides more reliable evaluation than DS-1000, evidenced by lower variance in model performance scores.",
       "evidence": "Table 2 standard deviations are reported (e.g., GPT-4o correct: 391.7±4.6 on DSCodeBench vs 450.7±2.9 on DS-1000); the comparison is inconsistent and not uniformly lower on DSCodeBench.",
       "supported": "weak"
     },
     {
       "claim": "Problem descriptions achieve 97.4% alignment between the description and ground truth code as judged by human experts and LLM judges.",
       "evidence": "Alignment section describes dual-stage validation using two author-experts and GPT-4o-mini/GPT-4o as judges; however, the LLM judges are among the models being benchmarked, introducing potential bias.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DSCodeBench introduces 1,000 realistic data science coding problems sourced from GitHub across 10 Python libraries, with average solutions of 22.5 lines and 200 test cases per problem—far exceeding prior benchmarks like DS-1000 (3.6 lines, 2.1 tests). The best evaluated model (GPT-4o) achieves only 0.392 pass@1, confirming the benchmark's difficulty, and unlike DS-1000, all model families exhibit consistent scaling behavior where larger models outperform smaller ones. Visualization libraries (Matplotlib, Seaborn) remain the hardest across all models with pass@1 as low as 0.010–0.141 for GPT-4o, revealing a persistent capability gap. Automatically generated test case scripts achieve 97.8% mean code coverage, supporting the benchmark's evaluation reliability.",
   "red_flags": [
     {
       "flag": "No human performance baseline",
       "detail": "The benchmark claims to measure realistic data science coding ability and to be calibrated to real-world difficulty, but provides no human performance baseline—making it impossible to contextualize LLM scores or validate difficulty calibration against human capability."
     },
     {
       "flag": "Circular construct validity argument",
       "detail": "'GitHub code is more realistic than Stack Overflow' is an assumption used as its own justification for construct validity; no domain expert surveys, observational studies of real data scientist workflows, or external validation criteria are used."
     },
     {
       "flag": "LLM-as-judge using benchmarked models",
       "detail": "Alignment validation used GPT-4o-mini and GPT-4o as LLM judges to assess whether problems are solvable, yet these are among the primary models being evaluated on the benchmark, creating a potential circularity in quality assessment."
     },
     {
       "flag": "Visualization scoring threshold unjustified",
       "detail": "The SSIM > 0.5 threshold for plot-drawing task evaluation is stated without justification, ablation, or discussion of sensitivity—an arbitrary threshold that may arbitrarily penalize visually acceptable solutions or accept poor-quality plots."
     },
     {
       "flag": "Scaling behavior used as post-hoc validity proxy",
       "detail": "Adherence to scaling laws is treated as evidence of benchmark quality ('validating its ability to distinguish model capabilities'), but scaling behavior could equally reflect task selection or perturbation methodology biased toward larger model strengths rather than being an independent quality signal."
     }
   ],
   "cited_papers": [
     {
       "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",
       "relevance": "Primary benchmark being compared against and improved upon; DSCodeBench is explicitly positioned as addressing DS-1000's three key limitations in code length, test coverage, and problem structure."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Introduces pass@k metric adopted throughout this paper and establishes the baseline paradigm for functional-correctness-based code generation benchmarking."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "relevance": "Compared directly in Table 1 as the leading general code generation benchmark; provides the competitive upper bound for complex benchmark design."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Compared in Table 1; represents an approach to contamination-resistant benchmarking that informs DSCodeBench's decontamination design."
     },
     {
       "title": "DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models",
       "relevance": "Directly compared as a data science–specific benchmark focusing on task diversity; noted for insufficient test coverage that DSCodeBench addresses."
     },
     {
       "title": "DataSciBench: An LLM Agent Benchmark for Data Science",
       "relevance": "Contemporary data science benchmark compared in Table 1; contrasted for smaller scale (222 problems, 2.3 avg test cases) and limited library coverage."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "relevance": "One of the primary open-source model families evaluated; key for assessing whether open-source coding models can compete with closed-source on realistic tasks."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Cited as an example of realistic software engineering benchmarks sourced from GitHub; contextualizes DSCodeBench's GitHub-based construction philosophy."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Practitioners evaluating LLMs for data science can directly use the publicly released benchmark to compare models across 10 specific Python libraries with a rigorous 200-test evaluation framework."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that GPT-4o achieves only 39% on realistic data science tasks is modestly surprising given the hype around LLM coding capability, but the direction is consistent with prior findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper focuses entirely on capability evaluation for data science productivity use cases."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper directly critiques DS-1000 as unreliable and misleading (irregular scaling, format ambiguity, insufficient tests), which is a mild controversy in the benchmarking community."
     },
     "demo_ability": {
       "score": 3,
       "justification": "The benchmark is publicly released on GitHub with code, evaluation framework, per-library problems, and full experiment results enabling immediate use and replication."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from King's College London, NUS, CAS, and Peking University—reputable institutions but not the top-tier AI labs that drive outsized HN and community attention."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "36184838",
         "title": "Reverse Engineering Self-Supervised Learning",
         "points": 86,
         "comments": 16,
         "url": "https://news.ycombinator.com/item?id=36184838",
         "created_at": "2023-06-04T11:43:46Z"
       },
       {
         "hn_id": "43870679",
         "title": "Show HN: I built an AI tool to practice technical interviews with",
         "points": 12,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43870679",
         "created_at": "2025-05-02T14:57:13Z"
       },
       {
         "hn_id": "45300655",
         "title": "Generalizable Geometric Image Caption Synthesis",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45300655",
         "created_at": "2025-09-19T12:05:01Z"
       },
       {
         "hn_id": "43405094",
         "title": "Politicians' misinformation behavior and public engagement, in 4 countries",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43405094",
         "created_at": "2025-03-18T21:03:45Z"
       },
       {
         "hn_id": "44324675",
         "title": "ProtoReasoning: Prototypes as the Foundation for Generalizable Reasoning in LLMs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44324675",
         "created_at": "2025-06-20T04:10:28Z"
       },
       {
         "hn_id": "43776339",
         "title": "The Bitter Lesson Learned from 2k Multilingual Benchmarks",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43776339",
         "created_at": "2025-04-23T20:31:54Z"
       },
       {
         "hn_id": "40488690",
         "title": "Neuromorphic dreaming: A pathway to efficient learning in artificial agents",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40488690",
         "created_at": "2024-05-27T08:03:31Z"
       }
     ],
     "top_points": 86,
     "total_points": 110,
     "total_comments": 17
   }
 }