ai-research-survey

scan.json (18611B)

---

 {
   "paper": {
     "title": "DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models",
     "authors": ["Yiming Huang", "Jianwen Luo", "Yan Yu", "Yitong Zhang", "Fangyu Lei", "Yifan Wei", "Shizhu He", "Lifu Huang", "Xiao Liu", "Jun Zhao", "Kang Liu"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.07331"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release our benchmark at https://da-code-bench.github.io' in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark with 500 examples is released at the benchmark website. The data sources come from Kaggle, GitHub, and other web sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Docker-based environment with Python, SQL, Conda, and database engines but does not provide a requirements.txt, Dockerfile, or detailed dependency list with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The benchmark website is referenced but no README with commands to replicate experiments is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3 and 4 report only point estimates (e.g., '30.5% score') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares multiple models and frameworks (e.g., DA-Agent vs OpenDevin) and claims superiority without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw score differences are reported but no standardized effect sizes (Cohen's d, etc.) are provided. Percentage differences lack baseline context beyond the raw numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 500 examples and the subset DA-Code-100 has 100 randomly sampled tasks, but no justification or power analysis is provided for these sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run numbers with greedy sampling. No variance, standard deviation, or multiple-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 compares DA-Agent against X-Agent, AutoGen, and OpenDevin. Table 1 compares DA-Code against DS-1000, Arcade, MLAgentBench, and DA-Bench."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include OpenDevin (2024), AutoGen (2023), and X-Agent (2023), which are contemporary agent frameworks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 presents ablation studies on reference plan provision and max history length (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Score, Completion Rate (%), #Avg Steps, and Executable Code (%) in Table 3, plus per-category breakdowns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system outputs is performed. Evaluation is entirely automated via execution-based scoring."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The full 500-example benchmark serves as the test set, and DA-Code-100 is a randomly sampled subset. No tuning was done on these examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides breakdowns by DW, ML, EDA and by difficulty level (Easy, Medium, Hard). Figure 3 shows fine-grained per-category performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 discusses error analysis with four categories: hallucination issues, inability to follow instructions, persistent code errors, and misinterpretation of task context."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that even the best model achieves only 30.5%, and discusses where models fail (e.g., DW tasks, prolonged task sequences not improving outcomes)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 30.5% accuracy for the best LLM, which matches GPT-4's score in Table 3. The claim that DA-Agent outperforms other frameworks is supported by Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation on reference plans (Section 5.3) uses controlled single-variable manipulation, showing that adding reference plans improves score from 31.5 to 39.7. The history length ablation is similarly controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Agent Data Science Code Generation Benchmark for Large Language Models' broadly, but the benchmark covers only specific data science subcategories and only tests a limited set of models. No explicit scope boundaries are stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed performance differences. For instance, DA-Agent's advantage over other frameworks could be due to environment-specific tuning rather than general superiority."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4', 'GPT-4o', 'Claude-3-Opus' without specific version identifiers or snapshot dates (e.g., no 'gpt-4-0613')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The agent's system prompt and prompt templates are not provided in the paper or appendix. Only the action space format is described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states greedy sampling strategy, maximum step length of 20, max history length of 15 steps, and action execution time limit of 300 seconds."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the DA-Agent framework in detail: Docker environment, action space (Bash, Python, SQL, Terminate), response mechanism, and memory windows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes the full annotation pipeline: data source selection, task rewriting/creation, task implementation, evaluation setup, and cross-validation with red team testing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the conclusion, discussing unexplored fine-tuning and the need for deeper investigation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is generic — it mentions fine-tuning LLMs was not explored and the benchmark warrants more investigation, but does not discuss specific threats like annotator bias, benchmark coverage gaps, or evaluation robustness concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not bound its claims to specific model families, data types, or task domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is released at https://da-code-bench.github.io, which should include the 500 task examples and evaluation suites."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes the annotation pipeline: data sources from Kaggle/GitHub, task rewriting process, environment setup, and cross-validation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We recruit ten annotators who are highly proficient in data analysis, SQL, and Python' but does not describe how they were recruited or their specific backgrounds."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 and Section 3.5 document the full pipeline from data source selection through task definition, implementation, evaluation setup, and cross-validation with red team testing."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists National Key R&D Program of China (No. 2022ZD0160503), NSFC (No.62376270), and CCF-BaiChuan-Ebtech Foundation Model Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Chinese Academy of Sciences, UC Davis, Microsoft Research Asia, Shanghai AI Laboratory."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government research programs (NSFC, National Key R&D Program) which have no financial stake in the benchmark results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4, GPT-4o, Claude-3-Opus, and others on the benchmark but does not state any model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark tasks or their source data appeared in any model's training data, despite data being sourced from Kaggle and GitHub."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data comes from Kaggle and GitHub which are common training data sources. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Annotators created the benchmark but are not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time per task are reported despite the agent making multiple LLM calls per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "The best LLM (GPT-4) achieves only 30.5% score on DA-Code using DA-Agent.",
       "evidence": "Table 3 shows GPT-4 achieving 30.5 total score across all 500 tasks.",
       "supported": "strong"
     },
     {
       "claim": "DA-Agent outperforms existing agent frameworks (OpenDevin, AutoGen, X-Agent) on DA-Code.",
       "evidence": "Table 4 shows DA-Agent scoring 31.5 vs OpenDevin 26.2, AutoGen 18.6, X-Agent 6.7 on DA-Code-100.",
       "supported": "moderate"
     },
     {
       "claim": "Providing a reference plan improves agent performance from 31.5 to 39.7 score.",
       "evidence": "Table 4 shows DA-Code with reference plan achieving 39.7 vs 31.5 without on DA-Code-100.",
       "supported": "moderate"
     },
     {
       "claim": "Model performance decreases with increasing task difficulty, validating the difficulty grading.",
       "evidence": "Table 3 shows GPT-4 scoring 45.4 (Easy), 27.8 (Medium), 23.4 (Hard).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DA-Code is a 500-example benchmark for evaluating LLMs on agent-based data science tasks covering data wrangling, machine learning, and exploratory data analysis. The best-performing model (GPT-4) achieves only 30.5% score, indicating substantial room for improvement. The DA-Agent framework outperforms existing frameworks like OpenDevin and AutoGen. Analysis reveals that models struggle with data wrangling tasks and that providing reference plans significantly boosts performance.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run with greedy decoding. No error bars, confidence intervals, or multiple-run variance is reported for any experiment."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Benchmark data is sourced from Kaggle and GitHub, which are common in LLM training corpora. No discussion of whether models may have seen this data during training."
     },
     {
       "flag": "Framework comparison on small subset",
       "detail": "The comparison with competing frameworks (Table 4) uses only DA-Code-100 (100 randomly sampled tasks), not the full 500-task benchmark, without justification for this sample size."
     },
     {
       "flag": "No model version specificity",
       "detail": "Models are identified only by marketing names (GPT-4, GPT-4o, Claude-3-Opus) without API versions or snapshot dates, making results non-reproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational LLM code generation benchmark."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez"],
       "year": 2023,
       "relevance": "Major repository-level code generation benchmark for LLM agents."
     },
     {
       "title": "Benchmarking large language models as AI research agents",
       "authors": ["Qian Huang"],
       "year": 2023,
       "arxiv_id": "2310.03302",
       "relevance": "MLAgentBench defines auto ML tasks in interactive environments, directly compared in this paper."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation",
       "authors": ["Qingyun Wu"],
       "year": 2023,
       "relevance": "Multi-agent framework used as baseline comparison in DA-Code experiments."
     },
     {
       "title": "Executable code actions elicit better LLM agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "arxiv_id": "2402.01030",
       "relevance": "CodeAct framework that underpins OpenDevin, a baseline in this paper."
     },
     {
       "title": "SWE-agent: Agent computer interfaces enable software engineering language models",
       "authors": ["John Yang"],
       "year": 2024,
       "relevance": "Agent framework for software engineering tasks with specialized action design."
     },
     {
       "title": "InterCode: Standardizing and benchmarking interactive coding with execution feedback",
       "authors": ["John Yang"],
       "year": 2024,
       "relevance": "Inspired DA-Code's interactive sandbox environment design."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn"],
       "year": 2024,
       "relevance": "Foundational agent method for self-reflection and iterative improvement in code generation."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai"],
       "year": 2023,
       "relevance": "Prior data science code generation benchmark directly compared in Table 1."
     },
     {
       "title": "InfiAgent-DABench: Evaluating agents on data analysis tasks",
       "authors": ["Xueyu Hu"],
       "year": 2024,
       "arxiv_id": "2401.05507",
       "relevance": "Concurrent data analysis benchmark compared in Table 1."
     }
   ]
 }