ai-research-survey

scan.json (26573B)

---

 {
   "paper": {
     "title": "AutoKaggle: A Multi-Agent Framework for Autonomous Data Science Competitions",
     "authors": [
       "Ziming Li",
       "Qianbo Zang",
       "David Ma",
       "Jiawei Guo",
       "Tuney Zheng",
       "Minghao Liu",
       "Xinyao Niu",
       "Yue Wang",
       "Jian Yang",
       "Jiaheng Liu",
       "Wanjun Zhong",
       "Wangchunshu Zhou",
       "Wenhao Huang",
       "Ge Zhang"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.20424"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code is available at https://github.com/multimodal-art-projection/AutoKaggle' (footnote 1, page 1) and provides a project homepage URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses 8 publicly available Kaggle competitions (Titanic, Spaceship Titanic, House Prices, Monsters, Academic Success, Bank Churn, Obesity Risk, Plate Defect), all accessible via Kaggle. The datasets are public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency list is provided in the paper. The paper mentions using libraries like pandas, scikit-learn, matplotlib, and seaborn in code snippets but does not specify versions or a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While code is released on GitHub, the paper itself contains no 'Reproducing Results' section or specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment is repeated 5 trials, but results in Table 1 report only point estimates (e.g., '0.85', '0.82') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AutoKaggle 'surpassing the AIDE framework by 28%' (Section 4.2) and makes other comparative claims, but no statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context, e.g., 'average valid submission rate of 83% across all 8 Kaggle tasks, surpassing the AIDE framework by 28%' (Section 4.2), and comprehensive scores of 0.821 vs 0.641 for AIDE. Percentage improvements with baselines are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 8 Kaggle competitions with 5 trials each but provides no justification for why 8 competitions or 5 trials were chosen, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running 5 trials per task, no standard deviations, variance, or spread measures are reported. Table 1 reports single aggregate numbers per task (presumably averages or rates across 5 trials) with no spread information."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "AIDE (Schmidt et al., 2024) is used as a baseline, described as 'the best-performing framework in MLE-bench evaluation results' (Section 4.1). Additionally, model variants (GPT-4o vs o1-mini) serve as comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "AIDE (Schmidt et al., 2024) is a contemporary baseline from the same year, described as the best-performing framework on MLE-bench. This is a relevant and competitive comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents multiple ablation studies: machine learning tools (Table 2, progressive addition of DC, FE, MBVP tools), unit tests (Table 3, with/without), debugging times (Figure 5, 0/5/10/15 attempts), and competition date stratification."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: Made Submission, Valid Submission, Average Normalized Performance Score, and Comprehensive Score (Section 4.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AutoKaggle produces interpretable reports and is 'an educational tool' but provides no human evaluation of output quality, report usefulness, or code quality. All evaluation is automated via Kaggle submission scores."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated by submitting to the actual Kaggle platform, which uses a separate held-out test set. The paper notes they 'submitting them manually for evaluation' (Appendix B) rather than just using local train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-task breakdowns across all 8 competitions, and results are also stratified by Classic vs. Recent Kaggle competitions (Section 4.3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides a detailed error analysis with error type taxonomy (Table 4), counts of each error type (e.g., Value Error: 49, Key Error: 44), per-phase failure rates, and a worked debugging example."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that adding feature engineering tools actually decreased completion rate compared to DC tools only (Table 2: 0.88 with DC Tools vs 0.65 with DC & FE Tools), and that o1-mini performed worse than GPT-4o as Planner despite 'purported superior reasoning capabilities' (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'a validation submission rate of 0.85 and a comprehensive score of 0.82' — though Table 1 shows valid submission of 0.83 and comprehensive score of 0.821 for GPT-4o. The abstract states 0.85 which corresponds to the Made Submission rate rather than Valid Submission. This is slightly misleading but the numbers are from the paper's results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'the completion rate increases by 30% with the use of data cleaning phase tools'). The ablation design uses controlled single-variable manipulation: adding/removing tools, unit tests, or debugging attempts while holding other factors constant. This is adequate for causal inference about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Autonomous Data Science Competitions' but the paper only tests on 8 tabular data Kaggle competitions (classification and regression). The paper claims it provides 'end-to-end processing solutions for tabular data' and is a 'universal and comprehensive solution for a wide variety of data science tasks' (Section 1) without bounding these claims to the specific tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for the results. For example, the performance advantage over AIDE could be due to task selection (small tabular datasets where AIDE may not be optimized), the specific Kaggle competitions chosen, or the evaluation metrics used. None of these are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'GPT-4o', 'o1-mini', and 'GPT-4o-mini' but provides no snapshot dates or API versions (e.g., 'gpt-4o-2024-05-13'). Section 4.1 mentions 'GPT-4o, which is trained on data available until October 2023' but does not give the exact model version/endpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1.2 through C.1.6 provides the full prompt text for all five agents (Reader, Planner, Developer, Reviewer, Summarizer), including both role descriptions and task prompts with actual template text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API hyperparameters are reported for any of the LLM calls. Section 4.1 describes maximum iteration counts (3 iterations, 5 debug attempts) but not LLM sampling parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in extensive detail: five specialized agents (Section 3.1), phase-based workflow with six phases, iterative debugging and testing (Section 3.2, Algorithm 2), ML tools library (Section 3.3), tool retrieval via RAG (Appendix C.4), and agent interaction processes (Appendix C.1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes the dataset selection criteria: tabular datasets from Kaggle, classification/regression tasks, avoidance of large datasets, Classic (pre-Oct 2023, 500+ participants) vs Recent (2024+) categorization. Table 5 provides dataset statistics. The input preparation (overview.txt extraction) is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section in the paper. The conclusion (Section 5) makes no mention of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. While the error analysis (Section 4.4) discusses error types, it does not address threats to the validity of the evaluation itself."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show. The only partial acknowledgment is that large datasets were excluded due to resource limitations (Section 4.1 and Appendix B), but this is not framed as a scope limitation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the Kaggle datasets are publicly available, the raw experimental outputs (individual trial results, logs, submission files, agent traces) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix B describe data collection: 8 Kaggle competitions selected based on criteria (tabular data, classification/regression, dataset size constraints, competition date stratification). Table 5 provides detailed dataset characteristics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper evaluates a software system on public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: task selection criteria, input preparation (overview.txt + data files), 5 trials per task, manual Kaggle submission, metric computation (Equations 1-3). The full AutoKaggle pipeline is documented in Algorithm 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: M-A-P, ByteDance Inc., 2077AI, University of Melbourne, and Université du Luxembourg (page 1)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Authors are affiliated with ByteDance Inc., a company with commercial interests in AI, but there is no disclosure of whether ByteDance funded this work or has a stake in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'our analysis relies on GPT-4o, which is trained on data available until October 2023.' This is used to justify the Classic vs Recent Kaggle competition split."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 ('Study on Competition Date') explicitly addresses this: Tasks 1-4 are 'competitions potentially included in the training data of models such as GPT-4o and O1-mini' while Tasks 5-8 were 'derived from competitions launched in the current year' to assess out-of-distribution performance."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The Classic vs Recent split directly addresses contamination risk. The paper acknowledges Classic Kaggle competitions may be in training data and evaluates on Recent (2024+) competitions to test generalization. Results show 'only marginal performance degradation' (Section 4.3), which is transparently reported."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates a software system on public benchmarks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-example costs are reported despite the framework making many LLM calls across 5 agents over 6 phases per competition. The paper mentions resource limitations as a reason for excluding large datasets but does not quantify costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware specifications are reported. The paper mentions resource constraints prevented using MLE-bench's 24-hour setup but does not state what resources were actually used."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoKaggle achieves a valid submission rate of 0.85 and comprehensive score of 0.82 across 8 Kaggle competitions.",
       "evidence": "Table 1 shows AutoKaggle with GPT-4o achieves 0.83 valid submission rate and 0.821 comprehensive score (averages across 8 tasks). The abstract states 0.85 and 0.82 which appear to be rounded from Made Submission (0.85) and Comprehensive Score (0.821).",
       "supported": "moderate"
     },
     {
       "claim": "AutoKaggle with GPT-4o surpasses AIDE by 28% in valid submission rate.",
       "evidence": "Table 1 shows AutoKaggle GPT-4o valid submission rate of 0.83 vs AIDE GPT-4o at 0.58, a difference of 25 percentage points (Section 4.2 claims 28%, possibly calculated differently). Based on 5 trials per 8 tasks with no statistical tests.",
       "supported": "moderate"
     },
     {
       "claim": "The machine learning tools library increases completion rate by 27.5% compared to no tools.",
       "evidence": "Table 2 shows valid submission rate of 0.58 (No Tools) vs 0.85 (All Tools), a 27 percentage point increase. But this is based on only 4 of the 8 tasks (Tasks 1, 2, 3, 5), with 5 trials each.",
       "supported": "moderate"
     },
     {
       "claim": "Unit testing is essential — without it, completion rate drops from 0.85 to 0.10.",
       "evidence": "Table 3 shows completion rate of 0.10 without unit tests vs 0.85 with unit tests, across 4 tasks. This is a dramatic difference, though again limited to 4 tasks.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o outperforms o1-mini as the Planner model due to o1-mini's tendency toward excessive planning complexity.",
       "evidence": "Table 1 shows AutoKaggle GPT-4o outperforms AutoKaggle o1-mini on all three metrics (0.85 vs 0.73 MS, 0.83 vs 0.70 VS, 0.821 vs 0.759 CS). The hypothesis about excessive complexity is qualitative (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "AutoKaggle shows only marginal performance degradation on recent (post-2024) Kaggle competitions compared to classic ones.",
       "evidence": "Section 4.3 reports valid submission rate of 0.90 (classic) vs 0.75 (recent), and comprehensive score of 0.842 (classic) vs 0.800 (recent). Whether a 15-point VS drop and 4.2-point CS drop constitutes 'marginal' is debatable.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AutoKaggle, a multi-agent framework using GPT-4o, achieves 83% valid submission rate and 0.821 comprehensive score across 8 Kaggle tabular data competitions, outperforming AIDE. Ablation studies show that the machine learning tools library primarily improves completion rate rather than solution quality, while unit testing is critical (completion drops from 85% to 10% without it). Interestingly, o1-mini underperforms GPT-4o as the Planner agent, attributed to excessive planning complexity. The framework shows only modest degradation on post-2024 competitions not in the training data.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "Despite running 5 trials per task, no standard deviations, confidence intervals, or error bars are reported for any metric. Readers cannot assess result stability."
     },
     {
       "flag": "Very small benchmark size",
       "detail": "Only 8 Kaggle competitions are used, with ablation studies on only 4. This is a small sample for the broad claims made about framework effectiveness. No justification for this sample size."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims (e.g., 'surpassing AIDE by 28%') are made without any significance tests, despite the small number of tasks and trials."
     },
     {
       "flag": "Abstract misrepresents metric",
       "detail": "The abstract claims 'validation submission rate of 0.85' but Table 1 shows Valid Submission is 0.83 for GPT-4o. The 0.85 figure corresponds to Made Submission, a less stringent metric."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, despite significant scope restrictions (tabular data only, small datasets only, 8 competitions, single baseline)."
     },
     {
       "flag": "Overclaiming generality",
       "detail": "The paper calls AutoKaggle a 'universal and comprehensive solution for a wide variety of data science tasks' based on 8 tabular competitions. Non-tabular data, time series, NLP, vision, and large-scale datasets are untested."
     },
     {
       "flag": "Single baseline comparison",
       "detail": "Only AIDE is used as a baseline. Other relevant systems like Data Interpreter, DS-Agent, SELA, and MetaGPT are discussed in related work but not compared against experimentally."
     },
     {
       "flag": "No cost reporting",
       "detail": "The framework uses multiple LLM agents across many phases and iterations, but no API costs, token counts, or runtime are reported, making practical feasibility assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for evaluating ML agents on engineering tasks, directly relevant to assessing agentic AI capabilities."
     },
     {
       "title": "Data Interpreter: An LLM Agent for Data Science",
       "authors": ["Sirui Hong", "Yizhang Lin", "Bang Liu"],
       "year": 2024,
       "arxiv_id": "2402.18679",
       "relevance": "LLM-based data science agent providing evaluation metrics and comparison framework for AI-assisted data science."
     },
     {
       "title": "SELA: Tree-Search Enhanced LLM Agents for Automated Machine Learning",
       "authors": ["Yizhou Chi", "Yizhang Lin", "Sirui Hong"],
       "year": 2024,
       "arxiv_id": "2410.17238",
       "relevance": "Another approach to automated ML using LLM agents with tree search, relevant to agentic AI for data science."
     },
     {
       "title": "DS-Agent: Automated Data Science by Empowering Large Language Models with Case-Based Reasoning",
       "authors": ["Siyuan Guo", "Cheng Deng", "Ying Wen"],
       "year": 2024,
       "arxiv_id": "2402.17453",
       "relevance": "LLM-based automated data science agent using case-based reasoning, directly comparable to AutoKaggle."
     },
     {
       "title": "DSBench: How Far Are Data Science Agents to Becoming Data Science Experts?",
       "authors": ["Liqiang Jing", "Zhehui Huang", "Xiaoyang Wang"],
       "year": 2024,
       "arxiv_id": "2409.07703",
       "relevance": "Benchmark for evaluating data science agents on comprehensive tasks, relevant to the evaluation methodology space."
     },
     {
       "title": "ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code",
       "authors": ["Xiangru Tang", "Yuliang Liu", "Zefan Cai"],
       "year": 2024,
       "arxiv_id": "2311.09835",
       "relevance": "Benchmark for LLM agents on ML tasks at repository level, relevant to evaluating AI coding capabilities."
     },
     {
       "title": "Benchmarking Data Science Agents",
       "authors": ["Yuge Zhang", "Qiyang Jiang", "Xingyu Han"],
       "year": 2024,
       "arxiv_id": "2402.17168",
       "relevance": "Benchmark evaluation study for data science agents relevant to understanding the evaluation landscape."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Edward Berman"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Foundational work on self-reflective language agents, relevant to agentic AI methodology."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Seminal work on combining reasoning and action in LLM agents, foundational to agentic AI frameworks."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Self-refinement approach for LLMs relevant to iterative debugging methodology in agentic systems."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Language Model Society",
       "authors": ["Guohao Li", "Hasan Abed Al Kader Hammoud", "Hani Itani"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Multi-agent communication framework for LLMs, relevant to multi-agent collaboration research."
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Comprehensive survey of LLM-based agents relevant to understanding the broader landscape of agentic AI research."
     }
   ]
 }