ai-research-survey

scan.json (23515B)

---

 {
   "paper": {
     "title": "Large Language Models for Constructing and Optimizing Machine Learning Workflows: A Survey",
     "authors": [
       "Yang Gu",
       "Hengyu You",
       "Jian Cao",
       "Muran Yu",
       "Haoran Fan",
       "Shiyou Qian"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2411.10478",
     "doi": "10.1145/3773084"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey provides a taxonomy of LLM applications across five ML workflow stages: data preprocessing, feature engineering, model selection, hyperparameter optimization, and workflow evaluation. It categorizes LLM-assisted model selection into retrieval-based and generation-based approaches, and HPO into execution-based and prediction-based methods. The paper identifies six open challenges (data leakage, prompt engineering complexity, hallucination, interpretability, resource consumption, social impact) and proposes end-to-end workflow construction and hybrid LLM-specialized model integration as future directions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository: 'visit our repository: https://github.com/t-harden/LLM4AutoML' in the footnote on page 1. This is a curated list of papers, not analysis code, but it is a released artifact."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset or search corpus is released. The GitHub repo is a curated reading list, not the underlying survey data (search queries, inclusion/exclusion decisions, extraction sheets)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The survey has no computational pipeline to reproduce."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the survey methodology (search strategy, screening process, data extraction) are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments or quantitative analysis of its own."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments or statistical comparisons."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper positions itself against Tornede et al. [2023a]: 'it is the first survey to systematically address every stage of the ML workflow (Fig. 1), distinguishing it from previous reviews, such as Tornede et al. [2023a], which primarily explore the broader opportunities of an integration of LLMs and AutoML' (Section 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison baseline (Tornede et al. 2023) is contemporary and relevant."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no experiments with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no experiments."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes reviewed work by workflow stage (Sections 3-5) and provides Table 1 with a per-method breakdown across five workflow components (data preprocessing, feature engineering, model selection, HPO, evaluation)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Each subsection discusses limitations and failure modes of the reviewed approaches. Section 6 dedicates substantial discussion to open challenges including hallucination, data leakage, prompt engineering difficulties, and resource consumption."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently discusses what doesn't work: LLM misinterpretation of data structures (Section 3.1.2), biased feature selection (Section 3.2.1), hallucination in feature synthesis (Section 3.2.3), stochastic model selection (Section 4.1.2), and inaccurate performance predictions (Section 4.2.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'comprehensive and up-to-date review' covering 'data and feature engineering, model selection and hyperparameter optimization, and workflow evaluation.' The paper delivers on this structure across Sections 3-5 with Table 1 summarizing 22 methods."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes claims like LLMs 'streamline and enhance ML workflow modeling process' (abstract) and LLMs 'have shown great potential for automating and enhancing various stages' without assessing the quality of evidence from the papers making these claims. The survey does not evaluate whether the improvements reported in cited papers are methodologically sound."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'our focus is on capturing the breadth and possibility of research within the specific context of constructing and optimizing ML workflows, rather than providing an exhaustive examination of all LLM-related methods at each individual stage' (Section 1). The scope is clearly bounded to ML workflow construction."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not consider alternative explanations for the positive results reported in the literature, such as publication bias toward positive results, overfitting to benchmarks, or whether improvements are due to factors other than LLMs. Section 6 discusses challenges of LLMs but not whether the claimed benefits are real."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no models used by the authors."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no prompting used by the authors."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no experiments conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No paper selection pipeline is described. There is no mention of which databases were searched, what search queries were used, what inclusion/exclusion criteria were applied, or how many papers were screened at each stage. The survey does not explain how the 22 methods in Table 1 were identified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Open Challenges and Future Directions' contains substantial discussion across six subsections (data leakage, prompt engineering, hallucination, interpretability, resource consumption, social impact)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses challenges and limitations of LLM-driven ML workflows in general, but does not discuss threats to the validity of the survey itself — e.g., selection bias in paper collection, coverage gaps, or limitations of the non-systematic review methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our focus is on capturing the breadth and possibility of research within the specific context of constructing and optimizing ML workflows, rather than providing an exhaustive examination of all LLM-related methods at each individual stage' (Section 1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw survey data (search results, screening decisions, extraction sheets) is available. The GitHub repo is a curated reading list, not the survey methodology artifacts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the surveyed papers were identified. No databases, search queries, time periods, or inclusion criteria are specified. The methodology for collecting the 22 methods in Table 1 is entirely opaque."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper selection process (analogous to 'recruitment' for a survey) is not described, but this question is about human participant recruitment specifically."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline from paper collection to final survey is documented. There is no PRISMA-like flow diagram or description of screening stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Shanghai Jiao Tong University (Gu, You, Cao, Fan, Qian) and Stanford University (Yu)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no method of its own with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no computational work performed."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no registered protocol, no systematic search strategy. The paper does not describe which databases were searched, what queries were used, or how papers were screened. The selection methodology for Table 1's 22 methods is entirely undocumented."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of the papers it reviews. All 22 methods in Table 1 are treated equivalently regardless of their experimental rigor, sample sizes, or validation quality. Claims from reviewed papers are reported at face value."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether the literature on LLMs for ML workflows skews toward positive results, or whether negative results (LLMs failing at workflow tasks) are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "This is the first survey to systematically address every stage of the ML workflow for LLM integration, distinguishing it from prior reviews like Tornede et al. (2023).",
       "evidence": "Section 1 states: 'To the best of our knowledge, it is the first survey to systematically address every stage of the ML workflow (Fig. 1), distinguishing it from previous reviews, such as Tornede et al. [2023a], which primarily explore the broader opportunities of an integration of LLMs and AutoML.'",
       "supported": "moderate"
     },
     {
       "claim": "LLMs have shown great potential for automating and enhancing various stages of the ML pipeline through language understanding, reasoning, interaction, and generation.",
       "evidence": "The paper reviews 22 methods across five workflow stages (Table 1), citing specific systems like CAAFE for feature engineering, HuggingGPT for model selection, LLAMBO for Bayesian optimization, and VML for workflow evaluation. However, results from these systems are reported as described by their authors without independent validation.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-driven ML workflows face significant challenges including hallucination, data leakage, complicated prompt engineering, interpretability issues, high resource consumption, and social impact concerns.",
       "evidence": "Section 6 provides detailed discussion of each challenge with citations to specific systems exhibiting these issues (e.g., MLCopilot for hallucination, LLAMBO for resource consumption, CAAFE for hallucinated features).",
       "supported": "moderate"
     },
     {
       "claim": "End-to-end ML workflow construction using LLMs is a promising future direction where LLMs autonomously handle every pipeline stage.",
       "evidence": "Section 6.2 discusses this direction citing AIDE, ResearchAgent, CodeActAgent, and DS-Agent as existing implicit stage-wise solutions, but acknowledges these 'often lack transparency' and notes explicit end-to-end generation is not yet achieved.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The survey does not describe how papers were identified, which databases were searched, what search queries were used, or what inclusion/exclusion criteria were applied. The 22 methods in Table 1 appear to be an ad-hoc collection without a reproducible selection process."
     },
     {
       "flag": "No quality assessment of reviewed papers",
       "detail": "All reviewed papers are treated equally regardless of their methodological rigor. Claims from reviewed papers (e.g., that LLMs improve workflow efficiency) are reported at face value without assessing whether the underlying experiments are well-designed, properly controlled, or reproducible. This effectively launders the signal-to-noise ratio of the sources."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The survey does not consider whether the literature on LLMs for ML workflows is biased toward positive results. Papers showing LLMs failing at workflow tasks are likely underrepresented, which would systematically overstate the potential of LLM-driven workflows."
     },
     {
       "flag": "Overclaiming from limited evidence base",
       "detail": "The paper claims to provide a 'comprehensive' review but covers only 22 methods. Several workflow stages have very few entries (e.g., workflow evaluation has only 4 systems discussed). The comprehensiveness claim is not supported by evidence of systematic coverage."
     }
   ],
   "cited_papers": [
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLM agents on ML experimentation tasks, directly relevant to agentic AI evaluation methodology."
     },
     {
       "title": "Mle-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for evaluating ML engineering capabilities of LLM agents."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundation model used in many of the surveyed ML workflow systems."
     },
     {
       "title": "HuggingGPT: Solving AI tasks with ChatGPT and its friends in Hugging Face",
       "authors": ["Yongliang Shen", "Kaitao Song", "Xu Tan"],
       "year": 2024,
       "relevance": "LLM-based agent for AI task solving through model selection and orchestration, key example of agentic AI workflow."
     },
     {
       "title": "Large language models for automated data science: Introducing CAAFE for context-aware automated feature engineering",
       "authors": ["Noah Hollmann", "Samuel Müller", "Frank Hutter"],
       "year": 2024,
       "relevance": "LLM-driven automated feature engineering system demonstrating LLM capabilities in ML workflow automation."
     },
     {
       "title": "DS-Agent: Automated Data Science by Empowering Large Language Models with Case-Based Reasoning",
       "authors": ["Siyuan Guo", "Cheng Deng", "Ying Wen"],
       "year": 2024,
       "arxiv_id": "2402.17453",
       "relevance": "Agentic LLM system for automated data science with case-based reasoning."
     },
     {
       "title": "AutoML-Agent: A Multi-Agent LLM Framework for Full-Pipeline AutoML",
       "authors": ["Patara Trirat", "Wonyong Jeong", "Sung Ju Hwang"],
       "year": 2024,
       "arxiv_id": "2410.02958",
       "relevance": "Multi-agent LLM framework covering full AutoML pipeline, directly relevant to agentic AI workflow construction."
     },
     {
       "title": "Verbalized Machine Learning: Revisiting Machine Learning with Language Models",
       "authors": ["Tim Z Xiao", "Robert Bamler", "Bernhard Schölkopf", "Weiyang Liu"],
       "year": 2024,
       "arxiv_id": "2406.04344",
       "relevance": "Novel approach using LLM prompts as ML model parameters, relevant to LLM capability evaluation."
     },
     {
       "title": "Large Language Models to Enhance Bayesian Optimization",
       "authors": ["Tennison Liu", "Nicolás Astorga", "Nabeel Seedat", "Mihaela van der Schaar"],
       "year": 2024,
       "arxiv_id": "2402.03921",
       "relevance": "LLM-enhanced Bayesian optimization (LLAMBO) for hyperparameter tuning, demonstrating LLM integration into optimization."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique used in many LLM-based ML workflow systems."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source platform for AI coding agents, relevant to agentic software development workflows."
     },
     {
       "title": "AutoML in the age of large language models: Current challenges, future opportunities and risks",
       "authors": ["Alexander Tornede", "Difan Deng", "Theresa Eimer"],
       "year": 2023,
       "arxiv_id": "2306.08107",
       "relevance": "Prior survey on LLM-AutoML integration that this paper positions against, directly relevant to survey methodology comparison."
     },
     {
       "title": "Github copilot AI pair programmer: Asset or liability?",
       "authors": ["Arghavan Moradi Dakhel", "Vahid Majdinasab", "Amin Nikanjam"],
       "year": 2023,
       "doi": "10.1016/j.jss.2023.111734",
       "relevance": "Evaluation of GitHub Copilot for programming, relevant to LLM code generation productivity assessment."
     }
   ]
 }