ai-research-survey

scan.json (19382B)

---

 {
   "paper": {
     "title": "CatDB: Data-catalog-guided, LLM-based Generation of Data-centric ML Pipelines",
     "authors": ["Saeed Fathollahzadeh", "Essam Mansour", "Matthias Boehm"],
     "year": 2025,
     "venue": "PVLDB",
     "doi": "10.14778/3742728.3742754"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository explicitly provided: https://github.com/CoDS-GCS/CatDB.git, listed under 'PVLDB Artifact Availability'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The artifact availability statement says 'source code, data, and/or other artifacts have been made available' at the GitHub link. The 20 datasets used are described in Table 3 and appear to be real-world datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions Ubuntu 22.04, OpenJDK 11, Python 3.10, and hardware specs, but no requirements.txt, Dockerfile, or detailed dependency listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-level guidance for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 5 and 7 are point estimates. Figure 11 shows box plots for 10 iterations but no confidence intervals or error bars with explicit values are reported in tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CatDB achieves 'comparable or better' accuracy than baselines but no statistical significance tests are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., 'up to 52% gains' (Section 5.3), '8x to 14x speedup' (Section 5.4), and absolute accuracy values for all systems enabling comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 20 datasets were chosen, why 10 iterations were used for the repeated experiments, or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 11 shows box plots across 10 iterations for three datasets, depicting variance across runs. The success ratios are also annotated on the plots."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines: LLM-based (CAAFE, AIDE, AutoGen), AutoML tools (AutoGluon, H2O, FLAML, Auto-Sklearn), and AutoML with preprocessing workflows (SAGA, L2C + AutoML)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include AIDE (2024), AutoGen (2024), CAAFE (2023), H2O, FLAML, AutoGluon — all recent and state-of-the-art systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 studies metadata impact with different configurations (Table 1, Figure 10). Section 5.3 compares original vs refined data catalog. CatDB vs CatDB Chain is also an ablation of the chaining strategy."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "AUC for binary classification, AUC-ovr for multiclass, R² for regression. Token cost and runtime are also reported as secondary metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The system generates ML pipelines evaluated by automated metrics (accuracy, AUC, R²). Human evaluation of generated code quality is not clearly relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: 'We divided all datasets into 70/30 train and test sets.' Tables report both train and test accuracy separately."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (20 datasets), per LLM (GPT-4o, Gemini-1.5, Llama3.1-70b), and per task type (binary, multiclass, regression)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses error types and frequencies (Figure 8). Tables show N/A and failure cases. Section 5.4 discusses where baselines fail. Section 4.3 discusses system limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 notes 'more metadata does not always improve pipeline quality.' CatDB underperforms some baselines on certain datasets (e.g., NYC regression in Table 7). Yelp shows increased runtime due to one-hot encoding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'accuracy comparable to or better than existing LLM-based systems, standalone AutoML tools' and 'orders of magnitude faster performance' are supported by Tables 5-8 and Figures 11-14."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about metadata improving pipeline quality are supported by ablation-style experiments in Section 5.2 (varying metadata configurations) and Section 5.3 (original vs refined catalog)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states limitations: only supervised learning, basic cleaning tasks, no library constraints. The scope is bounded to tabular data with specific dataset characteristics described in Table 3."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for observed results. For example, whether the improvements come from the data catalog approach specifically or from better prompt engineering generally is not disentangled."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o', 'Gemini-1.5-pro', and 'Llama3.1-70b' but does not specify snapshot dates or API versions for GPT-4o or Gemini-1.5-pro."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows a complete example prompt with actual content. Figures 6(a) and 6(b) show prompt templates. The system rules and catalog data structure are documented in detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 states 'LLM temperature set to zero.' The maximum error correction attempts (α₂) parameter is discussed in Algorithm 4."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline scaffolding is described: data profiling (Algorithm 1), metadata/rules extraction (Algorithm 2), prompt construction (Algorithm 3), pipeline generation with error handling (Algorithm 4, Figure 7)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes dataset preparation: 70/30 split, profiling, metadata storage. Table 3 describes all datasets with characteristics. The data catalog refinement process is detailed in Section 3.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'System Limitations' provides a dedicated discussion of three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 identifies specific limitations: only basic data cleaning supported, only supervised learning tasks, no library constraints enforcement. These are specific to this system."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states what is NOT supported: entity resolution, data augmentation from data lakes, unsupervised tasks, time-series and image data, library constraints."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact availability statement indicates data is available at the GitHub repository. Table 3 lists all 20 datasets with characteristics."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 describes all datasets with number of tables, rows, columns, task types, and classes. The error trace dataset is described in Table 2 with total request counts per LLM."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Datasets are standard benchmarks and real-world datasets described in Table 3."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from dataset profiling to prompt construction to pipeline generation is documented in Algorithms 1-4 and Sections 3-4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Concordia University and Technische Universität Berlin. No affiliation with evaluated LLM providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs (GPT-4o, Gemini-1.5-pro, Llama3.1-70b) on their ability to generate pipelines for datasets but does not state model training cutoff dates. The paper even acknowledges that LLMs may have seen benchmark datasets (Section 1 mentions Titanic)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly discusses that LLMs may have been exposed to well-known datasets during training (e.g., Titanic), which is why CatDB focuses on unseen datasets. This is a core motivation of the paper."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper motivates the problem of LLMs knowing benchmark datasets, it does not systematically analyze which of the 20 evaluation datasets may have been in the LLMs' training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption is reported extensively in Figures 12 and 13 across all datasets and LLMs. Section 4.1 provides a formal cost analysis (Equations 1-2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies hardware (Intel Core CPU with 32 vcores, 148 GB RAM). Runtime is reported in Tables 6 and 8, and Figures 12 and 9(a)."
       }
     }
   },
   "claims": [
     {
       "claim": "CatDB generates ML pipelines with accuracy comparable to or better than existing LLM-based systems, AutoML tools, and combined workflows.",
       "evidence": "Tables 5 and 7 show CatDB achieving top or competitive accuracy across 20 datasets against CAAFE, AIDE, AutoGen, H2O, FLAML, AutoGluon, and Auto-Sklearn.",
       "supported": "strong"
     },
     {
       "claim": "CatDB delivers up to orders of magnitude faster pipeline execution on large datasets.",
       "evidence": "Table 6 shows CatDB execution times of 1-4 seconds vs hundreds of seconds for baselines on most datasets. Table 8 shows CatDB completing all 8 datasets while baselines fail on several.",
       "supported": "strong"
     },
     {
       "claim": "Data catalog refinement improves pipeline accuracy by up to 52%.",
       "evidence": "Table 5 compares original vs refined accuracy on 6 datasets, with EU IT improving from 39.2% to 91.8% test accuracy (Section 5.3).",
       "supported": "strong"
     },
     {
       "claim": "CatDB is robust to data quality issues (outliers and missing values) compared to AutoML tools.",
       "evidence": "Figure 14 shows CatDB maintaining performance under increasing outlier and missing value injection while AutoML tools deteriorate.",
       "supported": "moderate"
     },
     {
       "claim": "More metadata does not always improve pipeline quality.",
       "evidence": "Section 5.2 and Figure 10 show that simple schema metadata sometimes matches or outperforms more detailed metadata configurations.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CatDB leverages data catalog information to guide LLMs in generating data-centric ML pipelines, achieving accuracy comparable to or better than state-of-the-art LLM-based systems (CAAFE, AIDE, AutoGen) and AutoML tools (H2O, FLAML, AutoGluon) across 20 diverse datasets. The system's data catalog refinement step can improve accuracy by up to 52% on datasets with data quality issues. CatDB demonstrates orders-of-magnitude faster pipeline execution times compared to baselines on large datasets, with zero failures across all 20 datasets and 3 LLMs. The chain prompting variant (CatDB Chain) effectively handles large datasets that exceed LLM context limits.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "Comparative claims of 'better' or 'comparable' performance are based on raw number comparisons without any statistical tests, despite the acknowledged variance in LLM outputs."
     },
     {
       "flag": "Unspecified model versions",
       "detail": "GPT-4o, Gemini-1.5-pro are used without snapshot dates or API version identifiers. LLM behavior can change across versions, making results potentially non-reproducible."
     },
     {
       "flag": "Selective dataset contamination discussion",
       "detail": "The paper acknowledges LLMs may know benchmark datasets (Titanic example) but does not analyze which of the 20 evaluation datasets may be in the training data of GPT-4o, Gemini, or Llama."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models for Automated Data Science: Introducing CAAFE for Context-Aware Automated Feature Engineering",
       "authors": ["Noah Hollmann", "Samuel Müller", "Frank Hutter"],
       "year": 2023,
       "relevance": "LLM-based feature engineering baseline directly compared to CatDB."
     },
     {
       "title": "AIDE: Human-Level Performance in Data Science Competitions",
       "authors": ["Dominik Schmidt", "Yuxiang Wu", "Zhengyao Jiang"],
       "year": 2024,
       "relevance": "End-to-end LLM-based ML solution generator used as a primary baseline."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework used as baseline for ML pipeline generation."
     },
     {
       "title": "Can Foundation Models Wrangle Your Data?",
       "authors": ["Avanika Narayan", "Ines Chami", "Laurel J. Orr", "Christopher Ré"],
       "year": 2022,
       "relevance": "Studies LLM capabilities for data wrangling tasks, directly relevant to LLM-based data processing."
     },
     {
       "title": "Calibrated Language Models Must Hallucinate",
       "authors": ["Adam Tauman Kalai", "Santosh S. Vempala"],
       "year": 2023,
       "arxiv_id": "2311.14648",
       "relevance": "Theoretical work on LLM hallucination that motivates CatDB's error handling approach."
     },
     {
       "title": "FLAML: A Fast and Lightweight AutoML Library",
       "authors": ["Chi Wang", "Qingyun Wu", "Markus Weimer", "Erkang Zhu"],
       "year": 2021,
       "relevance": "AutoML baseline system compared against CatDB."
     },
     {
       "title": "AutoGluon-Tabular: Robust and Accurate AutoML for Structured Data",
       "authors": ["Nick Erickson", "Jonas Mueller"],
       "year": 2020,
       "arxiv_id": "2003.06505",
       "relevance": "AutoML baseline for tabular data compared against CatDB."
     },
     {
       "title": "SAGA: A Scalable Framework for Optimizing Data Cleaning Pipelines for Machine Learning Applications",
       "authors": ["Shafaq Siddiqi", "Roman Kern", "Matthias Boehm"],
       "year": 2023,
       "relevance": "Data cleaning pipeline framework used in AutoML workflow baselines."
     }
   ]
 }