ai-research-survey

scan.json (24315B)

---

 {
   "paper": {
     "title": "Instruction Tuning of Large Language Models for Tabular Data Generation—in One Day",
     "authors": [
       "Milad Abdollahzadeh",
       "Abdul Raheem",
       "Zilong Zhao",
       "Uzair Javaid",
       "Kevin Yee",
       "Nalam Venkata Abhishek",
       "Tram Truong-Huu",
       "Biplab Sikdar"
     ],
     "year": 2025,
     "venue": "ICML 2025 (PMLR 267)",
     "arxiv_id": "2511.23220",
     "doi": "10.48550/arXiv.2511.23220"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Instruction tuning Llama3.1-8B-Instruct on a small (7K) but high-quality instruction dataset for tabular data generation, using a single A100 GPU for under 6 hours, yields fidelity and utility metrics competitive with GPT-4o across 20 datasets. The base LLM without instruction tuning largely fails to follow tabular generation instructions (~80% of output is irrelevant). The approach generalizes to another base LLM (TableLlama) in supplementary experiments.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The source datasets are publicly available, but the authors' constructed instruction dataset (10K instructions with metadata) is not released. No download link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions A100 80GB GPU, HuggingFace Transformers, and DeepSpeed ZeRO-2, but provides no requirements.txt, library versions, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method description in Sections 4-5 gives high-level steps but not enough detail to replicate without guessing."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates for Shape, Trends, AUC, and R2. No confidence intervals, error bars, or ± notation appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance 'on par with GPT-4o' based solely on comparing numbers in tables without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-2 report absolute metric values for base LLM, ITT-GEN, and GPT-4o, allowing the reader to assess the magnitude of improvement (e.g., breast cancer Shape: 55.31 → 84.12 → 78.65)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 20 datasets, 500 training instances per dataset, N=20 rows per instance, or any other sample size decision."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported across experimental runs. Results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against two baselines: the base LLM (Llama3.1-8B-Instruct) and GPT-4o (Section 5.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o is a contemporary and capable commercial LLM. The comparison is reasonable for the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The system includes metadata design, instruction format, and fine-tuning, but no experiment removes individual components (e.g., effect of metadata, effect of instruction format, effect of dataset size)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses fidelity metrics (Shape, Trends) and utility metrics (AUC, R2 via TSTR framework with three ML models) as described in Section 5.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated tabular data quality. All evaluation is automated via fidelity and utility metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "6 of 20 datasets are held out as unseen out-of-domain (OoD) evaluation datasets, separate from the 14 training datasets (Section 4.1, Table 3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide per-dataset results across all 20 datasets rather than just aggregate averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses base LLM failure (generating irrelevant text, Figure 1) but does not analyze when or why ITT-GEN itself fails. Some datasets show notably poor performance (e.g., job posting, Players2024) without discussion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows ITT-GEN performing well. No failed approaches, abandoned configurations, or ablations that hurt performance are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'performance on par with GPT-4o.' However, Tables 1-2 show GPT-4o substantially outperforms ITT-GEN on many datasets (e.g., adult Shape 85.73 vs 92.34; california housing Shape 73.29 vs 96.27; bank utility 0.616 vs 0.820). 'On par' overstates the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that instruction tuning improves tabular data generation — is supported by comparing the same base model before and after fine-tuning, which is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Large Language Models' (plural) but the main experiments test only Llama3.1-8B-Instruct. TableLlama results in the appendix use a weaker, older model. Claims of general LLM applicability are not bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the improvement could partly come from the model memorizing the 20 dataset formats rather than learning general tabular generation, or the metadata could be doing most of the work."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Shape (distributional similarity), Trends (correlation preservation), and TSTR utility, and frames these as tabular data generation quality metrics. The claims match the granularity of the measurements without overclaiming to a broader construct."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Llama3.1-8B-Instruct is specific, but GPT-4o is used without a snapshot date or API version. Per schema, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows a complete example instruction including the full prompt structure with metadata, input table, question, and expected answer. Figure 3 shows the GPT-4o prompt template for metadata generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports: learning rate 2e-5, batch size 3, 2 epochs, A100 80GB GPU, DeepSpeed ZeRO-2 stage."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is straightforward fine-tuning and inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the dataset construction pipeline: 20 public datasets sampled across 10 topics, 14/6 train/eval split, 500/100 instances per dataset, N=20 rows per instance, metadata generated via GPT-4o with manual review."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 1 has a 'Limitations' paragraph about the research gap (high data/compute requirements), not about limitations of the authors' own method. No dedicated limitations section discusses the shortcomings of ITT-GEN."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. No acknowledgment of potential issues with the evaluation, dataset selection, or generalizability of results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings, models, or data types the results do not apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The constructed instruction dataset is not released. Source datasets are public, but the processed instruction data and generated outputs are not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: 20 public datasets from 10 topics, instruction construction procedure, metadata generation via GPT-4o, and manual review process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets listed in Table 3."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from dataset selection through instruction creation to training is documented in Section 4.1: dataset sampling → metadata generation → instruction construction → train/eval split → mixing and shuffling."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Singapore Institute of Technology, Betterdata AI (a synthetic data company), and National University of Singapore."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making independence unverifiable. Betterdata AI is a synthetic data company with direct commercial interest in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement. Multiple authors are from Betterdata AI, a company whose product is synthetic data generation — directly relevant to the paper's findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff is stated for GPT-4o or Llama3.1-8B-Instruct. The public datasets used (e.g., adult, iris, boston housing) are decades old and almost certainly in the training data of both models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the public evaluation datasets (or similar tables) appeared in the pre-training data of GPT-4o or Llama3.1."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Classic ML datasets like iris, adult, boston housing, and diabetes have been on the internet for decades and are almost certainly in LLM training data. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per generation is reported. GPT-4o API costs for the comparison are not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states training used 'an A100 80GB GPU for 2 epochs' and the abstract/conclusion state 'less than 6 hours' of training time."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results are from one run or averaged over multiple."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The chosen hyperparameters (lr=2e-5, batch=3, epochs=2) appear without justification or search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the reported configuration was selected. No mention of validation-based selection or alternatives tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (ITT-GEN) against baselines without acknowledging author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "ITT-GEN uses a single A100 for 6 hours while GPT-4o is a massively larger model. This compute disparity is framed as a positive ('limited resources') but performance as a function of compute is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Shape, Trends, and TSTR metrics actually capture the quality of tabular data generation. The metrics are adopted from prior work without questioning their validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the approach. Models are evaluated directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Classic datasets like iris (1936), adult (1994), boston housing (1978) have been on the internet for decades and are almost certainly in the training data of both LLMs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation provides 20 input rows and asks for 20 generated rows — it is unclear whether models could be memorizing and regurgitating known rows."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Training and evaluation instances are constructed from the same datasets (different row samples), and the public datasets may be in the LLMs' pre-training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Instruction tuning on 7K instructions with a single A100 GPU for less than 6 hours yields tabular data generation performance on par with GPT-4o.",
       "evidence": "Tables 1-2 show fidelity (Shape, Trends) and utility (AUC, R2) metrics. ITT-GEN is competitive on some datasets but substantially lower on others (e.g., california housing Shape 73.29 vs 96.27, bank utility 0.616 vs 0.820).",
       "supported": "weak"
     },
     {
       "claim": "The base LLM (Llama3.1-8B-Instruct) fails to follow tabular data generation instructions without instruction tuning.",
       "evidence": "Figure 1 shows the base LLM generating irrelevant text instructions instead of tabular data. Section 5.2 notes ~80% of base LLM output is non-tabular.",
       "supported": "strong"
     },
     {
       "claim": "The approach is model-agnostic and also improves TableLlama's tabular data generation performance.",
       "evidence": "Supplementary Tables 4-5 show TableLlama base cannot generate structured output at all, while ITT-GEN fine-tuned TableLlama produces usable tabular data, though with a large gap to GPT-4o.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first work to explore instruction tuning for tabular data generation.",
       "evidence": "Section 1 and related work (Section 2) state that prior tabular instruction tuning focused on QA and reasoning tasks only.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Overclaiming: 'on par with GPT-4o'",
       "detail": "Tables 1-2 show GPT-4o substantially outperforms ITT-GEN on many datasets. For example, california housing Shape (73.29 vs 96.27), adult Shape (85.73 vs 92.34), bank utility (0.616 vs 0.820). Claiming 'on par' is generous."
     },
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results appear to be single-run point estimates with no uncertainty quantification. Given that fine-tuning with different random seeds can produce meaningfully different results, this is a significant omission."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Multiple authors (4 of 8) are from Betterdata AI, a commercial synthetic data company. The paper demonstrates that limited compute can produce competitive synthetic tabular data — directly relevant to the company's business. No competing interests statement is provided."
     },
     {
       "flag": "Contamination risk with classic datasets",
       "detail": "Evaluation uses well-known datasets (iris, adult, boston housing, diabetes) that are ubiquitous on the internet and almost certainly in both LLMs' training data. GPT-4o may have memorized these datasets. This is not discussed."
     },
     {
       "flag": "No ablation study",
       "detail": "The instruction dataset includes carefully designed metadata, but no ablation measures the contribution of metadata vs. instruction format vs. dataset diversity. It is unclear what drives the improvement."
     },
     {
       "flag": "Base LLM comparison is unfair",
       "detail": "Fidelity metrics for the base LLM are computed on only ~20% of its output (the portion that happens to be tabular), making the base LLM look more competitive than it actually is for fidelity but masking the real failure mode."
     }
   ],
   "cited_papers": [
     {
       "title": "TableLlama: Towards Open Large Generalist Models for Tables",
       "authors": ["Tianshu Zhang", "Xiang Yue", "Yifei Li", "Huan Sun"],
       "year": 2024,
       "relevance": "State-of-the-art open-source LLM for table understanding tasks, used as baseline in this paper's supplementary experiments."
     },
     {
       "title": "TableLLM: Enabling Tabular Data Manipulation by LLMs in Real Office Usage Scenarios",
       "authors": ["Xiaojuan Zhang", "Shuo Luo", "Bowen Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19318",
       "relevance": "Tabular instruction tuning for LLMs on operations like QA and Pandas code generation."
     },
     {
       "title": "Rethinking Table Instruction Tuning",
       "authors": ["Naihao Deng", "Rada Mihalcea"],
       "year": 2025,
       "arxiv_id": "2501.14693",
       "relevance": "Analyzes hyperparameter selection impact on efficient tabular instruction tuning."
     },
     {
       "title": "HARMONIC: Harnessing LLMs for Tabular Data Synthesis and Privacy Protection",
       "authors": ["Yuxin Wang"],
       "year": 2024,
       "arxiv_id": "2408.02927",
       "relevance": "LLM-based tabular data generation via text conversion and fine-tuning."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Base model family used in this study (Llama3.1-8B-Instruct)."
     },
     {
       "title": "GPT-4o System Card",
       "authors": ["Aaron Hurst"],
       "year": 2024,
       "arxiv_id": "2410.21276",
       "relevance": "Commercial LLM used as primary comparison baseline."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov"],
       "year": 2023,
       "relevance": "DPO method used in post-training of the base Llama model."
     },
     {
       "title": "Instruction Tuning for Large Language Models: A Survey",
       "authors": ["Shengyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.10792",
       "relevance": "Survey of instruction tuning methods for LLMs."
     },
     {
       "title": "Finetuned Language Models Are Zero-Shot Learners",
       "authors": ["Jason Wei"],
       "year": 2022,
       "arxiv_id": "2109.01652",
       "relevance": "Foundational work on instruction tuning improving LLM task generalization."
     }
   ]
 }