ai-research-survey

scan.json (22956B)

---

 {
   "paper": {
     "title": "Importance-Aware Data Selection for Efficient LLM Instruction Tuning",
     "authors": [
       "Tingyu Jiang",
       "Shen Li",
       "Yiyao Song",
       "Lan Zhang",
       "Hualei Zhu",
       "Yuan Zhao",
       "Xiaohang Xu",
       "Kenjiro Taura",
       "Hao Henry Wang"
     ],
     "year": 2025,
     "venue": "arXiv (AAAI 2026 copyright notice)",
     "arxiv_id": "2511.07074",
     "doi": "10.48550/arXiv.2511.07074"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes MIWV, a metric that selects instruction tuning data by measuring loss differences with and without one-shot ICL examples. Selecting just the top 1% of data by MIWV outperforms training on the full dataset across Alpaca and WizardLM datasets on LLaMA/LLaMA2/Qwen2.5 models. The method requires no external LLM or model training for data selection, taking 85 minutes versus 120-300 for competitors. Ablation studies show random selection, high prompt loss, and low MIWV all fail to match the full-data baseline.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (Alpaca, WizardLM, NIV2) and references them with citations. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 mentions PyTorch 2.0.1, Linux, 984GB RAM, Intel Xeon 8369B, Nvidia A100 80GB GPUs, but no requirements.txt, Dockerfile, or library version list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper mentions using the 'Alpaca codebase' but provides no scripts or commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported in any table or figure despite claiming experiments are repeated three times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of outperformance are based solely on comparing win rates and benchmark scores."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report absolute benchmark scores and win rates with baseline context (e.g., win rate relative to 1.0 baseline, specific ARC/HellaSwag/MMLU/TruthfulQA scores for both baseline and selected-data models)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1%, 5%, 10%, 15%, 20%, 25% data ratios were chosen, or why three repetitions are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'All experiments are repeated three times with arithmetic mean results reported' (Section 4.2) but no standard deviations or variance measures are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against 8 methods: IFD Score, SelectIT, Superfiltering, Alpagasus, Deita, DiverseEvol, Nuggets, RECOST. Full-dataset training serves as the primary baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent works like Deita (ICLR 2024), SelectIT (2024), Superfiltering (ACL 2024), RECOST (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 provides ablation on data selection strategy (random, high prompt loss, low MIWV vs. high MIWV) and on embedding model choice (Bge-en-large, Multilingual-e5-large, Gte-base-en-v1.5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation approaches: Pair-wise win rate (GPT-4 judge), Open LLM Leaderboard (ARC, HellaSwag, MMLU, TruthfulQA), and AlpacaEval."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. GPT-4 serves as the judge; no human evaluation of outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Five separate test datasets (Vicuna, Koala, WizardLM, Self-instruct, LIMA) are used, distinct from the training datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down Open LLM Leaderboard into ARC, HellaSwag, MMLU, TruthfulQA. Results are shown per test set and per data ratio."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure analysis. The single case study (Figure 5) shows only a success. No discussion of where the method breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Section 4.6 explicitly shows that random selection, high prompt loss, and low MIWV all produce models worse than the full-data baseline."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that top 1% outperforms full dataset are supported by Table 1 (win rates > 1.0 at 1%) and Figures 2a/2b."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('MIWV improves performance') are supported by controlled ablation: same model, same training setup, only the data selection strategy varies. The ablation design isolates the MIWV variable adequately."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims a 'universal data selection method' applicable to 'all LLMs' (Contribution 1), but tests only on LLaMA, LLaMA2, and Qwen2.5 model families. The title and abstract make broad claims not bounded to tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, the high-MIWV subset might simply have higher diversity (the t-SNE analysis hints at this) rather than the ICL mechanism being causal."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 win rate as a proxy for instruction-following quality without discussing limitations of LLM-as-judge (position bias acknowledged but deeper proxy issues unaddressed). Win rate measures GPT-4 preference, not necessarily actual capability improvement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified as 'LLaMA-7B', 'LLaMA2-7B/13B', 'Qwen2.5-7B/14B' without exact version strings or checkpoint identifiers. GPT-4 is used as judge without specifying the API version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix F) provides the full evaluation prompt used for GPT-4 judging. The one-shot ICL prompt construction is described mathematically in Section 3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A reports learning rate (2e-5), batch size (128), epochs (3), max input lengths (512/1024/2048). Embedding model specified as Bge-en-large."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a data selection pipeline, not an agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes dataset sizes and filtering: 'adopt existing methodologies to filter samples subject to AI censure in WizardLM, resulting in 63,655 instances.' The MIWV computation pipeline is documented in Section 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. The paper claims universality ('applicable to all LLMs') without stating what it does NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The MIWV scores, selected subsets, and per-sample rankings are not released. Only aggregate results are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Datasets are standard public benchmarks (Alpaca, WizardLM, NIV2) and their origins are described with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: embedding computation → cosine similarity retrieval → MIWV computation → ranking → subset selection. Figure 1 provides an overview diagram."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was supported by JST CREST Grant Number JPMJCR21M2, including the AIP Challenge Program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Alibaba Cloud Computing, Independent Researcher, University of Tokyo."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "JST CREST is a Japanese government research funding agency with no commercial stake in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Alibaba Cloud could have commercial interest in efficient training methods."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (LLaMA, LLaMA2, Qwen2.5)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether test datasets (Vicuna, Koala, etc.) overlap with the pre-training data of LLaMA/LLaMA2/Qwen2.5."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like ARC, HellaSwag, MMLU, TruthfulQA were all published before LLaMA2/Qwen2.5 training. No contamination discussion."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports data selection time: MIWV takes 85 minutes, compared to 8-300 minutes for alternatives."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (A100 GPUs) but total GPU hours for training experiments are not reported. Only data selection time is given."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Paper states experiments repeated 3 times but reports only means — no per-seed results or sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'All experiments are repeated three times with arithmetic mean results reported.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Training parameters are stated but it's unclear if they were tuned."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Multiple data ratios (1-25%) are tested but no clear validation-based selection criterion is described for choosing the best ratio."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across models, datasets, ratios, and test sets are made with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors re-implement baselines for comparison (Table 2) without acknowledging self-implementation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models trained on 1% data use far less compute than 100% baseline, but this compute advantage is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4-as-judge validity is not discussed beyond noting position bias mitigation. No discussion of whether win rate measures actual capability improvement."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved; this is a direct fine-tuning comparison."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LLaMA/LLaMA2/Qwen2.5 training data includes solutions to test benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the instruction tuning data or evaluation setup leaks information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Alpaca and WizardLM datasets were generated by LLMs; potential overlap with test sets is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Selecting the top 1% of data by MIWV outperforms training on the full dataset.",
       "evidence": "Table 1 shows win rates > 1.0 for 1% subsets across LLaMA2-7B/13B on both Alpaca and WizardLM. Figure 2 shows similar results.",
       "supported": "moderate"
     },
     {
       "claim": "MIWV outperforms 8 competing data selection methods.",
       "evidence": "Table 2 shows MIWV win rates of 1.119/1.211/1.178/1.234 at 1-15% ratios, higher than all competitors on the WizardLM test set.",
       "supported": "moderate"
     },
     {
       "claim": "The method is universal and applicable to all LLMs.",
       "evidence": "Tested on LLaMA-7B, LLaMA2-7B/13B, Qwen2.5-7B/14B, and T5-11B (Appendix D). All show improvement.",
       "supported": "weak"
     },
     {
       "claim": "ICL-guided data selection is more effective than IFD Score alone.",
       "evidence": "Table 3 shows ICL+IFD Score wins more than original IFD Score across 5 test sets, and MIWV outperforms both.",
       "supported": "moderate"
     },
     {
       "claim": "Higher MIWV samples are more diverse and cover the instruction space more uniformly.",
       "evidence": "Figure 6 t-SNE visualization shows high-MIWV samples spread uniformly while low-MIWV samples cluster. Figure 7 shows higher quality scores on 5 of 6 dimensions.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or error bars despite repeated experiments",
       "detail": "The paper states experiments are repeated 3 times but reports only means. Given the known high variance in LLM fine-tuning, the absence of any spread measure makes it impossible to assess whether differences are meaningful."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All claims of outperformance are based on raw number comparisons (win rates, benchmark scores) without any significance tests. Many differences appear small."
     },
     {
       "flag": "GPT-4 as sole automated judge",
       "detail": "Primary evaluation metric is GPT-4 pairwise comparison, known to have systematic biases. No human evaluation validates these judgments."
     },
     {
       "flag": "Overclaiming universality",
       "detail": "Claims method is 'universal' and 'applicable to all LLMs' from testing on only 3 model families (LLaMA, Qwen, T5). No limitations section acknowledges this gap."
     },
     {
       "flag": "No limitations section",
       "detail": "Complete absence of limitations, threats to validity, or scope boundaries discussion."
     },
     {
       "flag": "Company affiliation without conflict acknowledgment",
       "detail": "6 of 9 authors are from Alibaba Cloud, which has commercial interest in efficient LLM training. No competing interests statement."
     },
     {
       "flag": "AlpacaEval budget limitation",
       "detail": "Appendix B.3 admits 'Due to budget constraints, evaluations are conducted on only 5% of the dataset' for AlpacaEval, but Table 1 presents these results without caveat."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions",
       "authors": ["Y. Wang", "Y. Kordi", "S. Mishra"],
       "year": 2023,
       "arxiv_id": "2212.10560",
       "relevance": "Foundational work on instruction data generation for LLM alignment."
     },
     {
       "title": "AlpaGasus: Training a Better Alpaca with Fewer Data",
       "authors": ["L. Chen", "S. Li", "J. Yan"],
       "year": 2024,
       "relevance": "Baseline method for instruction data selection using ChatGPT filtering."
     },
     {
       "title": "What Makes Good Data for Alignment? A Comprehensive Study of Automatic Data Selection in Instruction Tuning",
       "authors": ["W. Liu", "W. Zeng", "K. He"],
       "year": 2024,
       "relevance": "Deita method — baseline for instruction data quality and diversity selection."
     },
     {
       "title": "From Quantity to Quality: Boosting LLM Performance with Self-Guided Data Selection for Instruction Tuning",
       "authors": ["M. Li", "Y. Zhang", "Z. Li"],
       "year": 2024,
       "relevance": "IFD Score method for self-guided instruction data selection, key baseline."
     },
     {
       "title": "Superfiltering: Weak-to-Strong Data Filtering for Fast Instruction-Tuning",
       "authors": ["M. Li", "Y. Zhang", "S. He"],
       "year": 2024,
       "relevance": "Efficient data filtering baseline achieving fastest selection time."
     },
     {
       "title": "LESS: Selecting Influential Data for Targeted Instruction Tuning",
       "authors": ["M. Xia", "S. Malladi", "S. Gururangan"],
       "year": 2024,
       "relevance": "Data selection via importance resampling for instruction tuning."
     },
     {
       "title": "LIMO: Less is More for Reasoning",
       "authors": ["Y. Ye", "Z. Huang", "Y. Xiao"],
       "year": 2025,
       "arxiv_id": "2502.03387",
       "relevance": "Recent work showing minimal high-quality data suffices for LLM reasoning."
     },
     {
       "title": "Recost: External knowledge guided data-efficient instruction tuning",
       "authors": ["Q. Zhang", "Y. Zhang", "H. Wang"],
       "year": 2024,
       "arxiv_id": "2402.17355",
       "relevance": "Baseline method using external knowledge for data-efficient instruction tuning."
     },
     {
       "title": "WizardLM: Empowering Large Language Models to Follow Complex Instructions",
       "authors": ["C. Xu", "Q. Sun", "K. Zheng"],
       "year": 2023,
       "arxiv_id": "2304.12244",
       "relevance": "Source dataset and baseline for instruction tuning evaluation."
     },
     {
       "title": "Lima: Less is more for alignment",
       "authors": ["C. Zhou", "P. Liu", "P. Xu"],
       "year": 2024,
       "relevance": "Influential work showing small curated datasets suffice for LLM alignment."
     }
   ]
 }