ai-research-survey

scan.json (27043B)

---

 {
   "paper": {
     "title": "CLASS-IT: Conversational and Lecture-Aligned Small-Scale Instruction Tuning for BabyLMs",
     "authors": [
       "Luca Capone",
       "Alessandro Bondielli",
       "Alessandro Lenci"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.25364"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or repository URL is provided in the paper. The dataset name 'colinglab/CLASS_IT' is mentioned but the paper states 'The full dataset will be released in the future following appropriate validation,' indicating it is not yet available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'colinglab/CLASS_IT' as the dataset name but explicitly states 'The full dataset will be released in the future following appropriate validation.' Only a representative portion was used and no download link is provided. Pre-training data is from the BabyLM Challenge organizers (publicly available), but their processed instruction tuning dataset is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is provided. The paper describes model architectures and training hyperparameters but does not specify library versions, framework versions, or dependency information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While Tables 1 and 2 give architecture and training hyperparameters, there is no README, reproduction script, or detailed guide for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports IQR (Inter-Quartile Ranges) for z-score distributions across tasks (Figures 3, 5, 6), but does not report confidence intervals or error bars for the individual task results. The bar charts in Figures 2 and 4 show point estimates without any uncertainty measure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'instruction tuning yields modest but measurable gains,' 'sequential curricula outperforming merged data') but does not use any statistical significance tests. Comparisons are based on visual inspection of bar charts and z-score distributions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, percentage improvements with baselines) are reported. The paper describes results qualitatively ('small but consistent gains,' 'markedly worse') without quantifying the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests two model sizes (100M and 140M) with multiple instruction tuning configurations but does not justify why these sizes were chosen, nor is there any discussion of whether the number of configurations or evaluation tasks provides sufficient power for their claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "IQR is reported for z-score distributions across tasks (Figures 3, 5, 6), but this measures variance across tasks, not across experimental runs. There is no indication that experiments were run multiple times with different seeds, and no standard deviation across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against official BabyLM Challenge baselines: bl_gpt2-100M, bl_gptbertmixed-100M, and bl_simpo (Section 5, shown in blue in Figure 2). Pre-trained versions of their own models also serve as baselines for measuring instruction tuning effects."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are the official baselines from the BabyLM Challenge, which is the contemporary competition context for this work. These represent the current standard for this specific task and setting."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares: (1) pre-trained only vs. instruction-tuned, (2) merged vs. sequential curriculum, (3) different orderings of sequential instruction tuning (switch_wiki vs. wiki_switch), and (4) training on individual datasets (Switchboard only, Wikipedia only). This constitutes an ablation of the instruction tuning components and strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on multiple metrics across multiple benchmarks: SuperGLUE tasks (fine-tuning accuracy on BoolQ, CB, COPA, MultiRC, RTE, WSC, QQP, MNLI), BLiMP, EWoK, WUGs, entity tracking (accuracy), and psycholinguistic correlation (change in R²). Z-scores are also computed as a global summary metric."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates small-scale language models on benchmark tasks. The claims are about benchmark performance and correlations with existing human psycholinguistic data, not about output quality that requires human judgment. Human evaluation is not clearly relevant to the claims made."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'Models are evaluated on the test set' for fine-tuning evaluation, with 'a randomly sampled 10k portion of the original training set for the task' used for fine-tuning. The zero-shot evaluation uses separate benchmark datasets. The pre-training data was split 90/10 into training and validation sets (Section 4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figures 2 and 4 provide per-task breakdowns for all SuperGLUE tasks and zero-shot tasks respectively. The paper discusses performance differences across individual tasks (e.g., QQP, MNLI, WUGs, entity tracking, R² tasks) rather than reporting only aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where their models fail: 100M models are 'markedly worse' on QQP and MNLI (Section 5), all models struggle on entity tracking (Section 5), instruction tuning does not consistently help on zero-shot tasks, and the Switchboard-only model performs worst. The Discussion section (Section 6) analyzes why instruction tuning may hurt zero-shot performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: instruction tuning does not consistently transfer to zero-shot tasks, merged instruction tuning is worse than sequential, Switchboard-only instruction tuning performs worst, and instruction-tuned models that have seen more data do not consistently outperform pre-trained ones (Section 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'small but consistent gains in fine-tuning scenarios, with sequential curricula outperforming merged data' and that 'improvements do not consistently transfer to zero-shot tasks.' These are supported by the results in Figures 2-6 and the discussion in Sections 5-6. The claims are hedged appropriately ('small but consistent,' 'do not consistently transfer')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through its ablation design (instruction tuning causes improvements in fine-tuning). These are supported by controlled single-variable manipulations: comparing pre-trained vs. instruction-tuned variants, merged vs. sequential, and different orderings, while holding other variables constant. The experimental design is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently bounds its claims to 'BabyLM-scale models,' 'small-scale LMs,' and 'ecological training limits.' The abstract and conclusion explicitly note results are about models with 100M-140M parameters trained on ~100M words. The discussion acknowledges that 'larger models have been shown to not suffer from similar issues' (Section 6), properly bounding generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses alternative explanations: the instruction tuning dataset imbalance between conversational and QA data may bias results; the evaluation criteria (classification tasks, log-likelihood) may undermine instruction-tuned models; the Switchboard corpus may be too limited in register. Section 7 also discusses how smaller models' better psycholinguistic correlation may reflect a known phenomenon rather than their specific approach."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'LLaMA-3.2-3B-Instruct' for data augmentation (Section 3.2) but does not provide a snapshot date or specific version identifier. For their own models, architecture details are given in Table 1 but these are custom-trained models, not versioned API models. The LLaMA reference lacks a precise version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompt used for generating the Simple Wikipedia augmentation data is provided in Section 3.2: 'Based on the following text, generate 3 questions and detailed, informative answers...' This is the only prompt used in the study (for data augmentation with LLaMA-3.2-3B-Instruct)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide detailed hyperparameters: vocab size, max length, hidden size, attention heads, layers, trainable parameters (Table 1), and learning rate, batch size, epochs, LR scheduler, warm-up steps for both pre-training and instruction tuning (Table 2). Section 4 also states 'We used the same token-level cross-entropy loss' and that loss is computed only on target tokens during instruction tuning."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard pre-training and instruction tuning pipeline with no tool use, retry logic, or agent-based architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes pre-training data preprocessing: special character removal, discarding entries with two words or fewer, concatenation of same-speaker Switchboard utterances. Section 3.2 describes instruction tuning data processing: merging consecutive same-speaker utterances, sliding window segmentation scheme for Switchboard, and the LLaMA augmentation process for Simple Wikipedia. Word counts are provided at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusion (Section 7), discussing constraints of instruction tuning dataset size, data balance issues, and partial validation of the augmentation process."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) the instruction tuning dataset is small relative to pre-training, which may reduce impact; (2) conversational data is underrepresented relative to QA data, potentially biasing results; (3) the Simple Wikipedia augmentation was only partially validated. Section 7 also discusses the evaluation methodology limitation specific to their setting."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: results apply to BabyLM-scale models (100M-140M parameters), ecological training data limits (~100M words), decoder-only architectures, and the specific evaluation tasks from the BabyLM Challenge. Section 6 notes 'larger models have been shown to not suffer from similar issues' and Section 7 acknowledges the evaluation criteria may undermine their models' true capabilities."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The pre-training data comes from the BabyLM Challenge (publicly available from organizers), but the processed instruction tuning dataset (colinglab/CLASS_IT) is stated as not yet released: 'The full dataset will be released in the future following appropriate validation.' Raw experimental results (individual task scores) are shown in figures but not provided as downloadable data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: pre-training data comes from the BabyLM Challenge organizers (~100M words from CHILDES, Gutenberg, BNC, OpenSubtitles, Switchboard, Simple Wikipedia). The instruction tuning data construction process is described for both Switchboard (sliding window approach, 38,802 items, ~1.3M words) and augmented Simple Wikipedia (97,697 items, 18M words)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The paper uses existing corpora and benchmark datasets for model training and evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Sections 3.1 and 3.2: raw data from organizers → preprocessing (special character removal, short entry filtering, speaker concatenation) → 91M words pre-training corpus. For instruction tuning: Switchboard → sliding window segmentation → 38,802 items; Simple Wikipedia → LLaMA augmentation → 97,697 items → subset selection to stay within 100M word limit. Word counts are given at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding: PRIN 2022 Project (CUP I53D23004050006) from the Italian Ministry of University and Research (MUR), and PNRR FAIR project funded by the European Commission under NextGeneration EU."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Luca Capone and Alessandro Lenci at CoLing Lab, University of Pisa (Dept. of Philology, Literature and Linguistics); Alessandro Bondielli at both the CoLing Lab and Dept. of Computer Science, University of Pisa."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are the Italian Ministry of University and Research and the European Commission (NextGeneration EU). These are government research funding bodies with no financial stake in the outcome of BabyLM experiments."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While this is an academic paper from a university lab with government funding (low conflict risk), the absence of an explicit declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLaMA-3.2-3B-Instruct for data augmentation but does not state its training data cutoff date. For their own trained BabyLM models, the training data is specified but no formal cutoff is discussed relative to the evaluation benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between pre-training data and evaluation benchmarks. The pre-training corpus includes Gutenberg, BNC, OpenSubtitles, and CHILDES; some evaluation benchmarks (BLiMP, SuperGLUE) could have overlapping content, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation benchmarks (BLiMP published 2020, SuperGLUE, EWoK) were available before the model's training data was assembled, but the paper does not discuss whether any benchmark examples could appear in the pre-training corpora. Additionally, LLaMA-3.2-3B-Instruct was used for data augmentation, and its potential contamination with benchmark data is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of language models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for either the data augmentation with LLaMA-3.2-3B-Instruct or the evaluation of their models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, training time, or total compute budget is reported. The paper states word counts processed (908M words total) but does not specify the computational resources used."
       }
     }
   },
   "claims": [
     {
       "claim": "Instruction tuning yields small but consistent gains in fine-tuning scenarios for BabyLM-scale models.",
       "evidence": "Figure 2 shows instruction-tuned models are generally competitive with or slightly above pre-trained baselines on SuperGLUE tasks. Figure 3 z-score distributions show instruction-tuned variants generally have comparable or higher medians than pre-trained models. Section 5 states 'in all cases there is at least an instruction-tuned model better than the pre-trained one.'",
       "supported": "moderate"
     },
     {
       "claim": "Sequential curricula (presenting conversational and QA data separately) outperform merged data for instruction tuning.",
       "evidence": "Figure 3 and Section 5 show that sequentially-trained models 'have a very similar median score, but a much smaller IQR, and are the only two models with all z-scores above zero.' Figure 6 confirms 'tuning the model sequentially on different datasets is consistently better than doing so on a mixture of the datasets.'",
       "supported": "moderate"
     },
     {
       "claim": "Instruction tuning improvements do not consistently transfer to zero-shot tasks.",
       "evidence": "Figure 4 and Section 5 report 'we do not observe striking differences between pre-trained and instruction-tuned models' in zero-shot evaluation. Figure 5 z-score distribution shows 'no clear trend in favour or against instruction tuning.' Section 6 discusses this as a potential trade-off between interaction-focused adaptation and broad linguistic generalization.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller (100M) models correlate better with human psycholinguistic data than larger (140M) models.",
       "evidence": "Section 5 reports '100 million model variants are vastly superior to both the 140 million models and the baselines' on R²-based psycholinguistic correlation tasks (Figure 4 right panel). Section 6 notes this is 'in line with previous literature where smaller models often correlate better with human psychometric data.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Instruction tuning of BabyLM-scale models (100M-140M parameters) provides small but measurable gains on fine-tuning tasks (SuperGLUE), particularly when conversational and QA datasets are presented sequentially rather than merged. However, these improvements do not consistently transfer to zero-shot tasks (BLiMP, EWoK, WUGs, entity tracking, psycholinguistic correlation), suggesting instruction tuning may narrow model capabilities at this scale. Smaller 100M-parameter models showed stronger correlations with human psycholinguistic data than 140M models, consistent with prior work on scaling and cognitive modeling.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (instruction tuning helps, sequential beats merged, etc.) are based on visual inspection of bar charts and z-score distributions without any significance testing. Given the small and inconsistent differences reported, it is unclear whether observed gains are reliable or within noise."
     },
     {
       "flag": "No multi-run variance reporting",
       "detail": "Results appear to be from single training runs. No standard deviations across random seeds are reported. For small models where training is cheap, the absence of multi-seed experiments makes it impossible to assess result stability."
     },
     {
       "flag": "No compute or cost reporting",
       "detail": "Despite training multiple model configurations across pre-training and instruction tuning stages, no GPU hours, hardware, or training time is reported, making it difficult to assess practical reproducibility."
     },
     {
       "flag": "Augmented data quality unvalidated",
       "detail": "The Simple Wikipedia augmentation using LLaMA-3.2-3B-Instruct was 'only partially validated' per the Limitations section. The quality of this synthetic data directly affects the instruction tuning results but is not systematically evaluated."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Foundational paper on emergent abilities in LLMs, relevant to understanding scaling effects and capability emergence in AI models."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges the emergent abilities narrative, arguing it may be a measurement artifact, relevant to methodology of evaluating LLM capabilities."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "relevance": "Describes the LLaMA model family used for data augmentation, relevant to understanding LLM architectures and their applications."
     },
     {
       "title": "Baby's CoThought: Leveraging large language models for enhanced reasoning in compact models",
       "authors": ["Zheyu Zhang", "Han Yang", "Bolei Ma"],
       "year": 2023,
       "relevance": "Uses LLMs to create educational datasets for training smaller models, directly relevant to LLM-assisted training and knowledge distillation."
     },
     {
       "title": "BabyStories: Can reinforcement learning teach baby language models to write better stories?",
       "authors": ["Xingmeng Zhao", "Tongnian Wang", "Sheri Osborn", "Anthony Rios"],
       "year": 2023,
       "relevance": "Explores reinforcement learning for training small language models, relevant to alternative training paradigms for LMs."
     },
     {
       "title": "Elements of World Knowledge (EWoK): A cognition-inspired framework for evaluating basic world knowledge in language models",
       "authors": ["Anna A. Ivanova"],
       "year": 2024,
       "relevance": "Evaluation framework for assessing world knowledge in LMs, relevant to LLM evaluation methodology."
     },
     {
       "title": "Curriculum learning: A survey",
       "authors": ["Petru Soviany", "Radu Tudor Ionescu", "Paolo Rota", "Nicu Sebe"],
       "year": 2022,
       "relevance": "Comprehensive survey of curriculum learning approaches, relevant to training methodology for AI models."
     },
     {
       "title": "Bridging the data gap between children and large language models",
       "authors": ["Michael C. Frank"],
       "year": 2023,
       "relevance": "Discusses the data efficiency gap between human and machine learning, relevant to understanding LLM data requirements."
     },
     {
       "title": "Explica: Evaluating explicit causal reasoning in large language models",
       "authors": ["Martina Miliani", "Serena Auriemma", "Alessandro Bondielli"],
       "year": 2025,
       "relevance": "Evaluates causal reasoning in LLMs, relevant to LLM capability assessment methodology."
     }
   ]
 }