ai-research-survey

scan.json (26722B)

---

 {
   "paper": {
     "title": "Using Scaling Laws for Data Source Utility Estimation in Domain-Specific Pre-Training",
     "authors": [
       "Oleksiy Ostapenko",
       "Charles Guille-Escuret",
       "Luke Kumar",
       "Max Tian",
       "Denis Kocetkov",
       "Gopeshh Subbaraj",
       "Raymond Li",
       "Joel Lamy-Poirier",
       "Sebastien Paquet",
       "Torsten Scholak"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.22250",
     "doi": "10.48550/arXiv.2507.22250"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references Fast-LLM (github.com/ServiceNow/Fast-LLM) as the training framework, but does not release experiment code, configuration files, or scripts for reproducing the scaling law analysis."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The base datasets (FineWeb-Edu, Dolma) are public, but the paper's curated domain-specific datasets (MBF-filtered data, WRAP-generated data, instruction-augmented data) are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 64 H100 GPUs, FlashAttention 2, ZeRO stage 3, and Fast-LLM with a specific git SHA (ff1486d), but provides no requirements.txt, Dockerfile, or library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology but does not include runnable scripts or a README with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Figures show point estimates connected by scaling curves without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims about one method outperforming another are based on visual comparison of scaling curves without any formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Brier Score Δ and Exact Match Δ relative to the full replay baseline (Equation 2), providing both the magnitude and direction of improvements in context. Figures 5 and 6 show these deltas across compute scales."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 6 annealing durations (1k-36k steps) were chosen, nor why 2 seeds for baseline and 1 seed for other methods is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We run two seeds and average the results for the full replay baseline, and only use a single seed for other baselines to minimize compute cost' (Section 3.3). No standard deviations or variance measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Full replay (continued training on pretraining data without domain-specific upsampling) serves as the baseline. Multiple data sourcing methods are compared against it."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include WRAP (Maini et al., 2024), TinyGSM-MIND (OLMo et al., 2024), instruction augmentation (Cheng et al., 2024), and MBF following recent practices. All are contemporary methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 ablates WRAP variants: WRAP with/without MMLU-style Q/A and WRAP+Q/A on unrelated Wikipedia documents, isolating the effect of formatting from domain knowledge. MBF threshold ablation is also reported (Appendix D.1)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Brier Score (primary) and Exact Match across multiple evaluation formats: continuation (CF), multiple choice (MC), and generative, as detailed in Table 1."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to this paper's claims about data source scaling behavior for model training."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation is conducted on standard MMLU benchmark tasks, GSM8k, and Hendrycks MATH — established test sets separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (medical vs math), evaluation format (CF, MC, generative), and individual data sourcing method. Table 1 lists all tasks by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses WRAP's diminishing returns at scale due to low diversity (Section 4.1), and instruction augmentation's failure to outperform full replay on CF tasks (Section 4.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Instruction augmentation does not outperform full replay on CF tasks (Section 4.1). TinyGSM shows poor scaling (R²=0.1) on math domain. WRAP's utility degrades at scale."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that rankings are not invariant across scales (demonstrated in Fig. 1), scaling curves can be constructed per source (Figs. 5-6), and validates on 7B model with medical and math domains. All are supported by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is from ablation studies: Fig. 3 shows controlled removal of MMLU-style Q/A from WRAP, demonstrating the formatting effect. The diversity hypothesis for WRAP's poor scaling is appropriately framed as a hypothesis ('We hypothesize,' Section 4.1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract and Section 3 explicitly state validation is on 'a pre-trained model with 7 billion parameters' with two specific domains. Section 4.3 acknowledges inability to verify generalization to larger scales."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses diversity as an alternative explanation for scaling differences (Section 4.1, Figs. 4 and 15), and format sensitivity as an alternative explanation for apparent method effectiveness (Section 3.2, Fig. 3)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines the utility function U(D) (Equation 2) and discusses why Brier Score is preferred over accuracy as a metric, citing Schaeffer et al. (2023) on how metric choice affects emergent behavior detection."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Meta-Llama-3-70B-Instruct for annotation, Meta-Llama-3.2-3B-Instruct for WRAP, Qwen2.5-7B-Instruct for MIND rephrasing. Their own base model architecture is described as Mistral-7b-based with full training details in Appendix C.1."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Figures 7-12: 5-point scoring prompts for math and medical MBF (Figs. 7-8), and WRAP style prompts for scholar, Q/A, MMLU-style Q/A, and Wikipedia formats (Figs. 9-12)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters in Section 3.3 and Appendix C.1: learning rate 3e-4, AdamW with β1=0.9, β2=0.95, batch size 256, sequence length 8192, 10% upsampling ratio, annealing steps 1k-36k, MBF threshold 2.5."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains and evaluates language models directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "MBF filtering pipeline is described: 500k examples annotated by Llama-3-70B, BERT regressor trained, threshold of 2.5 selected via ablation (Appendix D.1). WRAP generation process is detailed in Appendix D.2 with seed selection criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'Limitations' provides substantive discussion of the approach's constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses specific threats: batch sampling stochasticity at small compute, WRAP's variability distorting scaling coefficients, inability to ablate upsampling ratio and checkpoint sensitivity, and unknown generalization to larger compute scales."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states what was not tested: sensitivity to upsampling ratio, impact of starting checkpoint/learning rate, generalization to scales 'orders of magnitude larger,' and ablations across tunings of a single data source."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-run metrics, individual evaluation scores) is released. Only aggregated results in figures are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: MBF uses BERT regressor trained on 500k Llama-3-70B annotations (Appendix D.1), WRAP uses 1M highly-scored MBF documents as seeds (Appendix D.2), TinyGSM generation process is cited from Liu et al. (2023)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (FineWeb-Edu, Dolma) and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw data to results is documented: pretraining mix → checkpoint selection → MBF/WRAP/synthetic data generation → annealing with 10% upsampling → evaluation on benchmarks. Appendices D and E provide additional detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: ServiceNow Research, Mila — Quebec AI Institute, and Reka AI. Two authors are noted as having done work during internship at ServiceNow."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ServiceNow Research is the primary affiliation and likely funded the compute. As the employer of most authors, ServiceNow has a potential interest in the framework's success. No explicit funding independence statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state a training data cutoff date. They describe training on FineWeb-Edu and Dolma but do not specify the temporal coverage of these datasets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU, GSM8k, or Hendrycks MATH test examples appear in FineWeb-Edu or Dolma training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2020), GSM8k (2021), and Hendrycks MATH are well-established benchmarks likely present in web-crawled training data. The paper does not address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed FLOPs cost analysis for each data sourcing method (Appendix E), including back-of-envelope estimates ($500K-$1M for 100B synthetic tokens). Figures 5 plot performance vs. compute cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Base model training: 32,500 H100-hours on 64 H100 GPUs (Appendix C.1). Annealing runs: 2.1B to 75B tokens. Curation costs estimated in FLOPs (Appendix E)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only 2 seeds for the full replay baseline, and a single seed for all other methods (Section 3.3). No seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly states: 'We run two seeds and average the results for the full replay baseline, and only use a single seed for other baselines.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The 10% upsampling ratio was 'selected based on a hyperparameter search conducted on MBF data in the medical domain' but the number of configurations tried and compute spent is not reported. MBF threshold sweep (2 to 5 in 0.5 increments) is mentioned but total search cost is not."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "MBF threshold of 2.5 was selected from a parameter sweep (2 to 5 in 0.5 increments, Appendix D.1) based on downstream performance. The 10% upsampling ratio was selected via search on medical domain MBF data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate all data sourcing methods themselves. No discussion of potential bias from self-evaluation or comparison with independent implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the core contribution. Figures 5 and 6 plot performance (Brier Score Δ, Exact Match Δ) as a function of compute budget (FLOPs) for each data sourcing method."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Fig. 3 explicitly discuss construct validity: CF format is more robust than MC format for measuring knowledge (MC is sensitive to formatting artifacts). The paper advocates Brier Score over accuracy citing Schaeffer et al.'s analysis of emergent abilities as metric artifacts."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper evaluates data sources through direct model training and standard benchmark evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2020), GSM8k (2021), and Hendrycks MATH were published before the model's training data was likely collected. No discussion of temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (particularly WRAP's MMLU-style Q/A augmentation) leaks information about the test format or content."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data (FineWeb-Edu, Dolma) and benchmark test data share structural similarities or content overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No decontamination, canary strings, or membership inference tests."
       }
     }
   },
   "claims": [
     {
       "claim": "Data source rankings are not invariant across compute scales — WRAP outperforms MBF at small compute but this reverses at larger scales in the medical domain.",
       "evidence": "Figure 1 and Figure 5 (top) show WRAP outperforming MBF at low token counts but MBF overtaking at higher compute budgets on MMLU Medical CF tasks.",
       "supported": "strong"
     },
     {
       "claim": "Scaling curves can be estimated per data source to predict utility at larger scales, enabling cost-effective resource allocation.",
       "evidence": "Figure 5 shows fitted scaling laws with R² values: MBF R²=0.8, WRAP R²=0.9 (medical curation-only). The extrapolated curves correctly predict relative rankings at the highest tested scale.",
       "supported": "moderate"
     },
     {
       "claim": "WRAP's poor scaling is due to low diversity compared to MBF.",
       "evidence": "Figure 4 shows MBF has higher distinct n-gram scores across all n-gram levels; Figure 15 shows higher entropy for MBF. The authors frame this as a hypothesis (Section 4.1).",
       "supported": "weak"
     },
     {
       "claim": "Instruction augmentation improves MC-format evaluation but not CF-format, suggesting it enhances formatting rather than underlying knowledge.",
       "evidence": "Figure 5 shows instruction augmentation does not outperform full replay on CF tasks. Figure 13b shows it matches WRAP on MC tasks. Figure 3 demonstrates format sensitivity.",
       "supported": "strong"
     },
     {
       "claim": "CF evaluation format is more robust and format-invariant than MC or generative formats.",
       "evidence": "Figure 3 shows CF performance remains consistent across WRAP variants (with/without MMLU Q/A, Wiki Q/A), while MC and generative formats vary dramatically with formatting changes.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Data source rankings for domain-specific model annealing are not invariant across compute scales, making single-point estimates (micro-annealing) potentially misleading. The paper demonstrates this with a 7B parameter model across medical and math domains, showing that WRAP synthetic data outperforms model-based filtering at small compute but the relationship reverses at scale, likely due to diversity limitations. The CF evaluation format is found to be more robust than MC format for assessing domain knowledge, as MC is highly sensitive to formatting artifacts. Cost-aware scaling curves can be constructed per data source to inform resource allocation decisions.",
   "red_flags": [
     {
       "flag": "Single-seed experiments",
       "detail": "All non-baseline experiments use a single random seed. The authors acknowledge stochasticity at small scales can distort scaling law coefficients, yet they do not mitigate this with multiple seeds. This undermines the reliability of the fitted scaling curves, particularly for methods like WRAP that 'show greater variability at low scales.'"
     },
     {
       "flag": "No contamination analysis",
       "detail": "The paper evaluates on MMLU, GSM8k, and Hendrycks MATH — all well-established benchmarks likely present in web-crawled training data (FineWeb-Edu, Dolma). No decontamination or overlap analysis is performed. Benchmark contamination could differentially affect data sourcing methods (e.g., MBF may preferentially select documents containing benchmark-adjacent content)."
     },
     {
       "flag": "Potential MMLU leakage through WRAP",
       "detail": "WRAP+MMLU-style Q/A explicitly generates training examples in MMLU format. This could leak test format or content into training, inflating MC evaluation scores. While the paper acknowledges this for MC, the potential contamination of CF evaluation is not discussed."
     },
     {
       "flag": "Company evaluating own framework",
       "detail": "ServiceNow Research employees evaluate a framework that enhances their training pipeline. The Fast-LLM training engine (also from ServiceNow) is used throughout. No independent evaluation or conflict acknowledgment."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work that this paper extends to data source utility estimation."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Introduces micro-annealing for data evaluation, the baseline approach this paper extends with scaling laws."
     },
     {
       "title": "Rephrasing the web: A recipe for compute and data-efficient language modeling",
       "authors": ["Pratyush Maini", "Skyler Seto"],
       "year": 2024,
       "arxiv_id": "2401.16380",
       "relevance": "Proposes WRAP method for synthetic data augmentation, one of the main data sourcing methods evaluated."
     },
     {
       "title": "2 OLMo 2 Furious",
       "authors": ["Team OLMo", "Pete Walsh", "Luca Soldaini"],
       "year": 2024,
       "arxiv_id": "2501.00656",
       "relevance": "Uses micro-annealing for data quality validation and introduces TinyGSM-MIND variant evaluated in this paper."
     },
     {
       "title": "Scaling laws for data filtering — data curation cannot be compute agnostic",
       "authors": ["Sachin Goyal", "Pratyush Maini"],
       "year": 2024,
       "relevance": "Demonstrates that high-quality filtered data loses utility when repeated, directly motivating scaling-aware data source evaluation."
     },
     {
       "title": "RegMix: Data mixture as regression for language model pre-training",
       "authors": ["Qian Liu", "Xiaosen Zheng"],
       "year": 2024,
       "arxiv_id": "2407.01492",
       "relevance": "Alternative approach to data mixture optimization using regression on small proxy models, compared conceptually to this work."
     },
     {
       "title": "Data mixing laws: Optimizing data mixtures by predicting language modeling performance",
       "authors": ["Jiasheng Ye"],
       "year": 2024,
       "arxiv_id": "2403.16952",
       "relevance": "Proposes functional relationships for predicting performance across data mixtures, complementary approach to per-source scaling laws."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Motivates the use of Brier Score over accuracy by showing emergent abilities can be metric artifacts."
     },
     {
       "title": "TinyGSM: achieving >80% on GSM8k with small language models",
       "authors": ["Bingbin Liu", "Sebastien Bubeck"],
       "year": 2023,
       "arxiv_id": "2312.09241",
       "relevance": "Proposes synthetic math data generation method evaluated as a data source in this paper."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao", "Peiyi Wang"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Demonstrates importance of large-scale synthetic data for math reasoning, motivating the need for scaling-aware data source evaluation."
     }
   ]
 }