ai-research-survey

scan.json (24076B)

---

 {
   "paper": {
     "title": "Neural Neural Scaling Laws",
     "authors": ["Michael Y. Hu", "Jane Pan", "Ayush Rajesh Jhaveri", "Nicholas Lourie", "Kyunghyun Cho"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.19831",
     "doi": "10.48550/arXiv.2601.19831"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "NEUNEU, a neural network trained on open-source model checkpoints, achieves 2.04% MAE in predicting downstream task accuracy across 66 tasks — a 38% reduction over logistic scaling laws (3.29% MAE). The model generalizes zero-shot to unseen model families, parameter counts, and downstream tasks. Token-level validation loss distributions carry predictive signal that averaging into a single loss value obscures. NEUNEU also achieves 75.6% ranking accuracy for predicting which model configuration will perform better, a 12.3% improvement over logistic baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/michahu/neuneu (stated at end of §1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it is trained entirely on open-source model checkpoints from HuggingFace (DataDecide model suite, Magnusson et al., 2025), which are publicly available. Validation data uses WebOrganizer dataset (Wettig et al., 2025)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The GitHub repo is linked but the paper does not describe reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ±2σ over 5 random seeds for neural models (§3) and 95% bootstrap confidence intervals for ranking accuracy (Figure 7)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims NEUNEU outperforms baselines but does not report statistical significance tests (p-values, t-tests, etc.). Comparisons are based on MAE differences without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context: '38% reduction compared to logistic scaling laws (3.29% MAE)' achieving 2.04% MAE, and '12.3% improvement' in ranking accuracy (0.756 vs 0.633)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 6 model sizes from DataDecide were used, or why 66 downstream tasks were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ±2σ over 5 random seeds for neural models (§3). The logistic baseline has only numerical imprecision as randomness."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: LOGISTIC (logistic scaling laws), LC-PFN (meta-learning method from Adriaensen et al., 2023), DIFFPROBE, NOLOSS, AVERAGE, and HISTDIFF ablations (§2.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LC-PFN (2023) and logistic scaling laws from Magnusson et al. (2025) and Gadre et al. (2025) are recent and represent the standard approach."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are provided: NOLOSS (no validation loss), AVERAGE (average loss only), HISTDIFF (histogram deltas), and DIFFPROBE (MLP on histograms). These isolate the contribution of token-level losses, temporal context, and architecture (§3, Table 3b)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports MAE for prediction accuracy, ranking accuracy for model selection (Figure 7), calibration of uncertainty estimates (Figure 6C), and per-task breakdowns (Table 3b)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for a scaling law prediction model — the outputs are numerical accuracy predictions that can be compared directly against ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Clear held-out evaluation: new random seeds, held-out pretraining datasets (C4), unseen model family (Pythia), and 13 withheld downstream tasks (§2.4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3b provides per-task MAE breakdowns for 10 OLMES tasks. Figure 3a shows performance across different generalization conditions (seeds, data, Pythia, unseen tasks)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases or tasks where NEUNEU performs poorly. Figures 10-12 show all task visualizations but there is no error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 shows that using token losses directly (instead of probabilities) produces worse results — a negative finding that informed the design. The AVERAGE ablation shows that average loss is worse than token-level, which is a constructive negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (2.04% MAE, 38% reduction, zero-shot generalization) are all supported by results in §3, Figure 3a, and Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablations ('token-level losses carry predictive signal that averaging obscures'). The ablation design (NEUNEU vs AVERAGE vs NOLOSS) is controlled single-variable manipulation, which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§5.2 Limitations explicitly bounds generalization: same validation set requirement, classification tasks only (not generative), specific model families tested. The title 'Neural Neural Scaling Laws' is descriptive of the method rather than overclaiming."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why NEUNEU outperforms baselines beyond the hypothesis that distributional information helps. No discussion of confounds like NEUNEU having more parameters or being trained on more data than LC-PFN."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match its measurements: MAE on accuracy predictions, ranking accuracy. It does not overframe these as broader claims about 'understanding scaling' — it specifically discusses prediction accuracy."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model sizes are listed: {90M, 150M, 300M, 530M, 750M, 1B} from DataDecide, and {70M, 1.4B, 2.8B, 6.9B} from Pythia. NEUNEU itself is ~20M parameters (§2.4)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting — NEUNEU is a neural network trained from scratch, and the evaluated language models are evaluated on downstream tasks using standard evaluation suites."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides comprehensive hyperparameters: hidden dimension 512, 6 transformer layers, 8 attention heads, batch size 256, learning rate 6×10⁻⁴, weight decay 0.033, 3 epochs, warmup ratio 0.1, CNN architecture details, etc."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. NEUNEU is a standard neural network with a forward pass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§2.3 details the data pipeline: how training samples are constructed from checkpoint accuracies, random dropping with p=0.4, gap absorption, token probability computation from 256,000 contiguous tokens, whitespace tokenization with subword combination."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§5.2 'Limitations and Future Work' discusses several specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§5.2 discusses specific threats: same validation set requirement, classification-only tasks (generative tasks may differ), and the need to interpret what the CNN learns. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§5.2 explicitly states scope boundaries: 'the downstream tasks in our evaluation suite are classification tasks where accuracy is the natural metric, and generative tasks may exhibit different scaling dynamics' and 'NEUNEU is designed to use the same validation set.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All training data is from open-source HuggingFace checkpoints (DataDecide model suite). The paper emphasizes 'anyone can train our model from open-source data' (§2.3)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§2.3 describes exactly how training data is constructed: which model checkpoints, which seeds, how samples are created from trajectories, what validation data is used (256,000 tokens from WebOrganizer)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is standard public model checkpoints."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "§2.3 documents the full pipeline: checkpoint accuracies → gap imputation → random dropping → subsequence construction → target generation. Equations 14-16 formalize each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSF Graduate Research Fellowship, NYU HPC, IITP/MSIT grant, Samsung Advanced Institute of Technology, and NSF Award 1922658."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with New York University (stated in header). The paper evaluates open-source models, not products from the authors' institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSF, IITP/MSIT, Samsung, NYU) have no direct stake in whether NEUNEU outperforms logistic scaling laws."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The language models being evaluated (DataDecide, Pythia) have known training data, but the paper does not state training cutoff dates for these models relative to the OLMES benchmark tasks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the downstream evaluation tasks (OLMES suite) appeared in the pretraining data of the DataDecide or Pythia models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The OLMES tasks include well-known benchmarks (ARC, MMLU, HellaSwag) that could have appeared in pretraining data. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "§2.4 states 'at inference time, NEUNEU is not significantly more expensive to run than logistic scaling laws, taking only a few seconds of wall clock time on CPU for one downstream task.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total training compute budget is stated for training NEUNEU itself. NYU HPC is acknowledged but GPU hours or training time are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "§3 states 'we also report ±2σ over 5 random seeds as an error bar' for neural models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§3 explicitly states 5 random seeds for neural models."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how hyperparameters in Table 1 were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the final hyperparameter configuration was selected from potential alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 7 methods across 66 tasks without any multiple comparison correction. No formal statistical tests are applied at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all methods including baselines (LOGISTIC, LC-PFN) without acknowledging potential bias from implementing competitors' methods."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "NEUNEU has ~20M parameters and is trained for 3 epochs while logistic scaling laws are simple curve fits. The compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether MAE on accuracy prediction is the right metric for evaluating scaling law quality, or whether the OLMES task suite is representative."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — all methods are evaluated on the same prediction task with the same inputs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the OLMES benchmark tasks (ARC, MMLU, etc.) were in the pretraining data of the models whose trajectories NEUNEU learns from."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether NEUNEU's training on certain model families leaks information about the evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The training data (DataDecide models) and test data (Pythia, held-out seeds/datasets) share the same downstream evaluation tasks. This structural similarity is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "NEUNEU achieves 2.04% MAE on 66 downstream tasks, a 38% reduction compared to logistic scaling laws (3.29% MAE).",
       "evidence": "Figure 3a and Table 3b show MAE comparisons across all evaluation conditions and per-task breakdowns.",
       "supported": "strong"
     },
     {
       "claim": "NEUNEU generalizes zero-shot to unseen model families, parameter counts, and downstream tasks.",
       "evidence": "Figure 3a shows Pythia generalization; Figure 5 shows performance on 13 unseen tasks is lower MAE than logistic scaling laws achieve on seen tasks.",
       "supported": "strong"
     },
     {
       "claim": "Token-level validation losses carry predictive signal that averaging obscures.",
       "evidence": "Ablation comparing NEUNEU vs AVERAGE vs NOLOSS in Figure 3a and Table 3b, plus HISTDIFF showing distributional features help.",
       "supported": "moderate"
     },
     {
       "claim": "NEUNEU's uncertainty estimates are well-calibrated, with 74.9% of ground truth landing in the 10%-90% interquantile range (expected 80%).",
       "evidence": "Figure 6C shows calibration analysis.",
       "supported": "moderate"
     },
     {
       "claim": "NEUNEU achieves 75.6% ranking accuracy for predicting which model configuration achieves better final performance, a 12.3% improvement over logistic baselines (63.3%).",
       "evidence": "Figure 7 with 95% bootstrap confidence intervals.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis for underlying benchmarks",
       "detail": "The OLMES tasks (ARC, MMLU, HellaSwag) are well-known benchmarks that may be contaminated in the pretraining data of the models being predicted. If the models' benchmark scores are inflated by contamination, NEUNEU is learning to predict contaminated scores."
     },
     {
       "flag": "Advantageous baseline setup for LOGISTIC",
       "detail": "The paper gives LOGISTIC ground truth future validation loss (ℓ_{t+K}) to be maximally fair, but this also means LOGISTIC is solving a fundamentally different (easier in one sense) problem. The comparison is somewhat apples-to-oranges."
     },
     {
       "flag": "No hyperparameter search budget reported",
       "detail": "The 20M parameter NEUNEU model has many design choices (CNN architecture, transformer size, drop probability, etc.) but no discussion of how these were selected or how many configurations were tried."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper that NEUNEU builds upon and aims to improve."
     },
     {
       "title": "An empirical analysis of compute-optimal large language model training",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "Chinchilla scaling laws paper that established the dominant power-law functional form NEUNEU replaces."
     },
     {
       "title": "Inverse scaling: When bigger isn't better",
       "authors": ["Ian R. McKenzie"],
       "year": 2023,
       "relevance": "Documents inverse scaling phenomenon that motivates the need for more expressive scaling law predictors."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Documents emergent capabilities in LLMs that are hard to predict with simple scaling laws."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges the emergence narrative and shows metric choice affects apparent emergence — directly relevant to scaling law prediction."
     },
     {
       "title": "Scaling laws are unreliable for downstream tasks: A reality check",
       "authors": ["Nicholas Lourie", "Michael Y. Hu", "Kyunghyun Cho"],
       "year": 2025,
       "relevance": "Companion paper by same group showing parametric scaling laws are unreliable for downstream tasks."
     },
     {
       "title": "DataDecide: How to predict best pretraining data with small experiments",
       "authors": ["Ian Magnusson"],
       "year": 2025,
       "relevance": "Provides the training data (model checkpoints) used to train NEUNEU and the logistic scaling law baseline."
     },
     {
       "title": "Pythia: a suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman"],
       "year": 2023,
       "relevance": "Provides the held-out model family used to test NEUNEU's zero-shot generalization."
     },
     {
       "title": "Broken neural scaling laws",
       "authors": ["Ethan Caballero", "Kshitij Gupta", "Irina Rish", "David Krueger"],
       "year": 2023,
       "relevance": "Proposes expressive parametric forms for scaling laws — a direct baseline approach that NEUNEU aims to improve upon."
     },
     {
       "title": "Efficient bayesian learning curve extrapolation using prior-data fitted networks",
       "authors": ["Steven Adriaensen"],
       "year": 2023,
       "relevance": "LC-PFN method used as a baseline — transformer-based meta-learning for learning curve prediction."
     }
   ]
 }