ai-research-survey

scan.json (23295B)

---

 {
   "paper": {
     "title": "Relative-Based Scaling Law for Neural Language Models",
     "authors": ["Baoqing Yue", "Jinyuan Zhou", "Zixi Wei", "Jingtao Zhan", "Qingyao Ai", "Yiqun Liu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.20387",
     "doi": "10.48550/arXiv.2510.20387"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper proposes Relative-Based Probability (RBP), a metric measuring whether the correct token ranks in the model's top-k predictions, and establishes a power-law scaling relationship between -log(RBP_k) and model size for small k. Experiments across 4 model families (GPT-2, OPT, Pythia, Qwen2.5) and 4+ datasets show R² ≥ 0.97 for k ≤ 100. The law breaks down when k approaches vocabulary size. The authors use the law to explain emergence phenomena as a macroscopic consequence of smooth token-level scaling and conjecture a lognormal rank distribution unifying cross-entropy and RBP scaling laws.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states 'Code is available at https://github.com/ybq22/relative-based-scaling-law.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Wikipedia, HumanEval, HotpotQA, Open Australian Legal Corpus, C4, and Pile. All are referenced with public URLs or citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (R² values, slopes) with no confidence intervals or error bars on the fitted parameters."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of power-law fit rely solely on R² values without testing whether the fit is significantly better than alternatives."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Scaling exponents (slopes) are reported numerically across all conditions in Table 1, providing magnitude context for the scaling relationships (e.g., slope = -0.087 for cross-entropy, -0.079 for RBP1 on Wiki)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 24 models across 4 families is sufficient to establish the scaling law. The paper states 'at least 5×10⁵ tokens' per data point but does not justify this threshold."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported across runs or token samples. Single-point estimates throughout."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Cross-entropy scaling law serves as the baseline comparison throughout. The paper directly compares R² and slopes between CE-based and RBP-based scaling laws (Figure 8a, Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Qwen2.5 (2024) is included as a contemporary model family, and the paper references Chinchilla (Hoffmann et al., 2022) and other recent scaling law work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies k across three regimes (k=1, moderate k, large k) to show which conditions support the scaling law, functioning as an ablation of the threshold parameter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses both cross-entropy loss and RBP_k across multiple k values as evaluation metrics, plus R² for fit quality."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to establishing mathematical scaling laws for token prediction metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on separate datasets (Wikipedia, HumanEval, HotpotQA, AusLegal) that are not used for model training parameter selection. The models are pre-trained and used as-is."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset, per-model-family, and per-k-value breakdowns of R² and slope values, totaling hundreds of individual results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly discusses where the scaling law breaks down (large k regime), with Figure 5 showing the deterioration. GPT-2 anomalous behavior is noted."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The large-k regime (Section 4.3) is an explicitly reported negative result where the power-law relationship fails. Low R² for GPT-2 on Github (0.091 for CE) is also reported in Table 1."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about power-law scaling of RBP_k, robustness across datasets/model families, and applications to emergence explanation are all supported by the experimental sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'scaling up models fundamentally reshapes the ranking of tokens' (Section 4.1) — a causal claim from observational data. Models differ in architecture, training data, and training procedure, not just size. No controlled experiment isolates model size as the causal factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Neural Language Models' broadly but experiments are limited to autoregressive decoder-only transformers. No encoder-only, encoder-decoder, or non-transformer architectures are tested. The abstract says 'scaling large language models' without bounding to the tested architectures."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the observed power-law relationships. Could the relationship be an artifact of how these particular model families were trained? No consideration of confounds like training data composition varying with model size."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes RBP as measuring top-k ranking probability vs. cross-entropy measuring absolute probability, and explicitly discusses when each is relevant (e.g., RBP for greedy decoding, Section 1 and 3.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model series and sizes are named: Pythia 14M–12B (Biderman et al., 2023), GPT-2 (Radford et al., 2019), OPT (Zhang et al., 2022), Qwen2.5 0.5B–14B (Yang et al., 2024). These are open-weight models with specific published versions."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It evaluates next-token prediction on corpora, not prompted generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the evaluation pipeline (e.g., context length, batch size, tokenization settings). The paper states 'at least 5×10⁵ tokens' but provides no further details."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates raw model predictions on corpora."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the evaluation corpora were preprocessed. How were Wikipedia, HumanEval, HotpotQA, and AusLegal tokenized and split? What portions were used? Not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper acknowledges the large-k breakdown in Section 4.3 but does not have a structured limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. The paper does not address whether the observed patterns could be artifacts of the specific model families or datasets chosen."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not clarify that results are limited to autoregressive decoder-only transformers, or that the lognormal conjecture is unproven."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw per-token prediction data is not released. Only aggregate RBP values and fitted curves are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section A describes model selection (4 families spanning 5 orders of magnitude) and dataset selection (4 benchmarks covering different domains). The RBP computation procedure is clearly defined in Section 3.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and pre-trained models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from corpus to RBP computation is conceptually described but lacks implementation details: how tokens were sampled, what context lengths were used, how the 5×10⁵ token minimum was enforced."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Tsinghua University. No products being evaluated are affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the model families used. The paper evaluates pre-trained models on benchmarks like HumanEval without discussing temporal overlap."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation corpora (Wikipedia, HumanEval, etc.) overlap with the training data of GPT-2, OPT, Pythia, or Qwen models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021; Qwen2.5 was trained well after. Wikipedia is likely in all models' training data. No contamination discussion whatsoever."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or compute time reported despite running 24 models across 6 datasets with millions of tokens each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The paper uses pre-trained models with deterministic evaluation (next-token prediction), but does not discuss whether different corpus subsamples yield different results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "No statement of how many runs produced each data point. The paper states 'at least 5×10⁵ tokens' but not whether this was computed once or averaged over multiple samples."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search is involved. The method computes RBP directly from model outputs with k as a parameter, not a tuned hyperparameter."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is involved. The paper reports results across all k values rather than selecting a best one."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a new metric and law, not a system competing against baselines. There are no re-implementations of competing methods."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper studies how model performance scales with model size, not compute budget. Compute is not a variable in the experiments."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation corpora (Wikipedia, HumanEval, HotpotQA, AusLegal) are appropriate for establishing general scaling laws. Wikipedia may be memorized; HumanEval is a small benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated on raw next-token prediction."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. HumanEval (2021), HotpotQA (2018), and Wikipedia are all available before training of later models like Qwen2.5."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation corpora are in the models' training data, which would constitute direct feature leakage for a next-token prediction task."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of overlap between training and evaluation data. Wikipedia is almost certainly in the training data of all models tested."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods used despite high contamination risk."
       }
     }
   },
   "claims": [
     {
       "claim": "-log(RBP_k) follows a power-law relationship with model size S for small k (k << vocabulary size)",
       "evidence": "Figures 3-4 and Table 1 show R² ≥ 0.97 across 4 datasets and 4 model families for k ≤ 100. 24 models spanning 5 orders of magnitude.",
       "supported": "strong"
     },
     {
       "claim": "The power-law relationship breaks down when k approaches vocabulary size",
       "evidence": "Figure 5 shows scattered results for k = 20,000 and 30,000. Section 4.3 discusses the deterioration with GPT-2 showing non-monotonic behavior.",
       "supported": "strong"
     },
     {
       "claim": "Cross-entropy and -log(RBP_1) have nearly identical scaling exponents",
       "evidence": "Figure 8a shows slopes of -0.087 (CE) vs -0.079 (RBP_1) on Wiki data. Table 1 shows similar slopes across datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Emergence phenomena can be explained as a macroscopic consequence of smooth token-level scaling under top-k sampling",
       "evidence": "Section 5.1 derives the sigmoidal emergence curve from the power-law (Equation 8). Figure 7 shows empirical confirmation across multiple k values.",
       "supported": "moderate"
     },
     {
       "claim": "A lognormal rank distribution can unify cross-entropy and RBP scaling laws",
       "evidence": "Section 5.2 and Appendix C present the theoretical derivation. Figure 8c shows lognormal fits to empirical rank distributions for Pythia models.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The paper evaluates models on Wikipedia (almost certainly in training data) and HumanEval (published 2021, before Qwen2.5 training) without any contamination discussion. If models have memorized evaluation corpora, the RBP measurements may reflect memorization rather than genuine scaling of predictive ability."
     },
     {
       "flag": "Confounded model comparisons",
       "detail": "Models within each family differ not only in size but potentially in training data composition, training duration, and hyperparameters. The paper attributes scaling behavior to model size without controlling for these confounds."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single point estimates with no error bars, confidence intervals, or variance across subsamples. For a paper establishing a 'law', the precision of the fitted parameters is unknown."
     },
     {
       "flag": "Overclaiming scope",
       "detail": "Title says 'Neural Language Models' but only autoregressive decoder-only transformers are tested. No encoder models, encoder-decoders, SSMs, or other architectures."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B Brown"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling law paper establishing cross-entropy power-law relationships."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws for compute-optimal training of LLMs."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Defined emergence phenomena in LLMs; this paper proposes an alternative explanation."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Argues emergence is a measurement artifact; this paper extends the argument with relative-based metrics."
     },
     {
       "title": "Pythia: A suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman", "Hailey Schoelkopf"],
       "year": 2023,
       "relevance": "Model suite used extensively in this paper's experiments for scaling analysis."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "HumanEval benchmark used as one of four evaluation datasets."
     },
     {
       "title": "Scaling laws for autoregressive generative modeling",
       "authors": ["Tom Henighan", "Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2010.14701",
       "relevance": "Extended scaling laws beyond language to autoregressive generation broadly."
     },
     {
       "title": "Scaling laws for dense retrieval",
       "authors": ["Yan Fang", "Jingtao Zhan", "Qingyao Ai"],
       "year": 2024,
       "relevance": "Applied scaling laws to information retrieval; by overlapping authors with this paper."
     }
   ]
 }