ai-research-survey

scan.json (21947B)

---

 {
   "paper": {
     "title": "MoSE: Mixture of Slimmable Experts for Efficient and Adaptive Language Models",
     "authors": ["Nurbek Tastan", "Stefanos Laskaridis", "Karthik Nandakumar", "Samuel Horváth"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.06154"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MoSE extends Mixture-of-Experts architectures by making each expert slimmable, enabling a continuous accuracy-compute trade-off at inference time. Across GPT2-Small, Standard, and Medium models trained on OpenWebText, MoSE with test-time training consistently shifts the Pareto frontier, achieving 20-36% FLOP savings at comparable perplexity to standard MoE. The learned sharpness parameter γ transfers across datasets (OpenWebText to LAMBADA) without re-calibration.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used (OpenWebText, WikiText-103, LAMBADA, WSC) are publicly available standard benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'NVIDIA A100-SXM4-40GB GPU machine' and '4 GPUs' with DDP but provides no software environment details (library versions, requirements.txt, etc.)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and Figures 4-10 are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MoSE 'matches or improves upon' MoE but provides no statistical significance tests for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "FLOP savings are reported as percentages with baseline context (e.g., '20.3% FLOPs', '35.9% FLOPs') and absolute perplexity values with baselines are given in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of model sizes, training token budgets, or number of benchmark evaluation examples."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard MoE is used as the baseline throughout, with direct comparisons in Table 1 and all Pareto frontier figures."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is standard MoE. Recent elastic MoE works (RoE, EMoE) are discussed in Related Work but not compared experimentally."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 10 ablates inference modes (uniform-width vs. normalized-probability vs. TTT shared vs. TTT layer-wise). Figures 4-8 systematically vary model size, routing configuration, and token budget."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Perplexity on OpenWebText and WikiText-103, accuracy and perplexity on LAMBADA, accuracy on WSC, and MFLOPs per token are all reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant for this architecture/efficiency paper evaluating language modeling perplexity and zero-shot benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are on held-out test splits: 'a held-out split of OpenWebText', standard evaluation splits of WikiText-103, LAMBADA (5153 entries), and WSC (273 entries)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model scale (Small/Standard/Medium), routing configuration (E8A2/E8A4/E16A4), training budget (3B/15B tokens), and inference mode."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or scenarios where MoSE underperforms MoE are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows MoSE matching or outperforming MoE. No negative results or failed approaches are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MoSE 'matches or improves upon standard MoE at full width' and 'achieving comparable performance with significantly fewer FLOPs', both supported by Table 1 and Figures 4-10."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'width identification improves performance' are supported by controlled ablations (Figure 10) comparing inference modes with all other variables held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Efficient and Adaptive Language Models' broadly, but experiments are limited to GPT2-scale models (55M-1B parameters) trained on OpenWebText. Section 5 acknowledges 'limited to small-scale LLMs' but the title and abstract do not bound this."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why MoSE outperforms MoE. For example, whether the multi-width training acts as a regularizer is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures perplexity and zero-shot accuracy and presents them as such, without overclaiming these as measures of broader capabilities. FLOPs per token is used as the compute proxy, which is well-matched to the efficiency claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "All models are trained from scratch with exact architectures specified in Table 2 (layers, heads, hidden dimensions, parameter counts). No pre-trained API models are used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. Models are evaluated on standard language modeling (perplexity) and zero-shot benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 provides detailed hyperparameters: AdamW lr=6e-4, weight decay=0.1, β=(0.9,0.95), gradient clipping=1.0, warmup=200 iterations, load balancing loss=0.01, router z-loss=0.001, wmin=0.25, wmax=1.0, step size=0.05, TTT lr=0.01 with SGD."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this architecture paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing details are provided. The paper simply states models are 'pre-trained on the OpenWebText corpus' without describing tokenization, filtering, or data preparation steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Discussion & Limitations' discusses limitations including scale constraints and open questions about post-training width adaptation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 raises a specific threat: 'Our analysis is limited to small-scale LLMs, which we are able to train from scratch' and notes that post-training slimmability 'remains an open question.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states the scope limitation: experiments are on small-scale LLMs trained from scratch, and whether multi-width operation can be instilled at post-training is left for future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model checkpoints, training logs, per-example predictions) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Training data (OpenWebText) and evaluation datasets (WikiText-103, LAMBADA, WSC) are standard public benchmarks with citations and dataset sizes provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from raw corpus to training batches (tokenization, sequence packing, etc.)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: MBZUAI, Amazon Science (with note 'Work done independently of Amazon'), and Michigan State University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Models are trained from scratch on OpenWebText with specified token budgets (3B-15B tokens), so the training data is fully known and controlled."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether OpenWebText training data overlaps with WikiText-103, LAMBADA, or WSC evaluation sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LAMBADA and WSC are public benchmarks that could appear in OpenWebText (web-sourced). No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is reported throughout as MFLOPs per token, with detailed breakdowns in Figure 12 and Pareto frontiers showing compute-quality trade-offs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 states 'NVIDIA A100-SXM4-40GB GPU machine' with '4 GPUs' using DDP. Training token budgets (3B-15B) and iteration counts are in Table 2."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. All experiments appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is disclosed despite many hyperparameters being set (learning rate, loss coefficients, width range, TTT calibration details)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of configurations (e.g., wmin=0.25, step size=0.05, TTT calibration of 50 batches) is not justified or explained as selected from alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors implement both MoSE and the MoE baseline. No acknowledgment of potential bias from implementing their own baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is organized around compute-vs-performance trade-offs, with Pareto frontiers (Figures 4-10) explicitly showing performance as a function of MFLOPs per token."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether perplexity and zero-shot accuracy on small benchmarks (WSC has only 273 examples) actually measure the capabilities claimed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly on language modeling and zero-shot tasks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Models are trained on OpenWebText which is web-sourced. LAMBADA (2016) and WSC (2012) predate OpenWebText's collection. No discussion of whether solutions appeared in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenWebText validation split overlap with external benchmarks is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "MoSE at full width (w=1.0) matches or improves upon standard MoE across all model scales and training budgets.",
       "evidence": "Table 1 shows MoSE (w=1.0) matching or beating MoE on OpenWebText, WikiText-103, LAMBADA, and WSC across GPT2-Small and GPT2-Standard at 3B and 15B tokens.",
       "supported": "moderate"
     },
     {
       "claim": "MoSE with test-time training achieves comparable performance to MoE with 20-36% fewer FLOPs.",
       "evidence": "Figures 4-5 report FLOP savings: 20.3% (GPT2-Small 3B), 35.9% (GPT2-Standard 3B), 30.6% (GPT2-Medium 7.5B), 29.1% (GPT2-Small 15B), 24.9% (GPT2-Standard 15B).",
       "supported": "moderate"
     },
     {
       "claim": "The learned sharpness parameter γ transfers across datasets without re-calibration.",
       "evidence": "Figure 9 shows γ calibrated on OpenWebText transfers to LAMBADA, with TTT variants outperforming uniform-width on both accuracy and perplexity.",
       "supported": "moderate"
     },
     {
       "claim": "MoSE training does not destabilize MoE pre-training.",
       "evidence": "Figure 3 shows MoSE closely tracking MoE training loss curves across iterations for both GPT2-Small and GPT2-Standard.",
       "supported": "strong"
     },
     {
       "claim": "MoSE degrades gracefully when inference-time routing differs from training-time routing.",
       "evidence": "Figure 8 shows a single E16A4-trained checkpoint evaluated at E16A2, E16A3, and E16A4, with smooth Pareto frontier degradation.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or error bars",
       "detail": "All results appear to be single-run without any uncertainty quantification. For small benchmarks like WSC (273 examples), point estimate differences could easily be within noise."
     },
     {
       "flag": "No comparison with contemporary elastic MoE methods",
       "detail": "RoE and EMoE are discussed in Related Work as direct competitors but are not compared experimentally, despite being concurrent/recent work addressing the same problem."
     },
     {
       "flag": "Small-scale experiments only",
       "detail": "All models are 55M-1B parameters, far below modern LLM scale. Whether MoSE works at scale is unknown, acknowledged by the authors but not reflected in the broad claims of the title/abstract."
     },
     {
       "flag": "Every experiment shows improvement",
       "detail": "MoSE matches or outperforms MoE in every single reported comparison. No negative results or failure modes are shown, which is suspicious for a method applied across many configurations."
     }
   ],
   "cited_papers": [
     {
       "title": "Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity",
       "authors": ["W. Fedus", "B. Zoph", "N. Shazeer"],
       "year": 2022,
       "relevance": "Foundational MoE architecture for scaling language models with sparse expert selection."
     },
     {
       "title": "Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer",
       "authors": ["N. Shazeer", "A. Mirhoseini", "K. Maziarz"],
       "year": 2017,
       "relevance": "Introduced sparsely-gated MoE layers enabling conditional computation in large models."
     },
     {
       "title": "Slimmable Neural Networks",
       "authors": ["J. Yu", "L. Yang", "N. Xu", "J. Yang", "T. Huang"],
       "year": 2019,
       "relevance": "Pioneered slimmable networks enabling runtime width adjustment, core technique extended by MoSE."
     },
     {
       "title": "Matryoshka representation learning",
       "authors": ["A. Kusupati", "G. Bhatt", "A. Rege"],
       "year": 2022,
       "relevance": "Nested/ordered representation learning relevant to MoSE's slimmable expert structure."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2024,
       "relevance": "Referenced for agentic settings where MoSE could adaptively select model width based on task difficulty."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "relevance": "Cited regarding accuracy metric sensitivity and non-monotonic behavior in discrete evaluations."
     },
     {
       "title": "Fast Inference from Transformers via Speculative Decoding",
       "authors": ["Y. Leviathan", "M. Kalman", "Y. Matias"],
       "year": 2023,
       "relevance": "MoSE could enable self-speculation using reduced-width execution as a draft model."
     },
     {
       "title": "Elastic MoE: Unlocking the inference-time scalability of mixture-of-experts",
       "authors": ["N. Gu", "Z. Zhang", "Y. Feng"],
       "year": 2025,
       "relevance": "Concurrent elastic MoE work addressing the same rigidity of top-k routing at inference time."
     }
   ]
 }