ai-research-survey

scan.json (25370B)

---

 {
   "paper": {
     "title": "CMoE: Converting Mixture-of-Experts from Dense to Accelerate LLM Inference",
     "authors": [
       "Zehua Pei",
       "Lancheng Zou",
       "Hui-Ling Zhen",
       "Xianzhi Yu",
       "Wulong Liu",
       "Sinno Jialin Pan",
       "Mingxuan Yuan",
       "Bei Yu"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2502.04416"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the abstract: 'We make our code publicly available at https://github.com/JarvisPei/CMoE.' A GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: WikiText-2, C4, SlimPajama, and standard benchmarks (BoolQ, PIQA, SciQ, WinoGrande, ARC-Challenge, HellaSwag). No custom datasets were created."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Hugging Face Transformers and PyTorch but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides a GitHub link, the paper itself does not contain step-by-step reproduction instructions, a README summary, or scripts to replicate the main experiments. The methodology is described algorithmically but not as reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-5 are reported as point estimates (single perplexity values, single accuracy numbers) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CMoE outperforms LLaMA-MoE and LLaMA-MoE-v2 based solely on comparing raw numbers in Tables 1 and 2, with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context throughout: e.g., '1.5x latency reduction at 25% activation', PPL values alongside the dense model baseline (5.27 vs 7.02 for CMoE 75%), and percentage of dense model accuracy recovered ('>76%'). Table 3 reports speedup ratios. Absolute baseline and result numbers are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The calibration dataset uses only 8 examples (2048 sequence length), and fine-tuning uses 2,048 samples. These choices are not justified by any formal analysis. Appendix B shows performance with different calibration sizes (8, 32, 64) but does not justify why 8 was selected as the default."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or multiple-run results are reported anywhere in the paper. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against the original dense models (Llama-2 7B, Llama-3 8B), LLaMA-MoE, and LLaMA-MoE-v2 as baselines in Tables 1 and 2. Table 4 also compares against Pythia-1.0B and TinyLlama-1.1B."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLaMA-MoE (EMNLP 2024) and LLaMA-MoE-v2 (arXiv 2024) are recent and represent state-of-the-art approaches for dense-to-MoE conversion. The comparison is appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 provides ablation studies on: (1) fine-tuning data size (Figure 2), (2) load balancing (Figure 3), (3) expert configuration at fixed activation ratio (Table 5), (4) calibration data size (Appendix B), and (5) scaling to larger models (Appendix C)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses perplexity on WikiText-2 and C4, downstream task accuracy on 6 benchmarks (BoolQ, PIQA, SciQ, WinoGrande, ARC-Challenge, HellaSwag), and inference speedup measurements."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this work. The paper evaluates a model compression/conversion technique using automated metrics (perplexity, benchmark accuracy, inference speedup), which is standard and appropriate."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Perplexity is measured on WikiText-2 and C4 validation sets (Table 1 explicitly states 'validation sets'). Fine-tuning is done on WikiText-2 training data. Downstream benchmarks are standard held-out evaluations."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-benchmark breakdowns across 6 tasks rather than just aggregate performance. Table 5 provides per-configuration breakdowns. Multiple activation ratios (25%, 75%) are separately reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that training-free CMoE at 25% activation has poor absolute perplexity (62.30 on WikiText-2) and underperforms pre-trained dense-small models (Table 4). The limitations section in Appendix D discusses cases where the approach may not work well."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 explicitly shows training-free CMoE-25% underperforms Pythia-1.0B and TinyLlama-1.1B. The S1A3E16 configuration with more complex routing shows worse performance than simpler configurations (Table 5). Diminishing returns from fine-tuning data are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'lossless precision in terms of perplexity' at 75% activation is supported by Table 1 (CMoE TF 7.02 vs Dense 5.27 on WikiText-2 for Llama-2 7B — this is close but not exactly lossless, though the claim is hedged with 'in terms of perplexity'). '1.5x latency reduction' is supported by Table 3. '>76% of dense model's downstream accuracy' is supported by Table 2 data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (removing/modifying components). The balanced assignment algorithm, router construction, and load balancing are each evaluated through controlled experiments. The ablation design uses single-variable manipulation (e.g., varying fine-tuning data size, expert configurations)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Accelerate LLM Inference' broadly, but experiments are limited to Llama-2 7B, Llama-2 13B, and Llama-3 8B. The conclusion states 'CMoE offers a practical approach for deploying large language models' without bounding to the tested models. No experiments on non-Llama architectures or significantly larger models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the improvements over LLaMA-MoE/LLaMA-MoE-v2 could stem from implementation differences rather than algorithmic superiority, or whether the calibration dataset (WikiText-2) biases the perplexity results on WikiText-2."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model names: 'Llama-2 7B', 'Llama-3 8B', 'Llama-2 13B'. These are specific model checkpoints from Meta with known architectures and weights, not ambiguous model names."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It is about model architecture conversion (dense to MoE), not about querying language models with prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: LoRA rank 8, alpha 32, optimizer Adam (beta1=0.9, beta2=0.95), learning rates (0.001 for router, 5.95e-5 for LoRA), load balancing bias update speed gamma=0.001, Ka=10 for activation status, calibration uses 8 examples of 2048 sequence length."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a model architecture conversion method, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes: calibration uses 8 randomly selected examples from WikiText-2 with 2048 sequence length. Fine-tuning uses 2,048 WikiText-2 samples for 1 epoch. Extended fine-tuning uses subsets of SlimPajama up to 1.2B tokens. The selection and usage of data at each stage is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D contains a 'Limitations' paragraph that discusses dependence on calibration dataset choice, potential limitations of the training-free router, and the need for future exploration across more architectures and tasks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section mentions specific threats: (1) 'performance of neuron activation profiling and subsequent expert grouping is influenced by the choice of the calibration dataset', (2) the training-free router 'may not fully capture all nuances of routing decisions' compared to end-to-end trained routers, and (3) need for validation 'across a wider array of model architectures'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. The limitations mention future work directions but do not explicitly bound the scope (e.g., 'our results do not demonstrate X'). The broad claims in the conclusion ('CMoE offers a practical approach for deploying large language models') are not bounded to the tested settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-example perplexity values, individual benchmark outputs, activation statistics) is made available. Only aggregated results in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: randomly selected samples from WikiText-2 for calibration and fine-tuning, standard benchmark datasets for evaluation, and SlimPajama for extended fine-tuning. The datasets are all public and well-known."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (WikiText-2, C4, SlimPajama, BoolQ, PIQA, SciQ, WinoGrande, ARC-Challenge, HellaSwag)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) calibration data collected from WikiText-2, (2) neuron activation profiling using ATopK metric, (3) expert grouping via balanced assignment, (4) router construction from activation statistics, (5) optional LoRA fine-tuning. Section 3 and Section 4 describe each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. Authors are from CUHK and Huawei Noah's Ark Lab, but no explicit funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: 'The Chinese University of Hong Kong' and 'Noah's Ark Lab, Huawei'. The Huawei affiliation is transparently disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present, so independence cannot be assessed. Authors from Huawei (a company that deploys LLMs) have a potential commercial interest in efficient LLM inference techniques, but no funding or conflict statement is provided to clarify the relationship."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that Huawei is a major technology company, the absence of any declaration is a gap."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates Llama-2 and Llama-3 on standard benchmarks but does not state the training data cutoff dates for these models. This is relevant because perplexity on WikiText-2 and downstream benchmark accuracy could be affected by data contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WikiText-2, C4, or the downstream benchmarks appeared in Llama-2/Llama-3 training data. This is a standard concern for these models and benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (BoolQ, PIQA, WinoGrande, ARC-Challenge, HellaSwag, SciQ) were all published before the training cutoff of Llama-2 and Llama-3. No contamination analysis is performed. However, since the paper compares different conversion methods applied to the same base model, contamination would affect all methods equally, somewhat mitigating the concern for relative comparisons."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports inference speedups for different configurations (e.g., 1.5x-1.6x full model speedup for 25% activation). Figure 2 shows construction time. The paper also states 'conversion completes in under five minutes on a single GPU'."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the conversion completes 'in under five minutes on a single GPU'. Fine-tuning takes '1 hour' for the 2k sample LoRA fine-tuning. Figure 2 shows construction time scaling with data size. While GPU type is not specified, the time budget is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "At 75% activation ratio, CMoE achieves lossless precision in terms of perplexity while maintaining 5% acceleration.",
       "evidence": "Table 1 shows CMoE 75% training-free achieves 7.02 PPL on WikiText-2 vs 5.27 for dense Llama-2 7B (33% higher, not truly lossless). Table 3 shows S3A3E8 (75% activation) achieves 1.05-1.12x full model speedup.",
       "supported": "moderate"
     },
     {
       "claim": "At 25% activation, CMoE reduces end-to-end latency by 1.5x while preserving usable perplexity without additional training.",
       "evidence": "Table 3 shows S1A1E8 (25% activation) achieves 1.5-1.6x full model speedup. Table 1 shows training-free PPL of 62.30 on WikiText-2 (vs 5.27 for dense), which is degraded but functional.",
       "supported": "moderate"
     },
     {
       "claim": "A brief LoRA fine-tuning process (1 hour, 2,000 samples) recovers over 76% of the dense model's downstream accuracy.",
       "evidence": "Table 2 shows CMoE 25% FT on Llama-2 7B achieves 55.04 BoolQ (vs 82.04 dense = 67%), 77.50 SciQ (vs 90.80 = 85%), 57.12 PIQA (vs 78.78 = 72%). The >76% claim appears to cherry-pick SciQ. Average across all 6 tasks: ~50.68/76.21 = ~66.5% recovery.",
       "supported": "weak"
     },
     {
       "claim": "CMoE substantially outperforms LLaMA-MoE and LLaMA-MoE-v2 in training-free and lightweight fine-tuning settings.",
       "evidence": "Table 1 shows LLaMA-MoE and LLaMA-MoE-v2 produce NaN or >10k PPL in training-free mode at 25% activation, while CMoE achieves 62.30. After 2k fine-tuning, CMoE PPL is 12.73 vs 468.00 (LLaMA-MoE) and 17.72 (LLaMA-MoE-v2). Table 2 downstream results confirm the advantage.",
       "supported": "strong"
     },
     {
       "claim": "With extended fine-tuning (1.2B tokens SlimPajama), CMoE-25% surpasses Pythia-1.0B and approaches TinyLlama-1.1B.",
       "evidence": "Table 4 shows CMoE-25% with 1.2B SlimPajama achieves 61.64 PIQA (vs 69.21 Pythia, 73.29 TinyLlama), 59.51 BoolQ (vs 57.83 both), 44.02 HellaSwag (vs 47.16 Pythia, 59.20 TinyLlama). CMoE surpasses Pythia on BoolQ and Winogrande but not on PIQA and HellaSwag.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CMoE converts dense LLMs (Llama-2 7B, Llama-3 8B) into Mixture-of-Experts architectures without retraining, using neuron activation profiling and balanced assignment clustering. At 75% activation ratio, the conversion achieves near-dense perplexity with modest speedup (5-12%). At 25% activation, it achieves 1.5x speedup but with significant quality degradation that lightweight LoRA fine-tuning partially recovers. CMoE substantially outperforms prior dense-to-MoE conversion methods (LLaMA-MoE, LLaMA-MoE-v2) which fail catastrophically in training-free settings.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are reported as single-run point estimates with no standard deviations, confidence intervals, or multiple-seed experiments. Given the sensitivity of MoE routing to initialization and calibration data selection, this is a significant omission."
     },
     {
       "flag": "Overstated lossless claim",
       "detail": "The abstract and conclusion claim 'lossless precision in terms of perplexity' at 75% activation, but Table 1 shows WikiText-2 PPL of 7.02 vs 5.27 for dense Llama-2 7B (33% higher) and 10.16 vs 6.14 for Llama-3 8B (65% higher). This is a substantial gap presented as negligible."
     },
     {
       "flag": "Cherry-picked accuracy recovery claim",
       "detail": "The '>76% of dense model downstream accuracy' claim appears to be based on the best-performing benchmark (SciQ). Averaging across all 6 benchmarks for Llama-2 7B at 25% activation gives approximately 66.5% recovery, not >76%."
     },
     {
       "flag": "Potential calibration data leakage",
       "detail": "WikiText-2 is used for both calibration/fine-tuning and perplexity evaluation (on the validation set). While the paper uses the validation set for evaluation and training set for calibration/fine-tuning, this domain overlap could inflate perplexity results compared to out-of-domain evaluation."
     },
     {
       "flag": "Missing GPU/hardware specification for speedup measurements",
       "detail": "Table 3 reports inference speedups but does not specify the GPU model used for benchmarking. Speedup ratios can vary significantly across hardware, making these results difficult to verify or compare."
     }
   ],
   "cited_papers": [
     {
       "title": "Llama-MoE: Building Mixture-of-Experts from LLaMA with Continual Pre-training",
       "authors": ["Tong Zhu", "Xiaoye Qu", "Daize Dong", "Jiacheng Ruan", "Jingqi Tong", "Conghui He", "Yu Cheng"],
       "year": 2024,
       "relevance": "Primary baseline for dense-to-MoE conversion using continual pre-training, demonstrating the standard approach CMoE improves upon."
     },
     {
       "title": "LLaMA-MoE v2: Exploring Sparsity of LLaMA from Perspective of Mixture-of-Experts with Post-Training",
       "authors": ["Xiaoye Qu", "Daize Dong", "Xuyang Hu", "Tong Zhu", "Weigao Sun", "Yu Cheng"],
       "year": 2024,
       "arxiv_id": "2411.15708",
       "relevance": "Second primary baseline using importance-based neuron partition and post-training for dense-to-MoE conversion."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["Aixin Liu", "Bei Feng", "Bing Xue"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "State-of-the-art MoE architecture whose load balancing mechanism CMoE adopts for its differentiable routing."
     },
     {
       "title": "DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models",
       "authors": ["Damai Dai", "Chengqi Deng", "Chenggang Zhao"],
       "year": 2024,
       "arxiv_id": "2401.06066",
       "relevance": "Influential MoE architecture design with shared and routed experts, which CMoE's architecture is modeled after."
     },
     {
       "title": "MoEfication: Transformer Feed-Forward Layers are Mixtures of Experts",
       "authors": ["Zhengyan Zhang", "Yankai Lin", "Zhiyuan Liu", "Peng Li", "Maosong Sun", "Jie Zhou"],
       "year": 2021,
       "arxiv_id": "2110.01786",
       "relevance": "Foundational work demonstrating that FFN layers can be decomposed into experts based on activation sparsity patterns."
     },
     {
       "title": "Deja Vu: Contextual Sparsity for Efficient LLMs at Inference Time",
       "authors": ["Zichang Liu", "Jue Wang", "Tri Dao"],
       "year": 2023,
       "relevance": "Prior work on exploiting contextual sparsity in FFN layers for efficient LLM inference, directly related to CMoE's activation sparsity observations."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method used by CMoE for optional lightweight adaptation after conversion."
     },
     {
       "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity",
       "authors": ["William Fedus", "Barret Zoph", "Noam Shazeer"],
       "year": 2022,
       "relevance": "Foundational MoE architecture work establishing routing and expert selection paradigms that CMoE builds upon."
     },
     {
       "title": "Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints",
       "authors": ["Aran Komatsuzaki", "Joan Puigcerver", "James Lee-Thorp"],
       "year": 2022,
       "arxiv_id": "2212.05055",
       "relevance": "Alternative approach to dense-to-MoE conversion via capacity expansion, representing the second paradigm CMoE contrasts against."
     },
     {
       "title": "Learn to be Efficient: Build Structured Sparsity in Large Language Models",
       "authors": ["Haizhong Zheng", "Xiaoyan Bai", "Xueshen Liu"],
       "year": 2024,
       "arxiv_id": "2402.06126",
       "relevance": "Training-time approach to learning activation sparsity in LLMs, complementary to CMoE's post-hoc conversion approach."
     }
   ]
 }