ai-research-survey

scan.json (18174B)

---

 {
   "paper": {
     "title": "CuMo: Scaling Multimodal LLM with Co-Upcycled Mixture-of-Experts",
     "authors": ["Jiachen Li", "Xinyao Wang", "Sijie Zhu", "Chia-Wen Kuo", "Lu Xu", "Fan Chen", "Jitesh Jain", "Humphrey Shi", "Longyin Wen"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2405.05949"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/SHI-Labs/CuMo. Abstract and introduction both state code and model weights are open-sourced."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all training data are publicly accessible open-sourced datasets (LLaVA-558K, ALLaVA, ShareGPT4V, etc.). Evaluation benchmarks are all public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found in the paper. Appendix B mentions hyperparameters but not software dependencies or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The GitHub repo is referenced but the paper does not include a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1-7 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CuMo 'outperforms' baselines based solely on comparing numbers in tables without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context across multiple benchmarks (e.g., CuMo Mistral-7B scores 73.9 on SQA vs LLaVA-NeXT Mistral-7B at 72.8), allowing readers to assess magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes or discussion of whether sample sizes are adequate for the claimed differences."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 notes some numbers are 'averaged by three inference runs of querying GPT API' (marked with †) but no standard deviation or variance is reported. Most results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines in Table 1 including InstructBLIP, Qwen-VL, LLaVA-v1.5, LLaVA-NeXT, Mini-Gemini, MM1, SPHINX, and others."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include LLaVA-NeXT (2024), Mini-Gemini (2024), MM1 (2024), and other recent models contemporary to the submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies in Tables 3-7 covering MLP-MoE configurations, CLIP-MoE, LLM-MoE, multi-resolution features, and pre-finetuning stage."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated on 12 benchmarks: SQA-IMG, TextVQA, GQA, POPE, MME, MMBench, MMVet, VQAv2, LLaVA-Wild, SEED-IMG, MMMU, MathVista."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a model architecture paper evaluated on established automated benchmarks. Human evaluation is not standard or necessary for the claims made."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks (VQAv2, GQA, MME, etc.) have standard test/val splits. The paper follows LLaVA evaluation settings on these established benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across 12 individual benchmarks and model size groups (7B, 13B, 7B MoE) in Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 qualitative analysis shows hallucination examples (e.g., '2 characters standing on the table') and acknowledges 'the need for further investigation to mitigate hallucinations.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows training MoE from scratch leads to 'clear performance drop.' Table 5 shows upcycled Mistral MoE is significantly worse than pre-trained Mixtral. Top 2-in-8 showed slight decline vs Top 2-in-4."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of outperforming state-of-the-art within model size groups are supported by Table 1 results, though 'outperforms' is sometimes marginal (< 1 point on some benchmarks)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about MoE components are supported by controlled ablation studies (Tables 3-7) that systematically add/remove components while keeping other factors fixed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to 'within each model size group' and specific benchmarks tested. The paper does not overclaim generality beyond the evaluated settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance gains. For example, the additional training data in the final model (1.65M vs 665K for baselines) could partly explain improvements, but this confound is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models specified: CLIP ViT-L, Mistral-7B, Mixtral 8x7B. GPT evaluation uses gpt-4-0613 and gpt-3.5-turbo with specific version identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains and fine-tunes models rather than prompting LLMs. Evaluation follows LLaVA settings which convert data into visual instructions, but this is standard benchmark protocol, not prompt engineering."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Learning rates specified (1e-3, 2e-5, 2e-6), auxiliary loss coefficients (αb=0.1, αz=0.01), greedy decoding, and Appendix B referenced for additional hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a model architecture and training paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents training datasets for each stage, with total data sizes (558K pre-training, ~1.65M instruction tuning). Appendix A provides detailed dataset breakdown."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion mentions hallucinations but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do not show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All training datasets are publicly available open-source datasets. Evaluation benchmarks are public with standard evaluation protocols."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the training datasets used at each stage with references to their original papers."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage training pipeline is documented (pre-training → pre-finetuning → visual instruction tuning) with datasets specified for each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section listing grants. The acknowledgments only thank individuals for discussions."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: SHI Labs @ Georgia Tech & UIUC, and ByteDance Inc."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ByteDance (a major tech company) employs most authors and has commercial interest in multimodal AI. No discussion of whether funding is independent of outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Mistral-7B and Mixtral 8x7B but does not state their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation benchmark data could have appeared in the LLM or CLIP pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like VQAv2 (2017), GQA (2019) predate the models' training. No discussion of contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'neglectable additional activated parameters during inference' but does not report actual inference latency, throughput, or cost measurements."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of GPU hours, hardware used for training, or total computational budget despite training multiple large models across three stages."
       }
     }
   },
   "claims": [
     {
       "claim": "CuMo Mistral-7B outperforms state-of-the-art 7B multimodal LLMs across multiple benchmarks.",
       "evidence": "Table 1 shows CuMo Mistral-7B achieves top or near-top scores among 7B models on SQA (73.9), TextVQA (67.0), MMBench-EN (73.0), MMVet (51.0), MMMU (39.1).",
       "supported": "moderate"
     },
     {
       "claim": "Co-upcycling is essential for stable MoE training in the vision encoder and MLP connector.",
       "evidence": "Table 3 shows training MoE from scratch drops performance (SQA 68.1 vs baseline 72.8), while upcycling recovers it (73.7). Table 4 confirms CLIP-MoE from scratch fails to converge.",
       "supported": "strong"
     },
     {
       "claim": "Pre-trained MoE LLMs (Mixtral) significantly outperform upcycled LLMs.",
       "evidence": "Table 5 shows Mixtral 8x7B outperforms upcycled Mistral 8x7B across all benchmarks (e.g., TextVQA 60.6 vs 56.4, MMVet 40.0 vs 35.7).",
       "supported": "strong"
     },
     {
       "claim": "CuMo enhances multimodal LLMs with negligible additional activated parameters during inference.",
       "evidence": "Table 1 shows CuMo Mistral-7B has 7.8B activated parameters vs 7.6B for LLaVA-NeXT Mistral-7B, but no actual inference cost measurements are provided.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CuMo introduces co-upcycled sparse Mixture-of-Experts blocks into the vision encoder and MLP connector of multimodal LLMs, achieving competitive or superior performance to larger models while adding minimal activated parameters. The co-upcycling initialization strategy is critical for training stability, as MoE blocks trained from scratch fail to converge. Pre-trained MoE LLMs (Mixtral) significantly outperform upcycled dense LLMs, so CuMo applies MoE only to the vision side. Ablation studies across six configurations validate each design choice.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Results are reported as single-run point estimates (except three GPT-evaluated benchmarks averaged over 3 runs without std dev). Performance differences between models are often within 1-2 points with no way to assess statistical significance."
     },
     {
       "flag": "Training data confound in main results",
       "detail": "The final CuMo models in Table 1 use ~1.65M training examples while some baselines (LLaVA-v1.5) use only 665K. Table 2 partially addresses this with a limited-data comparison, but the headline results conflate architecture improvements with additional training data."
     },
     {
       "flag": "Company evaluating its own product",
       "detail": "Most authors are from ByteDance, which has commercial interests in multimodal AI. No conflict of interest statement is provided."
     },
     {
       "flag": "No compute budget reported",
       "detail": "Training multiple large models across three stages likely required substantial compute, but no GPU hours or hardware details are provided, making cost-benefit assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "LLaVA: Visual Instruction Tuning",
       "authors": ["Haotian Liu", "Chunyuan Li", "Qingyang Wu", "Yong Jae Lee"],
       "year": 2023,
       "relevance": "Foundational multimodal LLM approach that CuMo builds upon; demonstrates visual instruction tuning methodology."
     },
     {
       "title": "Improved Baselines with Visual Instruction Tuning (LLaVA-v1.5)",
       "authors": ["Haotian Liu", "Chunyuan Li", "Yuheng Li", "Yong Jae Lee"],
       "year": 2023,
       "relevance": "Primary baseline architecture for CuMo; establishes MLP connector and training recipe."
     },
     {
       "title": "LLaVA-NeXT: Improved reasoning, OCR, and world knowledge",
       "authors": ["Haotian Liu", "Chunyuan Li", "Yuheng Li", "Bo Li", "Yuanhan Zhang", "Sheng Shen", "Yong Jae Lee"],
       "year": 2024,
       "relevance": "Key contemporary baseline for multimodal LLM benchmarking."
     },
     {
       "title": "MoE-LLaVA: Mixture of Experts for Large Vision-Language Models",
       "authors": ["Bin Lin"],
       "year": 2024,
       "relevance": "Related work applying MoE to multimodal LLMs in the LLM component rather than vision side."
     },
     {
       "title": "MM1: Methods, Analysis & Insights from Multimodal LLM Pre-training",
       "authors": ["Brandon McKinzie"],
       "year": 2024,
       "relevance": "Strong private baseline that CuMo compares against; summarizes essential steps for building multimodal LLMs."
     },
     {
       "title": "Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints",
       "authors": ["Aran Komatsuzaki"],
       "year": 2022,
       "relevance": "Core technique that CuMo adapts for vision encoder and MLP connector MoE initialization."
     },
     {
       "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity",
       "authors": ["William Fedus", "Barret Zoph", "Noam Shazeer"],
       "year": 2022,
       "relevance": "Foundational MoE scaling work relevant to understanding efficient LLM scaling approaches."
     }
   ]
 }