ai-research-survey

scan.json (29476B)

---

 {
   "paper": {
     "title": "Towards Resource-Efficient Multimodal Intelligence: Learned Routing among Specialized Expert Models",
     "authors": [
       "Mayank Saini",
       "Arit Kumar Bishwas"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.06441",
     "doi": "10.48550/arXiv.2511.06441"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a modular, cost-aware routing framework that dispatches queries across specialized models based on modality, complexity, and cost. On MMLU and VQA-v2, the system claims to match or exceed GPT-4 performance (88.5% vs 84.2% on MMLU, 93.2% vs 89.7% on VQA-v2) while reducing premium model usage by ~72%. Routing accuracy is reported at 92.3% across a 13-category classification task. However, key details are missing (model versions, hyperparameter values, dataset sizes) and several numerical claims are internally inconsistent.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses standard public benchmarks: MMLU, GSM8K, MBPP, XSum, CNN/DailyMail, VQA-v2, and FUNSD. However, the custom 13-way routing evaluation dataset constructed via stratified sampling is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or environment setup section is provided. Hardware specifications for inference are not mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports ±1.8% for MMLU and ±2.3% for VQA-v2. Table 2 reports ±1.1% for routing accuracy. However, it is unclear whether these represent standard deviations across runs or variance across data subsets."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'outperforming the Always-Premium baseline' are made by comparing point estimates without any statistical test (e.g., t-test, bootstrap)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: 88.5% vs 84.2% on MMLU, 93.2% vs 89.7% on VQA-v2, cost reductions to 68-70% of baseline, and 18% latency improvement. Both absolute values and relative improvements are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes is provided. The paper does not state how many examples from each benchmark were used, nor why those quantities are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'reported metrics are mean values; where applicable, we also provide variance to indicate stability across subsets' (Section 4.2). The ± values represent variance across subsets of evaluation data, not variance across multiple experimental runs. No multi-run variance or standard deviation across seeds is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against an 'Always-Premium' baseline where every query is handled by GPT-4 (Section 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is 'always use GPT-4.' The paper cites RouteLLM, HybridLLM, and FrugalGPT as related routing systems but does not compare against any of them experimentally. These are the most relevant contemporary baselines for a routing paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has many components (modality classifier, complexity analyzer, intent classifier, Couplet, MoE aggregator, context agent) but no ablation study is conducted to measure the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: task accuracy (MMLU, VQA-v2), routing accuracy, precision/recall, F1, TF-IDF cosine similarity, BERT cosine similarity, latency, throughput, and relative cost."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. Human annotators were used for dataset labeling but not for evaluating the quality of the system's generated responses."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'τ is dynamically calibrated on a held-out validation set' (Section 3.2.1), but does not explicitly state that the reported evaluation results are on a separate test set distinct from the validation set used for threshold tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks results down by benchmark (MMLU, VQA-v2, fine-grained routing). Table 4 breaks down by routing path (open-source vs premium). Table 5 breaks down latency by modality (text, vision). Figure 5 breaks down cost by model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses the main failure mode: 'The only source of routing error or ambiguity arises in the case of text queries that serve as follow-ups to previous non-text tasks... most failures arising from subtle or ambiguous textual references to prior modalities.' Follow-up detection accuracy is reported at 90%."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every reported experiment shows improvement over the baseline. No ablations that hurt performance, approaches tried and abandoned, or configurations that failed are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of matching/exceeding Always-Premium performance are supported by Table 3 (88.5% vs 84.2% MMLU, 93.2% vs 89.7% VQA-v2). The 'over 67%' reduction in costly model reliance is supported by Table 4 (72% handled by open-source). Whether the underlying numbers are trustworthy is a separate concern."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'intelligent routing can push the efficiency frontier, obtaining better results with fewer resources by leveraging specialized models' (Section 4.2). No controlled experiment isolates the causal contribution of routing vs. simply using different models. The comparison is uncontrolled (multi-model routing system vs. single-model baseline)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Resource-Efficient Multimodal Intelligence' and the abstract claims 'high-quality, resource-efficient AI at scale.' But the evaluation covers only MMLU, VQA-v2, and a custom routing dataset. The paper does not bound its claims to these specific benchmarks and settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the surprising MMLU result (routing system beating GPT-4) is not examined — it could be due to model selection effects, benchmark contamination differences, or evaluation methodology artifacts."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on specific benchmarks and relative cost, then frames this as 'resource-efficient multimodal intelligence' and 'enterprise-grade requirements for accuracy, efficiency, and cost control' (Section 5). The gap between benchmark accuracy and real-world deployment effectiveness is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'GPT-4', 'Claude', 'Gemini', 'LLaMA', 'Mistral-7B', 'Qwen-VL', 'Mixtral 8×7B'. No specific version identifiers (e.g., 'gpt-4-0613') or snapshot dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompts or system instructions used in experiments are provided. The intent classification and routing logic are described in natural language, but the prompts sent to LLMs for response generation are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper introduces many tunable parameters (α, β, γ for complexity scoring; τ for threshold; δm, δu, δt, λc for routing weights; θs, θt, θm for context scoring) but provides no actual values for any of them. They are described as 'tunable' or 'empirically derived' without reporting the values used in experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The routing architecture is described extensively: modality classification (Section 3.2.1), complexity scoring equations, intent classification pipeline (Section 3.2.3), multi-agent LangGraph workflow (Section 3.2.4), MoE aggregation (Eq. 4-7), memory management (Section 3.2.5), and feedback loop (Section 3.2.6). Figures 1-4 illustrate the workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the system's input preprocessing (MIME checks, modality classification), the evaluation data preparation is inadequately documented. The custom 13-way dataset was constructed via 'stratified sampling from established benchmarks... augmented with controlled generation' but the number of examples, per-category counts, and 'controlled generation' process are not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion (Section 5) includes a limitations paragraph discussing the fixed 13-category classification scheme, orchestration complexity, and dependence on model availability. It is substantive though embedded in the conclusion rather than a standalone section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are somewhat specific to this study: 'The current framework relies on a fixed 13-category classification scheme for task routing, which may not capture the full spectrum of emerging query types' and 'performance is also dependent on the quality and availability of specialized models for each domain.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The limitations paragraph discusses potential issues but does not define scope boundaries (e.g., 'our results do not apply to X' or 'we did not test Y'). The broad title and conclusion claims are unbounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, routing decisions, per-example results) is made available for verification. Only aggregate numbers are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the evaluation dataset construction: 'balanced 13-way dataset via stratified sampling from established benchmarks... augmented with controlled generation for audio, video, and image-generation prompts.' Annotation process described: 'Labels were produced by two independent annotators under written guidelines, with disagreements resolved by consensus and GPT-4 arbitration; we measured inter-annotator agreement (Cohen's κ).'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the system pipeline but not the data pipeline from raw benchmark examples to reported numbers. How many examples were drawn from each benchmark, any filtering applied, and how aggregate metrics were computed are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present. Both authors are from PwC US, a professional services firm, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list 'PwC US' as their affiliation with institutional email addresses (mayank.s.saini@pwc.com, arit.kumar.bishwas@pwc.com)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. PwC, as a consulting firm offering AI services, could benefit commercially from demonstrating cost-efficient AI deployment frameworks. The potential conflict is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. PwC employees proposing a framework for 'enterprise-scale' AI deployment have potential commercial interests that go undisclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-4, LLaMA, Mistral, Qwen, etc.), despite evaluating on public benchmarks like MMLU and GSM8K."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MMLU (2020), GSM8K (2021), MBPP (2021), and VQA-v2 (2015) are all well-known benchmarks that post-2023 models may have trained on."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All benchmarks used (MMLU, VQA-v2, GSM8K, MBPP, XSum, CNN/DailyMail) were published years before the models' training cutoffs. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative inference costs are reported in Table 3 (68-70% of baseline), Table 4 (query and cost share breakdown), and Figure 5 (cost breakdown by model). Latency is reported in Table 5 (419ms text, 530ms vision). However, no absolute costs (dollar amounts, tokens consumed) are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no GPU hours, total API spend, hardware specifications, or training/evaluation compute costs are mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. The paper mentions 'mean values' with 'variance to indicate stability across subsets' but does not state how many runs produced the results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Despite many tunable parameters (α, β, γ, τ, δm, δu, δt, λc, θs, θt, θm), no hyperparameter search budget is reported. The paper states τ is selected by 'sweeping candidate values' but does not report the range or number of configurations tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions selecting τ by 'sweeping candidate values and selecting the operating point that maximizes routing accuracy under a fixed cost budget' (Section 3.2.1) but does not report what configurations were tried or how the best was selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed, implemented, and evaluated their own routing system. No discussion of self-comparison bias, no independent evaluation, and no mitigation strategies are mentioned."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 plots accuracy vs. relative cost for both MMLU and VQA-v2, showing the cost-accuracy trade-off between the proposed approach and the Always-Premium baseline."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU, VQA-v2, or other benchmarks actually measure the capabilities the paper claims to evaluate. Benchmarks are used without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The routing scaffold IS the thing being evaluated — the paper proposes a routing system and evaluates it as a bundled product. The comparison is routing-system-as-a-whole vs. always-GPT-4, not model-vs-model through different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. All benchmarks (MMLU 2020, VQA-v2 2015, GSM8K 2021, MBPP 2021) predate the models used, meaning solutions could be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage or whether the evaluation setup provides information not available in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test data are independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used (no canary strings, membership inference, temporal splits, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "The routing system matches or exceeds Always-Premium (GPT-4) performance: 88.5% vs 84.2% on MMLU and 93.2% vs 89.7% on VQA-v2.",
       "evidence": "Table 3 (Section 4.2) reports these accuracy figures. The MMLU improvement of 4.3 percentage points over GPT-4 is surprising and unexplained — routing to smaller models should not improve knowledge benchmark accuracy.",
       "supported": "weak"
     },
     {
       "claim": "72% of queries are handled by open-source models, reducing premium model reliance by over 67%.",
       "evidence": "Table 4 shows 72% open-source query share vs 28% premium. This supports the 'over 67%' claim.",
       "supported": "moderate"
     },
     {
       "claim": "Inference cost reduced to roughly one-third of the Always-Premium approach.",
       "evidence": "Table 3 shows costs of 70% and 68% of baseline for MMLU and VQA-v2 respectively, which is roughly two-thirds, not one-third. The 42% figure for fine-grained routing is also not one-third. The 'one-third' claim is inconsistent with the reported numbers.",
       "supported": "weak"
     },
     {
       "claim": "Routing accuracy of 92.3% with fine-grained 13-way accuracy of 86.78%.",
       "evidence": "Table 2 reports these figures. However, the evaluation dataset is not released, the number of evaluation examples is not stated, and the dataset construction process lacks detail.",
       "supported": "moderate"
     },
     {
       "claim": "94% of responses have BERT similarity above 0.8 compared to Always-Premium outputs.",
       "evidence": "Stated in Section 4.2 text. Average BERT cosine similarity reported as 0.93. However, BERT similarity is an imperfect proxy for answer quality, and the threshold of 0.8 is arbitrary.",
       "supported": "moderate"
     },
     {
       "claim": "18% latency improvement and 20% throughput improvement over Always-Premium.",
       "evidence": "Table 5 reports text latency of 419ms vs 512ms (18.2%) and throughput of 54 vs 45 qps (20%). However, no variance or confidence intervals are provided for latency measurements.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Implausible MMLU result",
       "detail": "The paper claims 88.5% on MMLU, exceeding GPT-4's 84.2%. MMLU is a knowledge benchmark where routing to smaller models should not improve accuracy over GPT-4. This result is surprising and unexplained. It could stem from evaluation methodology artifacts, contamination differences between models, or unreported dataset subsetting."
     },
     {
       "flag": "Internal cost claim inconsistency",
       "detail": "The text claims 'inference cost was reduced to roughly one-third of the Always-Premium approach' but Table 3 shows costs of 68-70% of baseline (roughly two-thirds, not one-third). The 42% figure in the fine-grained eval is also not one-third. The claim overstates the measured savings."
     },
     {
       "flag": "No comparison to competing routing systems",
       "detail": "The paper cites RouteLLM, HybridLLM, and FrugalGPT as directly related routing systems but only compares against a naive 'always GPT-4' baseline. This makes the improvements appear larger than they might be against actual competing approaches."
     },
     {
       "flag": "Company evaluating its own product",
       "detail": "PwC employees propose a framework for 'enterprise-scale' AI deployment without disclosing potential commercial interests. No competing interests statement, no funding disclosure."
     },
     {
       "flag": "Many tunable parameters with no reported values",
       "detail": "The paper introduces 11+ tunable parameters (α, β, γ, τ, δm, δu, δt, λc, θs, θt, θm) but reports actual values for none of them, making reproduction impossible."
     },
     {
       "flag": "No contamination analysis on old benchmarks",
       "detail": "MMLU (2020), VQA-v2 (2015), GSM8K (2021), and MBPP (2021) are all well-known benchmarks likely in training data of models used. No contamination discussion despite this being critical to interpreting routing effectiveness."
     },
     {
       "flag": "Attachment detection precision/recall of 1.00",
       "detail": "Perfect attachment detection (precision and recall both 1.00) on the evaluation set is acknowledged as 'dataset-specific' but remains suspiciously clean. Either the evaluation set is too easy or too small for this task."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Directly relevant prior work on cost-efficient LLM routing via query-dependent cascades."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "LLM routing system that trains classifiers on human preference data to decide when to use large models."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding"],
       "year": 2024,
       "relevance": "Cost-efficient routing between small and large models for text queries."
     },
     {
       "title": "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face",
       "authors": ["Yongliang Shen"],
       "year": 2023,
       "relevance": "Multimodal orchestration framework using a controller LLM to invoke domain-specific expert models."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex evaluation paper, foundational for LLM code generation benchmarking."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Li"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent framework for decomposing tasks and coordinating LLM agents."
     },
     {
       "title": "Small Language Models are the Future of Agentic AI",
       "authors": ["Peter Belcak"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Argues compact SLMs can be competitive for agentic subtasks, relevant to routing cost-quality tradeoffs."
     },
     {
       "title": "Beyond Monoliths: Expert Orchestration for More Capable, Democratic, and Safe Large Language Models",
       "authors": ["Philip Quirke"],
       "year": 2025,
       "arxiv_id": "2506.00051",
       "relevance": "Addresses orchestration challenges for multimodal LLM deployment, directly related to routing architectures."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark paper, key benchmark used in this routing evaluation."
     },
     {
       "title": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone",
       "authors": ["Marah Abdin", "Sam Ade Jacobs", "Ammar Ahmad Awan"],
       "year": 2024,
       "arxiv_id": "2404.14219",
       "relevance": "Small language model used as an efficient expert in the routing framework."
     },
     {
       "title": "Mixtral of Experts",
       "authors": ["Albert Q Jiang"],
       "year": 2024,
       "arxiv_id": "2401.04088",
       "relevance": "Mixture-of-experts model used for follow-up queries in the framework, relevant to efficient LLM architecture."
     }
   ]
 }