ai-research-survey

scan.json (30924B)

---

 {
   "paper": {
     "title": "MMR-Bench: A Comprehensive Benchmark for Multimodal LLM Routing",
     "authors": ["Haoxuan Ma", "Guannan Lai", "Han-Jia Ye"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.17814",
     "doi": "10.48550/arXiv.2601.17814"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MMR-Bench provides an offline, cost-aware benchmark for evaluating routing strategies across 10 multimodal LLMs on 7 vision-language benchmarks (11,000 instances). The paper demonstrates that multimodal routing signals consistently outperform unimodal variants, that a routed system can match the strongest single model's accuracy at roughly 33% of its cost, and that matrix-factorization-based routers offer the most robust cross-dataset performance. Routing policies trained on multimodal data transfer zero-shot to text-only benchmarks (GSM8K, MMLU, ARC) without retraining.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'The code will be available at: https://github.com/Hunter-Wrynn/MMR-Bench' — this is a promise of future release, not a working URL at time of publication."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper's key data artifact — the offline outcome table with precomputed utilities and costs for each instance–model pair (over 100k pairs) — is not released. The underlying benchmarks (OCRBench, MMStar, etc.) are public, but the precomputed outcome table that enables offline routing evaluation is only promised as part of the future code release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency listing is provided. Section D mentions using CLIP encoders and standard ML libraries but does not specify versions or provide reproducible environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes methodology conceptually and provides implementation details in the appendix, but there are no concrete commands or scripts to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 and all other results tables report only point estimates (e.g., nAUC 0.7042, Ps 0.7533) with no confidence intervals, error bars, or ± notation. The paper notes 'results for stochastic routers are averaged over multiple runs' (Sec. 5.1) but provides no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'LinearMFRouter attains the highest nAUC') based solely on comparing numbers in tables. No statistical significance tests (p-values, t-tests, bootstrap, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports absolute performance values (nAUC, Ps) for all methods alongside baselines (Random, Best Single Model, Oracle), providing sufficient context to assess magnitudes. Table 3 reports explicit deltas (e.g., ΔnAUC = +0.3403 for KMeans). The '33% of cost' claim is contextualized against the strongest single model."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark comprises 11,000 instances across 7 datasets and 10 models, but no justification is given for why this size is sufficient. No power analysis or discussion of whether the sample allows for detecting meaningful routing differences."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'results for stochastic routers are averaged over multiple runs' but no standard deviations, interquartile ranges, or spread measures are reported in any table or figure. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes comprehensive baselines: Random (lower bound), Oracle (upper bound), Best Single Model, and four router families (Linear, MLP, KNN, KMeans) with variants, totaling 10 baseline methods (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The model zoo includes state-of-the-art models (GPT-5, Gemini 2.5, Claude 3.7 Sonnet, InternVL3, Qwen2.5-VL). Routing baselines reference contemporary work (RouterBench 2024, RouterEval 2025, EmbedLLM 2024). Table 1 compares against recent routing benchmarks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides systematic ablations: (1) adaptive vs equal-weight fusion (Table 3), (2) text-only vs image-only vs multimodal inputs (Figure 2), (3) within-scenario cross-dataset transfer (Table 4), and (4) cross-modality transfer (Table 5). Each isolates a specific factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three complementary metrics are defined and reported: nAUC (normalized area under cost-performance curve), Peak Score (Ps), and Quality-Neutral Cost (QNC). All three are formally defined in Section 4.3 and reported across experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates routing strategies on benchmarks with objective ground-truth answers (accuracy, scoring). Human evaluation is irrelevant to the claims about cost-accuracy trade-offs in automated routing."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'frozen 2:8 train/test splits. Routers are trained on the training split, test-time evaluation is purely offline by applying the learned router to benchmark features and indexing into the fixed {u_i,j, c_i,j}.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across all 7 datasets grouped into 3 scenarios (OCR, General VQA, Math Reasoning). Table S4 provides per-dataset QNC. Figure S1 shows per-dataset Pareto frontiers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 (Q1) discusses where text-only routing fails: 'text-in-image queries, dense OCR, charts, spatial reasoning, and low-resolution crops.' Section F provides qualitative case studies. Table 3 shows KNN regression with adaptive fusion (ΔnAUC = −0.0074). Table S4 shows QNC = ∞ for several methods."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports negative results: KNN shows mild regression with adaptive fusion (ΔnAUC = −0.0074). QNC = +∞ for KNN on multiple datasets (Table S4), meaning it cannot reach target accuracy. The paper also reports that text-only routing fails on vision-governed tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: (1) multimodal signals improve routing — supported by Figure 2, Table 2; (2) routed system matches strongest model at ~33% cost — supported by Figure 4 Pareto frontiers; (3) zero-shot generalization to new datasets and text-only benchmarks — supported by Tables 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('incorporating multimodal signals improves routing quality') are supported by controlled comparisons: Figure 2 compares text-only, image-only, and multimodal routers under identical model sets and cost accounting, isolating the modality signal as the only variable. Table 3 compares adaptive vs equal-weight fusion with all else held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 formally bounds scope to 'the two-modality case (text and image).' The conclusion states the benchmark covers specific routing scenarios. Specific claims reference the tested models and datasets. Though the title says 'Comprehensive,' the formal problem definition and experimental claims are appropriately scoped."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 provides explanations for why their approach works (modality-agnostic difficulty cues, fusion-induced regularization, cost-aware calibration) but does not consider alternative explanations that might account for the observed improvements, such as whether the gains come from better feature engineering rather than genuine multimodal signals, or whether dataset-specific artifacts drive results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy/utility and normalized cost — exactly what it claims to evaluate (cost-accuracy trade-offs in routing). The metrics (nAUC, Ps, QNC) are precisely defined in Section 4.3. No proxy gap exists between measurements and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section C.1 lists models: GPT-5-0807 and GPT-5-Nano-0807 include date suffixes, but 'Claude 3.7 Sonnet,' 'Gemini 2.5 Pro,' 'Gemini 2.5 Flash,' 'InternVL3-78B,' 'Qwen2.5-VL-3B/7B/72B,' and 'Gemma3-4B' lack specific version identifiers or snapshot dates. Marketing names without API versions are insufficient per the schema."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section B.2 describes prompt construction ('MCQ datasets include the question, candidate options, and an explicit selection instruction') but does not provide the actual prompt text. It says inputs are formatted using each model's 'official chat template' without showing the templates. The reader cannot reconstruct exact prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Router hyperparameters are partially described in Section D (fusion temperature η=5.0, ridge regression, SVD rank). However, critical MLLM inference hyperparameters (temperature, top-p, max tokens) are not reported. The paper only notes that 'thinking' traces were disabled for some models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The routing system is a direct feature-extraction → model-selection pipeline, not an agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 documents preprocessing: prompt construction formats for MCQ vs VQA datasets, image preprocessing (resize/interpolate/pad per model's official implementation), token logging for cost accounting, and use of VLMEvalKit framework."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The conclusion (Section 7) briefly summarizes contributions but does not discuss limitations substantively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of specific threats like benchmark selection bias, model version sensitivity, or the offline evaluation assumption."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3.1 scopes the problem to two modalities (text and image), the paper does not explicitly state what the results do NOT show. No discussion of excluded settings (video, audio, >2 modalities), populations (different deployment scenarios), or claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw outcome table (per-instance utilities and costs for each model) is not released. Only aggregated results are shown in tables and figures. The code and data are promised for future release."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes dataset selection with specific benchmarks named. Section B describes preprocessing. Section C describes cost computation methodology. The paper explains how model outputs were collected using VLMEvalKit with standardized inputs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (OCRBench, MMStar, MathVista, etc.)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: benchmark selection (Section 4.2) → model evaluation with VLMEvalKit (Section B.2) → cost normalization (Section C.2, Equation S1) → outcome table construction → frozen train/test splits → router training and evaluation. Each step is described with the methodology clearly laid out."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, acknowledgments, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from the School of Artificial Intelligence and National Key Laboratory for Novel Software Technology at Nanjing University. They are not affiliated with the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be assessed. The absence of a funding disclosure does not confirm the absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 10 MLLMs including GPT-5, Gemini 2.5, Claude 3.7, and various open-weight models but does not state training data cutoff dates for any of them."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models on public benchmarks (some dating to 2023-2024) without any discussion of whether the test examples appeared in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks (OCRBench 2024, MathVista 2023, MathVerse 2024, SEED-Bench 2023) were published before the likely training cutoffs of models like GPT-5 and Gemini 2.5. This contamination risk is not discussed. Contamination would affect the utility estimates that form the basis of all routing evaluations."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table S2 reports per-model output token prices ($/1M tokens) from OpenRouter. Cost normalization is formally defined (Equation S1). The entire evaluation framework is cost-aware with normalized costs reported throughout."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget for collecting model outputs across 11,000 instances × 10 models is not stated. No mention of total API spend, GPU hours, or wall-clock time for the data collection process."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'results for stochastic routers are averaged over multiple runs' but does not report seed sensitivity analysis or variation across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'averaged over multiple runs' (Section 5.1) but does not specify the exact number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Router hyperparameters (k for clusters/neighbors, λ for cost weight, SVD rank, MLP architecture) appear tuned but no search budget (number of configurations tried, search method, compute spent) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper sweeps over the number of clusters k and cost weight λ to generate a family of routing policies and reports the Pareto frontier (Section 3.2). This reports all operating points rather than cherry-picking a single best configuration. Routers are trained on training splits and evaluated on held-out test splits."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 7 datasets, 3 scenarios, and multiple routing methods. No statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose MMR-Bench and evaluate routing methods on it. They do not acknowledge the potential bias of designing and evaluating on their own benchmark, nor discuss whether independent evaluation might yield different conclusions."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire evaluation framework reports performance as a function of normalized cost. Figure 4 and Figure S1 plot cost-accuracy Pareto frontiers. The QNC metric explicitly measures cost relative to the strongest single model. Section 5.3 discusses cost efficiency."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper motivates why multimodal routing benchmarks are needed (Section 1-2) and compares against existing benchmarks (Table 1), but does not critically examine whether MMR-Bench actually measures routing quality as claimed. No discussion of potential validity gaps, such as whether offline precomputed utilities accurately reflect real-world routing decisions."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly with standardized prompts and decoding settings. The routing system is a feature-extraction → selection pipeline, not an agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss temporal leakage. Several benchmarks (MathVista 2023, SEED-Bench 2023, OCRBench 2024) were published well before the training cutoffs of models like GPT-5 (2025+), meaning models may have encountered benchmark solutions during training. This would distort the utility estimates that underpin all routing evaluations."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak answer information. For example, MCQ datasets include answer options in the prompt, which may provide signal about the correct answer."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether examples across the 7 benchmarks share structural similarities or whether train/test splits within datasets maintain independence (this is inherited from the original benchmarks without verification)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used. No canary strings, membership inference, n-gram overlap analysis, or temporal split verification."
       }
     }
   },
   "claims": [
     {
       "claim": "Multimodal signals improve routing quality compared to unimodal signals (text-only or image-only).",
       "evidence": "Figure 2 shows multimodal router consistently outperforms text-only and image-only variants across cost levels. Section 3.2 (Q2) demonstrates the gap widens in settings where visual clutter or linguistic complexity dominates. Table 3 shows adaptive fusion gains for most router families.",
       "supported": "moderate"
     },
     {
       "claim": "The routed system can match or exceed the strongest single model's accuracy at roughly 33% of its cost.",
       "evidence": "Figure 4 shows the Pareto frontier of routing strategies dominating single-model baselines. Section 5.3 states 'with only 33% of the strongest single model's cost, our router already matches its accuracy.'",
       "supported": "moderate"
     },
     {
       "claim": "Routing policies trained on a subset of models and tasks generalize zero-shot to new datasets and text-only benchmarks.",
       "evidence": "Table 4 shows cross-dataset routing within scenarios consistently beats the best single model (e.g., Math Ps 0.7914 vs 0.7592). Table 5 shows multimodal-trained router exceeding best single model on GSM8K (96.7 vs 94.5), MMLU (92.4 vs 91.2), and ARC (66.7 vs 65.7).",
       "supported": "moderate"
     },
     {
       "claim": "Matrix-factorization-based routers achieve the most robust performance across heterogeneous workloads.",
       "evidence": "Table 2 shows LinearMFRouter achieves highest full-dataset nAUC (0.7042) and Ps (0.7533). Section 5.3 states MF routers are 'consistently superior on general VQA' and have better cross-category robustness.",
       "supported": "moderate"
     },
     {
       "claim": "Adaptive fusion reveals a clear modality gap between image and text signals.",
       "evidence": "Table 3 shows large gains for KMeans (ΔnAUC +0.3403) and consistent gains for MLP and Linear routers. QNC improves from +∞ to finite values for KMeans and MLP. However, KNN shows mild regression (ΔnAUC −0.0074).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or error bars reported",
       "detail": "All results (Tables 2, 3, 4, 5, S4) report only point estimates despite stating results are 'averaged over multiple runs.' The reader cannot assess whether observed differences are meaningful or within noise. This is a Big Numbers No Error Bars pattern."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Claims of method superiority (e.g., 'LinearMFRouter attains the highest nAUC') are based on comparing raw numbers without any statistical tests. With unknown variance, observed differences could be within noise."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "The utility estimates that form the basis of all routing evaluations depend on model accuracy on public benchmarks. Models like GPT-5 may have been trained on benchmark data (MathVista 2023, SEED-Bench 2023), which would distort utility values and consequently routing decisions. This fundamental threat is not discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations, threats to validity, or scope boundaries. Key unaddressed limitations include: the offline evaluation assumption (real routing involves dynamic model updates), the small model zoo (10 models), and the restriction to text+image modalities despite the 'Comprehensive' title."
     },
     {
       "flag": "Artifacts not yet available",
       "detail": "Code, outcome tables, and scoring scripts are promised ('The code will be available at...') but not released. The paper's reproducibility claims cannot be verified."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational commercial LLM evaluated in the routing benchmark; relevant to survey scope on LLM capabilities."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Early work on cost-efficient LLM usage through model routing and cascading, directly relevant to AI deployment methodology."
     },
     {
       "title": "AutoMix: Automatically mixing language models",
       "authors": ["Pranjal Aggarwal", "Aman Madaan"],
       "year": 2024,
       "relevance": "Dynamic model mixing for LLMs; relevant to efficient agentic AI deployment and model selection."
     },
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "authors": ["Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li"],
       "year": 2024,
       "arxiv_id": "2403.12031",
       "relevance": "Text-only LLM routing benchmark; direct predecessor to MMR-Bench, relevant to evaluation methodology for LLM systems."
     },
     {
       "title": "RouterEval: A comprehensive benchmark for routing LLMs to explore model-level scaling up in LLMs",
       "authors": ["Zhongzhan Huang", "Guoming Ling", "Yupei Lin"],
       "year": 2025,
       "arxiv_id": "2503.10657",
       "relevance": "Contemporary text-only routing benchmark; relevant to evaluation methodology for LLM model selection."
     },
     {
       "title": "EmbedLLM: Learning compact representations of large language models",
       "authors": ["Richard Zhuang", "Tianhao Wu", "Zhaojin Wen"],
       "year": 2024,
       "arxiv_id": "2410.02223",
       "relevance": "LLM representation learning for model selection; relevant to efficient LLM deployment and routing."
     },
     {
       "title": "Capability instruction tuning: A new paradigm for dynamic LLM routing",
       "authors": ["Yi-Kai Zhang", "De-Chuan Zhan", "Han-Jia Ye"],
       "year": 2025,
       "relevance": "Dynamic LLM routing through capability-aware instruction tuning; directly relevant to agentic AI deployment."
     },
     {
       "title": "Large language model routing with benchmark datasets",
       "authors": ["Tal Shnitzer", "Anthony Ou", "Mírian Silva"],
       "year": 2023,
       "arxiv_id": "2309.15789",
       "relevance": "Early work on LLM routing using benchmark performance data; foundational for the model selection problem."
     },
     {
       "title": "MoE-LLaVA: Mixture of experts for large vision-language models",
       "authors": ["Bin Lin", "Zhenyu Tang", "Yang Ye"],
       "year": 2024,
       "arxiv_id": "2401.15947",
       "relevance": "Mixture-of-experts architecture for multimodal LLMs; relevant to efficient MLLM deployment and intra-model routing."
     },
     {
       "title": "GraphRouter: A graph-based router for LLM selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2024,
       "arxiv_id": "2410.03834",
       "relevance": "Graph-based approach to LLM routing; relevant to model selection methodology for AI systems."
     },
     {
       "title": "LLM-Blender: Ensembling large language models with pairwise ranking and generative fusion",
       "authors": ["Dongfu Jiang", "Xiang Ren", "Bill Yuchen Lin"],
       "year": 2023,
       "arxiv_id": "2306.02561",
       "relevance": "LLM ensembling approach; relevant to multi-model AI systems and output selection."
     },
     {
       "title": "Qwen2.5-VL technical report",
       "authors": ["Shuai Bai", "Keqin Chen", "Xuejing Liu"],
       "year": 2025,
       "arxiv_id": "2502.13923",
       "relevance": "Open-weight multimodal LLM used in the routing benchmark; relevant to MLLM capabilities."
     }
   ]
 }