ai-research-survey

scan-v5.json (20566B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing",
     "authors": [
       "Hao Li",
       "Yiqun Zhang",
       "Zhaoyan Guo",
       "Chenxu Wang",
       "Shengji Tang"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.07206",
     "doi": "10.48550/arXiv.2601.07206"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are substantiated: 400K+ instances from 21 datasets and 33 models are documented in Table 2; routing method convergence is shown in Fig 4; OpenRouter underperformance is shown in Fig 7; the Oracle gap driven by model-recall failures is shown in Fig 5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about embedding model impact are supported by ablation (Table 10, three embedding alternatives tested); diminishing returns from ensemble size are tested via Oracle-controlled subset experiments (Fig 6) with random vs. top-k comparison as a direct counterfactual.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states results do not generalize to domain-specific verticals, very long-context tasks, or multimodal benchmarks, and that only routing methods with public open-source implementations are covered.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper offers an alternative explanation for routing method convergence—that gains may be attributable to capturing coarse domain structure (math vs. code) rather than nuanced decision boundaries—evidenced by proximity to the Dataset Oracle in Fig 4.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes AvgAcc (what is measured) from Oracle performance (the ceiling), explicitly frames Gain@B and Gap@O as distinct quantities, and defines PerfGain and CostSave in terms of the Best Single baseline to clarify what improvement claims mean.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present listing three specific limitations: coverage of routing methods, dataset domain coverage, and the approximate nature of latency analysis.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific: (1) only routing methods with open-source implementations are covered; (2) domain-specific verticals, long-context, and multimodal settings are excluded by name; (3) latency estimates rely on token-level statistics from OpenRouter and are explicitly noted as 'indicative rather than definitive.'",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly names excluded settings (domain-specific verticals, very long-context, multimodal) and notes that benchmark formulations 'can be applied to these settings by adding new dataset evaluators,' implicitly distinguishing what is and is not currently supported.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states the work was supported by the Shanghai Municipal Science and Technology Major Project.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Shanghai Artificial Intelligence Laboratory; footnotes additionally note that Li, Guo, and Wang are affiliated with Northwestern Polytechnical University and that four authors were interns at Shanghai AI Lab.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is a government science/technology grant with no direct interest in any particular routing method or LLM provider evaluated; no authors are employees of the evaluated commercial services (OpenAI, Anthropic, Google, OpenRouter).",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests such as patents, equity, or consulting relationships anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLM routing is defined precisely in the opening sentence; performance-oriented vs. performance-cost tradeoff paradigms are explicitly distinguished; all evaluation metrics (AvgAcc, Gain@R, Gain@B, Gap@O, PerfGain, CostSave, ParetoDist) are given formal mathematical definitions in Section 3.4.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 ends with a bulleted list of three explicit contributions: large-scale timely data, an open-source unified routing framework with 10 baselines, and a systematic re-examination of the routing landscape.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 directly compares LLMRouterBench against five prior benchmarks on seven dimensions; the Related Work section covers both routing paradigms with citations to ten prior methods and explains how this work extends or corrects prior limitations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that routing capability requires diverse task coverage spanning mathematics, code, logic, knowledge, affective, instruction following, and tool use; dataset exclusion criteria (no saturated or uniformly-poor datasets) are explicitly stated to ensure benchmark discriminates routing quality.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Fig 5 explicitly characterizes query hardness by the number of models that answer correctly, showing a distribution from 1 to all 20 correct; the paper explicitly excludes saturated datasets (all models high) and uniformly difficult datasets (all models low) for each model pool.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 explicitly states saturated datasets are excluded for flagship models and excessively challenging datasets where all models perform uniformly poorly are excluded for lightweight models—a direct procedural check for ceiling and floor effects.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline performance is reported for any dataset; the only baselines are Random Router, Best Single, and Oracle (all model-based).",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Each metric is mathematically defined and justified: Gain@B measures improvement over the best single model, Gap@O measures distance to the theoretical ceiling, and ParetoDist captures multi-objective optimality; the paper explains why these reference points are meaningful rather than arbitrary.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark includes several older datasets (HumanEval, MBPP, BBH, MMLU-Pro) that are likely contaminated in modern LLM training sets; while some live benchmarks (LiveCodeBench, LiveMathBench) provide temporal freshness, contamination resistance is not systematically addressed as a design principle.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss how quickly models will saturate the benchmark, whether the fixed 33-model pool will become outdated, or what the update strategy is; temporal robustness is entirely absent from the Limitations and Conclusions sections.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The Limitations section covers coverage gaps (missing routing methods, missing task types) but does not discuss failure modes of the benchmark itself—such as routing methods overfitting to the fixed 70/30 train/test split, LLM-as-judge bias in three datasets, or the benchmark being gamed by embedding the evaluation datasets in model pretraining.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "All code and data are released at GitHub; Appendix B provides detailed implementation specifications for all 10 baselines including hyperparameters, embedding models, hardware, and 5-seed averaging; the framework uses modular adapters for plug-and-play reproduction.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Table 9 documents all 21 datasets with abbreviations, categories, evaluation metrics, and sizes; Appendix B.1 describes collection methodology including inference hardware (A800-80G, vLLM 0.8.4), API providers, temperature/sampling settings, and retry policies; Tables 7 and 8 document all 33 models with parameters and costs.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "The paper states 'All datasets and models used in this paper are publicly available and properly cited. Our usage complies with their original licenses and intended research purposes,' and provides a public GitHub repository for code and data access.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies what the benchmark enables (fair comparison, routing development, systematic re-evaluation) but does not state what conclusions should NOT be drawn—for example, it does not caution against using the benchmark as the sole basis for selecting a production router or against assuming results generalize to uncovered domains.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Most routing methods exhibit similar performance under unified evaluation despite ongoing methodological innovation.",
       "evidence": "Fig 4 shows EmbedLLM (70.29%), GraphRouter (71.88%), MODEL-SAT (71.94%), and Avengers (73.10%) are within ~2.8pp on AvgAcc; Fig 4 shows similar convergence across Gain@R, Gain@B, and Gap@O metrics averaged over 5 seeds.",
       "supported": "strong"
     },
     {
       "claim": "Several recent routing approaches, including the commercial router OpenRouter, fail to reliably outperform the Best Single baseline.",
       "evidence": "Fig 7 shows OpenRouter achieves -24.7% PerfGain relative to Best Single (GPT-5); HybridLLM and FrugalGPT are marked N/A for CostSave (fail to match Best Single accuracy at lower cost); Table 12 confirms OpenRouter AvgAcc of 49.67 vs. GPT-5's 65.96.",
       "supported": "strong"
     },
     {
       "claim": "A substantial performance gap to Oracle remains, driven primarily by model-recall failures on queries where only one model answers correctly.",
       "evidence": "Fig 4(d) shows Gap@O of 20.7-33.3% across all routers; Fig 5 shows that on 410 queries (11.9% of test set) where ≤3 models are correct, Avengers and EmbedLLM achieve only 24.6% and 23.2% accuracy respectively.",
       "supported": "strong"
     },
     {
       "claim": "Backbone embedding model choice has limited impact on routing performance.",
       "evidence": "Table 10 shows that replacing gte-qwen2-7B-instruct with much weaker all-MiniLM-L6-v2 (22.7M params) changes Avengers AvgAcc by only 0.91pp (71.94 vs. 71.03), GraphRouter by 2.24pp, and EmbedLLM by 0.29pp.",
       "supported": "moderate"
     },
     {
       "claim": "Expanding the model ensemble yields diminishing returns; a carefully selected small subset can outperform a larger random pool.",
       "evidence": "Fig 6 shows Oracle AvgAcc rising steeply from 2 to ~8 models then plateauing through 20; top-2 curated models (Qwen3-8B, NVIDIA-Nemo) outperform random k-model selections at k=6-20.",
       "supported": "strong"
     },
     {
       "claim": "Avengers-Pro achieves near-Pareto-optimal performance-cost tradeoff among all evaluated routing methods.",
       "evidence": "Fig 8 shows Avengers-Pro ParetoDist of 0.001 (near-zero) while all other methods range from 0.037 (RouteLLM) to 0.394 (OpenRouter); Fig 8 shows Avengers-Pro configurations spanning the empirical Pareto frontier.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "LLMRouterBench, comprising 400K+ instances across 21 datasets and 33 models, reveals that under unified evaluation most routing methods collapse to similar performance despite methodological diversity, and several recent approaches including the commercial router OpenRouter fail to outperform simply using the best single model. A large Oracle gap persists (20-33%), driven primarily by model-recall failures on hard queries where only one candidate model produces a correct answer. Common assumptions are empirically refuted: embedding model quality has negligible routing impact, and expanding ensemble size yields diminishing returns while careful model curation provides more benefit.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "All comparisons are model-to-model; there is no human performance reported for any of the 21 datasets, making it impossible to contextualize whether the Oracle gap represents a fundamental routing difficulty or a tractable human-level task."
     },
     {
       "flag": "Contamination not addressed for older benchmarks",
       "detail": "Many included datasets (HumanEval, MBPP, BBH, MMLU-Pro, GPQA) are widely used and likely present in modern LLM pretraining data, but the paper does not discuss or test for contamination effects on routing outcomes."
     },
     {
       "flag": "No competing interests statement",
       "detail": "Despite evaluating commercial APIs (GPT-5, Claude, Gemini, OpenRouter) and spending $2,771 in API costs, no competing interests statement is provided."
     },
     {
       "flag": "LLM-as-judge bias unaddressed",
       "detail": "Three datasets (HLE, SimpleQA, ArenaHard) use LLM judges (o3-mini, DeepSeek-V3); the paper does not discuss how judge model bias or judge-model correlation might favor certain frontier models in these datasets."
     },
     {
       "flag": "Temporal robustness absent",
       "detail": "The paper presents a static benchmark with a fixed 33-model pool and 21 datasets with no discussion of update strategy or how quickly routing methods will overfit once the benchmark becomes standard."
     }
   ],
   "cited_papers": [
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "relevance": "Direct predecessor benchmark for multi-LLM routing, used as primary comparison baseline in Table 1."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "relevance": "Performance-cost routing baseline trained on preference data, evaluated and compared in experiments."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "relevance": "Foundational cascaded inference routing method for performance-cost tradeoff, directly evaluated as baseline."
     },
     {
       "title": "HybridLLM: Cost-Efficient and Quality-Aware Query Routing",
       "relevance": "Binary routing method between small and large models, evaluated as performance-cost baseline."
     },
     {
       "title": "EmbedLLM: Learning Compact Representations of Large Language Models",
       "relevance": "Embedding-based routing baseline evaluated in performance-oriented setting; used for embedding ablation."
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "relevance": "Early performance-oriented routing method that motivated the field; cited as foundational work."
     },
     {
       "title": "GraphRouter: A Graph-Based Router for LLM Selections",
       "relevance": "Graph neural network routing baseline evaluated in both performance and performance-cost settings."
     },
     {
       "title": "RouterEval: A Comprehensive Benchmark for Routing LLMs to Explore Model-Level Scaling Up in LLMs",
       "relevance": "Competing routing benchmark compared in Table 1, providing context for LLMRouterBench's improvements."
     },
     {
       "title": "The Avengers: A Simple Recipe for Uniting Smaller Language Models to Challenge Proprietary Giants",
       "relevance": "Clustering-based routing method that achieves top performance in performance-oriented setting without neural training."
     },
     {
       "title": "RouterArena: An Open Platform for Comprehensive Comparison of LLM Routers",
       "relevance": "Recent routing benchmark that treats routing as a black box, cited as a limitation motivating per-prompt transparency."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "LLM routing directly addresses production cost and performance tradeoffs; the finding that commercial routers underperform simple baselines is immediately actionable for practitioners."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "The core finding that methodological innovation has not produced differentiated routing methods, and that OpenRouter performs worse than the best single model, directly challenges the field's progress narrative."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper raises no AI safety or risk concerns; it is purely about performance and cost optimization of LLM routing."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Showing that a widely used commercial router (OpenRouter) performs 24.7% worse than the Best Single model is mildly confrontational and newsworthy for the industry."
     },
     "demo_ability": {
       "score": 3,
       "justification": "All code and data are publicly released at GitHub with detailed implementation instructions, 10 plug-and-play baselines, and modular adapters for new routing methods."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Shanghai Artificial Intelligence Laboratory is a well-funded national lab but lacks the global brand recognition of Anthropic, Google DeepMind, or OpenAI in the ML research community."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }