ai-research-survey

scan.json (30862B)

---

 {
   "paper": {
     "title": "LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing",
     "authors": [
       "Hao Li",
       "Yiqun Zhang",
       "Zhaoyan Guo",
       "Chenxu Wang",
       "Shengji Tang",
       "Qiaosheng Zhang",
       "Yang Chen",
       "Biqing Qi",
       "Peng Ye",
       "Lei Bai",
       "Zhen Wang",
       "Shuyue Hu"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.07206",
     "doi": "10.48550/arXiv.2601.07206"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMRouterBench evaluates 10 routing methods across 21 datasets and 33 models (400K+ instances). Despite ongoing methodological innovation, most routing methods achieve nearly indistinguishable performance under unified evaluation, and several recent methods (including the commercial router OpenRouter) fail to outperform a simple Best Single baseline. A substantial gap to the Oracle persists, driven by model-recall failures on hard queries. Embedding model quality has minimal impact on routing, and expanding model pools yields diminishing returns compared to careful model curation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the abstract: 'All code and data are available at https://github.com/ynulihao/LLMRouterBench.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Same URL provides all data. The abstract states 'All code and data are available.' The benchmark comprises 400K+ instances across 21 datasets and 33 models."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'vLLM 0.8.4' and 'NVIDIA A800-80G GPUs' (Section B.1) and DeepSpeed for training, but does not provide a requirements.txt, Dockerfile, or comprehensive environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is provided with all code and data. The paper specifies exact random seeds (42, 999, 2024, 2025, 3407), train/test splits (70/30), embedding models, hyperparameters for each baseline (Appendix B.3), and data collection settings (Appendix B.1). A competent researcher could reproduce results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 5 random seeds (Appendix Tables 11-13) but no confidence intervals, error bars, or ± notation are reported. Only point estimates appear in all tables and figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'many routing approaches achieve broadly comparable performance') but provides no statistical significance tests. Differences between methods are compared by inspection of point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines and reports Gain@R, Gain@B, Gap@O, PerfGain, and CostSave metrics that contextualize improvements relative to baselines (Section 3.4). For example, 'up to a 4% average accuracy gain over Best Single' and '31.7% cost reduction' (Section 4.2.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the choice of 5 random seeds, 21 datasets, or 33 models. The scale is described but not justified through power analysis or formal reasoning."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 5 runs with different seeds, but no standard deviations, interquartile ranges, or other spread measures are reported in any table or figure. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three reference baselines are defined: Random Router, Best Single, and Oracle (Section 3.4). Additionally, 10 routing methods are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent methods (2024-2025): GraphRouter (ICLR 2025), Avengers-Pro (2025), MODEL-SAT (2025), RouteLLM (2024), plus the commercial OpenRouter. Models include GPT-5, Claude-4, Gemini-2.5-Pro."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Embedding model ablation (Table 10) tests 3 different embedding backbones across 3 routing methods. Model pool size ablation (Fig. 6) tests top-k vs. random selection across pool sizes 2-20."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Seven metrics are defined and used: AvgAcc, Gain@R, Gain@B, Gap@O for performance-oriented; PerfGain, CostSave, ParetoDist for performance-cost (Section 3.4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates routing methods on standard benchmark accuracy metrics. Human evaluation is irrelevant to the core claims about which router achieves higher automated accuracy scores."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2: 'All experiments are conducted using a 70% training and 30% test split, repeated five times with different random seeds.' Results are reported on the 30% test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-dataset breakdowns in Tables 4, 5, 11, 12, plus per-domain analysis (mathematics, code, logical, knowledge, affective) in Fig. 3 and throughout."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Model-recall failures are analyzed in Fig. 5: 'on queries where at most three experts answer correctly (410 queries, 11.9% of test set), Avengers and EmbedLLM achieve low accuracy (24.6% and 23.2%).' OpenRouter's failure is discussed (-24.7% PerfGain)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative findings: routing methods don't differentiate much from each other, OpenRouter fails to beat Best Single, embedding models have limited impact, larger ensembles show diminishing returns, binary routers (HybridLLM, FrugalGPT) struggle on cost savings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about complementarity (Fig. 3), similar method performance (Fig. 4), Oracle gap (Fig. 4d), embedding impact (Table 10), and diminishing returns (Fig. 6) are all supported by corresponding results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal-type claims are backed by controlled experiments: 'embedding models have limited impact' via ablation (Table 10, swapping 3 embedding backbones), 'diminishing returns from adding models' via systematic pool size variation (Fig. 6). The ablation designs are adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested setting. The paper qualifies with 'under unified evaluation' and 'in practice.' The Limitations section explicitly states untested domains: 'domain-specific verticals, very long-context tasks, and multimodal benchmarks are not included.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that 'a large fraction of routing gains may be attributable to capturing coarse-grained domain structure rather than learning highly nuanced decision boundaries' (Section 4.2.1), supported by comparison to the Dataset Oracle. Model-recall failure is identified as the primary driver of the Oracle gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims are stated at the level of benchmark accuracy and cost metrics, matching the granularity of measurements. The paper acknowledges latency as an additional practical dimension not fully captured (Section 4.2.2). No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models include version dates (deepseek-r1-0528, deepseek-v3-0324, Qwen3-235b-a22b-2507, GLM-Z1-9B-0414) but several flagship models use marketing names without snapshot dates: GPT-5-medium, Claude-sonnet-4, Gemini-2.5-flash, Gemini-2.5-pro (Tables 7-8). Model behavior changes across API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 0-shot or 3-shot evaluation and mentions 'official prompts' for LLM-as-judge tasks (Table 9) but does not include the actual prompt text used for any benchmark or routing evaluation. The reader cannot reconstruct exact prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters in Appendix B: temperature 0.2, top_p 1.0 for all models (B.1); learning rates, batch sizes, epochs, warmup ratios for each routing method (B.3); k=64 for Avengers clustering; 10,000 epochs for GraphRouter."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The routing methods are classification/optimization algorithms, not agent-based systems."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 documents the data collection pipeline: model deployment on A800 GPUs, API collection via OpenRouter, caching and retry logic (up to 10 retries), failure scoring (0), temperature/sampling settings. The modular Collector-Evaluator-Adaptor pipeline is described in Section 3.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section identifies three specific limitations: incomplete routing method coverage, restricted domain/task coverage, and approximate latency analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: 'we do not cover all existing approaches' with explanation that they focus on those with public implementations; 'domain-specific verticals, very long-context tasks, and multimodal benchmarks are not included'; latency estimates correspond to 'a specific provider configuration and should be interpreted as indicative.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what is not covered: 'domain-specific verticals, very long-context tasks, and multimodal benchmarks, are not included' and 'These estimates correspond to a specific provider configuration and should be interpreted as indicative rather than definitive.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'All code and data are available at https://github.com/ynulihao/LLMRouterBench.' This includes the 400K+ instances with per-prompt, per-model outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes data collection: open-source models deployed on A800 GPUs with vLLM 0.8.4, flagship models collected via OpenRouter and official APIs (GLM-4.6, Intern-S1), temperature 0.2, top_p 1.0, up to 10 retries, failures scored as 0."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and LLM API outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented through the modular framework (Section 3.3, Fig. 2): Collector handles API calls with caching/retries/cost tracking, Evaluator implements dataset-specific metrics, Adaptor converts to algorithm-specific inputs with consistent train/test splits. Table 2 provides summary statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work was supported by the Shanghai Municipal Science and Technology Major Project.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Shanghai Artificial Intelligence Laboratory and Northwestern Polytechnical University. Internship affiliations are also noted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Shanghai Municipal Science and Technology Major Project is a government funding source with no financial stake in which routing method performs best."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. Notably, several authors (Yiqun Zhang, Hao Li, Chenxu Wang, Peng Ye, Shuyue Hu) are also authors of Avengers and Avengers-Pro, methods that perform best in their benchmark. This conflict is not acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 33 models used. The paper evaluates these models on benchmarks without discussing when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (e.g., HumanEval from 2021, BBH, MMLU-Pro) appeared in the training data of the evaluated models. Some benchmarks predate many models by years."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple benchmarks (HumanEval 2021, BBH 2022, MMLU-Pro, GPQA) were publicly available before the training cutoff of many evaluated models. Some temporal benchmarks are used (LiveCodeBench, LiveMathBench) but contamination is never explicitly discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed cost analysis in Table 6 (per-model per-dataset costs), Table 13 (per-routing-method costs), and Fig. 8 (cost vs accuracy Pareto frontier). Total API costs stated as $2,771.84 (Table 2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 states: 'A800 1000 GPU hours' for lightweight model inference, '$2,771.84' for API costs including '$500 from LLM-based judging.' Individual routing method parameter counts are reported in B.3."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2: 'repeated five times with different random seeds (42, 999, 2024, 2025, and 3407).' Results are averaged over these seeds. However, no per-seed variance is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated in Section B.2: five runs with different random seeds. Appendix Tables 11-13 note results are 'averaged over five random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are specified for each method (Appendix B.3) but no search budget is reported. Some methods have modified hyperparameters from originals (e.g., GraphRouter epochs increased to 10,000, EmbedLLM batch size to 32,768) without justifying how these values were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For most methods, default or slightly modified configurations from original papers are used without formal justification. For Avengers-Pro, 101 configurations are swept (alpha from 0 to 1 in 0.01 increments), but selection criteria for final comparisons are not discussed."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous pairwise comparisons across 10 routing methods, 33 models, and 21 datasets. No statistical tests are performed, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several authors (Yiqun Zhang, Hao Li, Chenxu Wang, Peng Ye, Shuyue Hu) are authors of Avengers and Avengers-Pro — the methods found to be best-performing (performance-oriented) and Pareto-optimal (cost setting) in the benchmark. MODEL-SAT was re-implemented due to incomplete official code. Neither author-evaluation bias nor re-implementation bias is acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Routing method training costs vary enormously: RouterDC has ~7B parameters, MODEL-SAT ~14B, EmbedLLM ~12M, GraphRouter ~0.1M, and Avengers requires no training. Performance is not compared at matched training compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects benchmarks by excluding saturated and trivially-hard datasets, but does not discuss whether benchmark accuracy actually measures real-world routing quality. No formal construct validity analysis is provided."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models are queried directly through APIs with uniform settings. Routing methods are classification algorithms, not scaffolded agents."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Some temporal benchmarks are included (LiveCodeBench, LiveMathBench) which mitigate leakage, but the paper never explicitly discusses temporal leakage. Older benchmarks (HumanEval 2021, BBH 2022) are used alongside 2025-2026 models without addressing temporal overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup provides prompts to models; whether these prompts or their structure leaks information relevant to routing is not considered."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The 70/30 train/test split is applied per-dataset, but whether benchmark examples share structural similarities or come from related sources is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Models exhibit clear complementarity — no single model dominates all domains, which validates the central premise of LLM routing.",
       "evidence": "Fig. 3 shows different models leading different domains: Intern-S1-mini/Qwen3-8B in math, Fin-R1/Qwen-Coder in code, DS-Qwen3/MiniCPM in logic, Gemma-2-it in affective tasks. Tables 4-5 confirm this across all datasets.",
       "supported": "strong"
     },
     {
       "claim": "Under unified evaluation, leading routing methods achieve broadly comparable performance — despite continued methodological innovation, there is little practical differentiation.",
       "evidence": "Fig. 4 shows EmbedLLM (71.24), GraphRouter (70.29), MODEL-SAT (71.88), and Avengers (71.94) all within ~2 points AvgAcc. Gain@B ranges 3.9-8.3% across all methods. Table 11 confirms across per-dataset results averaged over 5 seeds.",
       "supported": "strong"
     },
     {
       "claim": "Several recent routing methods, including the commercial router OpenRouter, fail to outperform the Best Single model baseline.",
       "evidence": "Fig. 7 shows OpenRouter at -24.7% PerfGain, HybridLLM at -12.7%, FrugalGPT at -9.5% relative to GPT-5. OpenRouter achieves 49.67% AvgAcc vs. GPT-5's 65.96% (Table 12).",
       "supported": "strong"
     },
     {
       "claim": "A substantial performance gap persists relative to the Oracle baseline, driven primarily by model-recall failures on hard queries.",
       "evidence": "Fig. 4d shows Gap@O of 20.7-23.7% for top methods. Fig. 5 shows that on queries where at most 3 models answer correctly (410 queries, 11.9%), Avengers achieves 24.6% and EmbedLLM 23.2% routing accuracy.",
       "supported": "strong"
     },
     {
       "claim": "Backbone embedding models have limited impact on routing performance.",
       "evidence": "Table 10 shows replacing gte-qwen2-7B-instruct (3584-dim, 7B params) with nli-bert-base or all-MiniLM-L6-v2 (22.7M params) yields comparable results across GraphRouter, EmbedLLM, and Avengers. Differences are within ~1-3 points.",
       "supported": "moderate"
     },
     {
       "claim": "Expanding the model ensemble yields diminishing returns, but a well-chosen subset can substantially outperform a larger random pool.",
       "evidence": "Fig. 6 shows Oracle AvgAcc plateaus around 8-10 models for random selection. Top-2 selection (Qwen3-8B + NVIDIA-Nemo) matches random pools of 6-8 models. Top-4 selection outperforms larger random pools.",
       "supported": "moderate"
     },
     {
       "claim": "Avengers-Pro achieves near Pareto-optimal performance-cost tradeoff, dominating the frontier.",
       "evidence": "Fig. 8 shows Avengers-Pro with ParetoDist = 0.001, far below other methods (GraphRouter 0.037-0.087, FrugalGPT 0.152, OpenRouter 0.394). Nearly all Avengers-Pro configurations lie on or near the Pareto frontier.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Authors evaluate their own methods in their own benchmark",
       "detail": "Several LLMRouterBench authors (Yiqun Zhang, Hao Li, Chenxu Wang, Peng Ye, Lei Bai, Shuyue Hu) are also authors of Avengers (Zhang et al., 2025d) and Avengers-Pro (Zhang et al., 2025c). These methods are found to be top-performing in both the performance-oriented setting (Avengers: highest AvgAcc among routing methods) and the performance-cost setting (Avengers-Pro: Pareto-optimal with ParetoDist near zero). This conflict is not disclosed or acknowledged."
     },
     {
       "flag": "No variance or uncertainty quantification despite multiple runs",
       "detail": "Results are averaged over 5 random seeds but no standard deviations, confidence intervals, or error bars are reported in any table or figure. The reader cannot assess whether the observed differences between methods (often 1-3 points) are within or beyond run-to-run variance."
     },
     {
       "flag": "No contamination analysis for benchmarks predating model training",
       "detail": "HumanEval (2021), BBH (2022), MMLU-Pro, and other benchmarks were publicly available years before many evaluated models were trained. No contamination analysis is performed. If models have memorized benchmark answers, the routing evaluation is compromised — a router that learned which model memorized which answers would not generalize to novel queries."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes many comparative claims ('methods are broadly comparable', 'embedding models have limited impact') based solely on comparing point estimates. Without significance tests, it is unclear whether observed differences reflect genuine routing quality differences or random variation across seeds."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major code generation benchmark used in the performance-cost evaluation setting; central to evaluating LLM coding capability."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang", "Tianhao Wu", "Joseph E Gonzalez", "M Waleed Kadous", "Ion Stoica"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Trains routers from preference data to balance LLM performance and cost; key baseline in the performance-cost routing evaluation."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Pioneering work on cascaded LLM inference for cost reduction; baseline in the performance-cost setting."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang", "Robert Sim"],
       "year": 2024,
       "arxiv_id": "2404.14618",
       "relevance": "Routes queries between small and large LLMs based on predicted difficulty; baseline for performance-cost routing."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, one of the core evaluation datasets used in the performance-oriented setting."
     },
     {
       "title": "Humanity's Last Exam",
       "authors": ["Long Phan", "Alice Gatti", "Ziwen Han"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Frontier capability assessment benchmark used in the performance-cost setting to test flagship models on extremely hard questions."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark using temporal splits; used in both routing settings."
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "authors": ["Dongfu Jiang", "Xiang Ren", "Bill Yuchen Lin"],
       "year": 2023,
       "relevance": "Early work on LLM ensemble selection and output fusion; foundational to the routing research field."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major open-source reasoning model achieving near-GPT-5 performance at fraction of cost; central to cost-performance routing analysis."
     },
     {
       "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment",
       "authors": ["Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan"],
       "year": 2025,
       "arxiv_id": "2506.07982",
       "relevance": "Agentic tool-use benchmark used for latency-aware routing analysis; demonstrates latency as a third routing dimension."
     },
     {
       "title": "GraphRouter: A Graph-based Router for LLM Selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Graph-based approach to LLM routing using heterogeneous task-query-LLM interactions; baseline in both routing settings."
     }
   ]
 }