ai-research-survey

scan.json (27526B)

---

 {
   "paper": {
     "title": "Federate the Router: Learning Language Model Routers with Sparse and Decentralized Evaluations",
     "authors": [
       "Baris Askin",
       "Shivam Patel",
       "Anupam Nayak",
       "Andrea Vigano",
       "Jiin Woo",
       "Gauri Joshi",
       "Carlee Joe-Wong"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.22318",
     "doi": "10.48550/arXiv.2601.22318"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "This paper introduces the first federated framework for LLM query routing, enabling clients to collaboratively learn routing policies from local offline data without centralizing private queries. Across RouterBench-Data and ProxRouter-Data, federated routers (both MLP and K-Means) consistently improve the accuracy-cost frontier over client-local baselines, with K-Means benefiting more from federation. Theoretical convergence guarantees and suboptimality bounds show federated training reduces routing error through increased effective query and model coverage. Adaptive personalization further improves robustness under extreme client heterogeneity.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or archive is provided anywhere in the paper. No GitHub/Zenodo link found."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: RouterBench-Data (Hu et al., 2024) and ProxRouter-Data (Patel et al., 2025), both cited with references to their public releases."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies some components (all-mpnet-base-v2 encoder, MLP architecture details in Appendix C) and mentions using PSC Bridges-2 GPU, but provides no requirements.txt, Dockerfile, or library version listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithmic details are described but there are no runnable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (AUC values, accuracy-cost curves). No confidence intervals, error bars, or ± notation appear in any figure or table."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims federated routers 'improve' over local routers based solely on comparing AUC scores (e.g., 0.75 vs 0.63-0.72) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "AUC values are reported for all methods with baseline context (e.g., Federated 0.75 vs Client-Local ranging 0.63-0.72 in Figure 2), providing magnitude of improvement alongside absolute baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why N=10 clients was chosen, or why the Dirichlet concentration parameters (α=0.6, 0.03) were selected. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures across runs are reported. Results appear to be single-run experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: client-local (no-FL) routers trained independently on each client's data, and centralized training on pooled data (Appendix D.1, Figure 9)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include RouterBench (2024) and ProxRouter (2025) benchmarks, and the router architectures (MLP-Router, K-Means-Router) follow recent work by Hu et al. (2024) and Jitkrittum et al. (2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates multiple components: federated vs local training (Section 6.1-6.2), model expansion (Section 6.3), client expansion (Appendix D.3), and adaptive personalization vs pure federated/local (Section 6.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated using accuracy, average cost, and normalized AUC under the accuracy-cost tradeoff curve. The tradeoff parameter λ is swept to generate full Pareto frontiers."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to evaluating routing algorithms. The paper measures routing quality via accuracy-cost tradeoffs on benchmark data."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states: 'Within each client, we split its allocated data into local train/test with fractions 0.75/0.25, and the global train/test sets are defined as the unions of client train/test splits.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-client (Figures 10-11 show all 10 clients), per-router type (MLP vs K-Means), per-dataset (RouterBench vs ProxRouter in Appendix F), and per-heterogeneity regime (α=0.6 vs α=0.03)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly discusses when federated routing fails: 'under extreme client data heterogeneity, a federated router can underperform on some clients' local distributions.' Specific clients where this occurs are shown in Figure 5."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 reports that under extreme heterogeneity (α=0.03), 'federated MLP-Router can indeed underperform local routers for some clients' (e.g., Client 4 and 6 in Figure 5 top row)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'federated collaboration improves the accuracy-cost frontier over client-local routers' (Figures 2-3), 'both via increased effective model coverage and better query generalization' (Sections 6.1-6.2), 'theoretical results validate that federated training reduces routing suboptimality' (Theorems 5.3, 5.5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'federated collaboration improves the accuracy-cost frontier' are supported by controlled experiments comparing federated vs local training on the same data partitions, varying only the training procedure. This is analogous to a controlled ablation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded: the abstract says 'Across two benchmarks' and results are specific to RouterBench-Data and ProxRouter-Data. The title focuses on 'sparse and decentralized evaluations' which matches the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why federated routing improves over local routing beyond the proposed mechanisms (better query coverage, model coverage). No robustness checks against confounds like data quantity effects independent of federation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy and cost on routing benchmarks and claims routing quality improvements. The measurements directly correspond to the claims — no proxy gap exists between what is measured (accuracy-cost AUC) and what is claimed (better routing)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The sentence encoder is specified as 'all-mpnet-base-v2' (Song et al., 2020). The LLM pool models are specified with versions in Figure 8: GPT-4 1106 Preview, GPT-3.5 Turbo 1106, Claude Instant v1, Claude v1, Claude v2, WizardLM 13B V1.2, Llama 2 70B Chat, etc."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It trains routing algorithms (MLP, K-Means) on pre-existing benchmark evaluation data. No prompts are sent to LLMs as part of the method."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C reports: learning rate η=10⁻³, weight decay 3×10⁻⁴, batch size 128, dropout p=0.1, gradient clipping max-norm 1.0, hidden layer widths (512, 512), Klocal=15, Kglobal=20, ninit=3 restarts, partial participation rate 0.6."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach trains standard MLP and K-Means models for routing."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 and Appendix B-C describe: Dirichlet partitioning with α=0.6 over task labels for query heterogeneity, Dirichlet α=0.45 for model assignment, train/test split 0.75/0.25, and encoder embedding procedure."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion (Section 7) is four sentences mentioning online routing as future work but does not discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address concerns like whether synthetic Dirichlet partitioning reflects real federated heterogeneity, or whether single-run results are reliable."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show (e.g., real-world federated deployment, privacy guarantees beyond data locality, or scaling beyond 10 clients)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying benchmark datasets (RouterBench-Data and ProxRouter-Data) are publicly available as cited. The federated partitioning procedure is fully described with parameters, enabling reproduction."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data comes from published benchmarks: RouterBench-Data (11 LLMs over 8 datasets, Hu et al. 2024) and ProxRouter-Data (14 LLMs over 10 datasets, Patel et al. 2025). The federated simulation procedure is described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (RouterBench-Data, ProxRouter-Data)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark data to federated experiments is documented: Dirichlet partitioning (α values specified), model assignment (Dirichlet α=0.45), single-model-per-query assumption, train/test split (0.75/0.25), and encoding via all-mpnet-base-v2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses funding: US DOE grant DESC0025652, multiple NSF grants (CNS-2409138, CNS-2533813, CCF 2045694, etc.), AI2C Seed grant, and ONR N00014-23-1-2149."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Carnegie Mellon University, clearly stated on the first page. They are not evaluating a product from their own institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government agencies (NSF, DOE, ONR) and a university seed grant (AI2C), none of which have a financial interest in the routing results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capabilities on benchmarks. It trains routing algorithms (MLP, K-Means) on pre-existing LLM evaluation data. Contamination of the routing model is addressed by train/test splitting."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates routing algorithms, not pre-trained language models. Standard train/test splitting is used for the router."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination concerns about LLM training data seeing benchmark solutions are outside the scope of this routing paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates routing algorithms on benchmark data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core results are accuracy-cost tradeoff curves (Figures 2-5) where the x-axis reports average inference cost per query in dollars. Cost is a primary evaluation dimension."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgements mention using 'PSC Bridges-2 GPU at Pittsburgh Supercomputing Center through allocation CIS250087' but do not quantify GPU hours, training time, or total compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from a single run of the Dirichlet partitioning and training procedure."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating how many times the experiments were repeated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are stated (Appendix C) but the search budget is not reported. Klocal and Kglobal are said to be 'chosen with validation experiments' but the search space and number of configurations tried are not disclosed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Klocal=15 and Kglobal=20 are 'chosen with validation experiments' but does not show the validation results or explain the selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is moot. The absence of statistical testing is captured by significance_tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All methods (federated, local, personalized) are implemented by the authors. No discussion of potential author-implementation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The compute cost of federated training (communication rounds, local training steps) vs local-only training is not compared. Federated training involves T=unspecified rounds with τ local steps each, adding significant overhead that is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether RouterBench-Data and ProxRouter-Data actually measure realistic routing scenarios. No analysis of whether synthetic query-model evaluation data reflects real-world routing needs."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper trains and evaluates routing algorithms directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The train/test split is random, not temporal. Whether earlier benchmark evaluations could inform later ones is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether query embeddings or evaluation features could leak information about the test set routing outcomes."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence between train and test examples. Queries from the same benchmark task may share structural similarities, but this is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method beyond standard random train/test splitting."
       }
     }
   },
   "claims": [
     {
       "claim": "Federated routers consistently improve the accuracy-cost frontier over client-local routers on the global test distribution.",
       "evidence": "Figure 2 shows federated AUC of 0.75 for both MLP and K-Means vs client-local AUC ranging 0.63-0.72 (MLP) and 0.55-0.70 (K-Means) on RouterBench-Data. Similar results on ProxRouter-Data (Figure 15).",
       "supported": "moderate"
     },
     {
       "claim": "Federated learning improves in-distribution local performance via better model coverage.",
       "evidence": "Figure 3 shows federated routers outperform client-local routers even on individual clients' local test sets. Figures 10-11 confirm this across all 10 clients with mean AUC improvement of 0.05 (MLP) and 0.11 (K-Means).",
       "supported": "moderate"
     },
     {
       "claim": "Federated training achieves performance comparable to centralized training.",
       "evidence": "Figure 9 shows identical AUC (0.75) for both federated and centralized training for both router types on RouterBench-Data. Figure 19 shows similar parity on ProxRouter-Data.",
       "supported": "moderate"
     },
     {
       "claim": "Adaptive personalization improves robustness under extreme heterogeneity.",
       "evidence": "Section 6.4 and Figure 5 show that under α=0.03, personalized routing generally matches or exceeds the better of federated and local routers. However, for K-Means, federation alone already remains competitive (Figures 13-14 in appendix show mean AUC: personalized 0.75 vs federated 0.75 for MLP).",
       "supported": "moderate"
     },
     {
       "claim": "Both router families support lightweight adaptation to new models without full retraining.",
       "evidence": "Figure 4 shows accuracy-cost frontiers improve after adding 3 withheld models via lightweight calibration (MLP AUC: 0.732→0.748, K-Means: 0.732→0.749).",
       "supported": "moderate"
     },
     {
       "claim": "Federated training reduces routing suboptimality compared to local-only training (theoretical).",
       "evidence": "Theorem 5.3 shows suboptimality scales as O(1/√D) for federated vs O(1/√Di) for local, and Theorem 5.5 provides analogous bounds for K-Means-Router. Full proofs in Appendix G.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All experimental results are reported as single-point estimates without variance, confidence intervals, or multiple-run statistics. Given that the Dirichlet partitioning is stochastic, different random seeds could produce different client data distributions and thus different results. The stability of the reported improvements is unknown."
     },
     {
       "flag": "Synthetic federated simulation",
       "detail": "The federated setting is simulated by Dirichlet-partitioning existing centralized benchmark data across 10 synthetic clients. Real federated data heterogeneity, communication constraints, and privacy requirements may differ substantially from this simulation. The paper does not validate against any real federated deployment."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section despite the 34-page length (including appendix). Key limitations like the synthetic federated setup, single-run results, and unknown generalization to real deployments are not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu", "W.-L. Chiang", "T. Wu", "J. E. Gonzalez", "M. W. Kadous", "I. Stoica"],
       "year": 2025,
       "relevance": "Core LLM routing method that trains routers from human preference data; direct comparison point for routing approaches."
     },
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "authors": ["Q. J. Hu", "J. Bieker", "X. Li", "N. Jiang", "B. Keigwin", "G. Ranganath", "K. Keutzer", "S. K. Upadhyay"],
       "year": 2024,
       "relevance": "Primary benchmark used in this paper; establishes the evaluation framework for LLM routing systems."
     },
     {
       "title": "Hybrid LLM: Cost-efficient and quality-aware query routing",
       "authors": ["D. Ding", "A. Mallick", "C. Wang", "R. Sim", "S. Mukherjee", "V. Rühle", "L. V. S. Lakshmanan", "A. H. Awadallah"],
       "year": 2024,
       "relevance": "Parametric LLM routing approach for cost-quality tradeoffs; foundational work in the routing space."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2024,
       "relevance": "Pioneering work on reducing LLM inference costs through model selection and cascading strategies."
     },
     {
       "title": "GraphRouter: A graph-based router for LLM selections",
       "authors": ["T. Feng", "Y. Shen", "J. You"],
       "year": 2025,
       "relevance": "Graph-based approach to LLM routing that models relationships between models and queries."
     },
     {
       "title": "EmbedLLM: Learning compact representations of large language models",
       "authors": ["R. Zhuang", "T. Wu", "Z. Wen", "A. Li", "J. Jiao", "K. Ramchandran"],
       "year": 2025,
       "relevance": "Learns LLM embeddings for model comparison and selection; relevant to router design."
     },
     {
       "title": "Smoothie: Label free language model routing",
       "authors": ["N. Guha", "M. F. Chen", "T. Chow", "I. S. Khare", "C. Re"],
       "year": 2024,
       "relevance": "Label-free LLM routing approach using weak supervision; alternative paradigm for router training."
     },
     {
       "title": "Eagle: Efficient training-free router for multi-LLM inference",
       "authors": ["Z. Zhao", "S. Jin", "Z. M. Mao"],
       "year": 2024,
       "arxiv_id": "2409.15518",
       "relevance": "Training-free multi-LLM routing using Elo-based model comparison scores."
     },
     {
       "title": "ProxRouter: Proximity-weighted LLM query routing for improved robustness to outliers",
       "authors": ["S. Patel", "N. Jali", "A. Mallick", "G. Joshi"],
       "year": 2025,
       "arxiv_id": "2510.09852",
       "relevance": "Nonparametric proximity-weighted routing approach; second benchmark dataset used in this paper."
     },
     {
       "title": "BEST-route: Adaptive LLM routing with test-time optimal compute",
       "authors": ["D. Ding", "A. Mallick", "S. Zhang", "C. Wang", "D. Madrigal"],
       "year": 2025,
       "relevance": "Adaptive LLM routing that optimizes compute at test time; advances the parametric routing paradigm."
     },
     {
       "title": "Universal LLM routing with correctness-based representation",
       "authors": ["W. Jitkrittum", "H. Narasimhan", "A. S. Rawat", "J. Juneja", "Z. Wang"],
       "year": 2025,
       "relevance": "Nonparametric routing using correctness-based representations; key comparison point for K-Means routing."
     },
     {
       "title": "Communication-efficient learning of deep networks from decentralized data",
       "authors": ["B. McMahan", "E. Moore", "D. Ramage", "S. Hampson", "B. A. y Arcas"],
       "year": 2017,
       "relevance": "Foundational FedAvg algorithm used as the basis for the federated MLP-Router training in this paper."
     }
   ]
 }