ai-research-survey

scan-v5.json (25586B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Federate the Router: Learning Language Model Routers with Sparse and Decentralized Evaluations",
     "authors": [
       "Baris Askin",
       "Shivam Patel",
       "Anupam Nayak",
       "Andrea Vigano",
       "Jiin Woo"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.22318",
     "doi": "10.48550/arXiv.2601.22318"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by experimental results: federated improvement over client-local routers (Figures 2–3), accuracy-cost frontier gains (AUC comparisons across all clients), and theoretical suboptimality bounds (Section 5 and Appendix G).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('federated training improves...') are supported by controlled comparisons holding all conditions constant except federation, evaluated on held-out test sets across two independent benchmarks.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states federated learning 'offers a practical foundation for training LLM routers from privacy-sensitive, fragmented data,' but all experiments use a simulated N=10 client setup with artificial Dirichlet partitioning—no discussion of the simulation-to-deployment gap.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether simple data pooling (rather than the federated algorithm itself) could explain the gains; Appendix D.1 shows federated ≈ centralized but does not analyze whether dataset size alone drives improvements.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures routing accuracy (model correctness on actual queries) and inference cost, which are precisely what is claimed—no proxy conflation.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion mentions only future work (online routing) but does not systematically address limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats are discussed—e.g., that Dirichlet partitioning may not reflect real heterogeneity, that the uniform logging assumption required for K-Means theory is violated in experiments, or that N=10 may not scale.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries state what the results do NOT show—e.g., behavior with hundreds of clients, real privacy attacks, or non-stationary query distributions.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements explicitly list US DOE grant DESC0025652, NSF grants CNS-2409138, CNS-2533813, CCF 2045694, CNS-2112471, CPS-2111751, ONR N00014-23-1-2149, and the AI2C Seed grant.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Carnegie Mellon University affiliates in the author block.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "All funding sources are US government agencies (DOE, NSF, ONR) and an academic seed grant, independent of commercial LLM routing outcomes.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are formally defined: 'routing policy' (Section 3, Eq. 4), 'utility' (acc(x,m) − λ·cost(x,m), Eq. 1), 'suboptimality' (Definition 5.2), and the federated data model (Eq. 2).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit bullet-point contributions are listed at the end of Section 1: problem formulation, federated training procedures for both router families, theoretical guarantees, and empirical evaluation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Appendix A provide detailed engagement with both FL and LLM routing literature, explicitly positioning this as the first FL-routing combination and distinguishing from cascading, speculative decoding, and bandit-based approaches.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository is linked or mentioned anywhere in the paper.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses RouterBench-Data (Hu et al., 2024) and ProxRouter-Data (Patel et al., 2025), both publicly available prior benchmarks used unmodified.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or software environment specification is provided; only the MLP architecture and optimizer hyperparameters are described.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Appendix C provides experimental details but no step-by-step reproduction instructions; key implementation decisions (e.g., federated simulation code, embedding pipeline) are not specified.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All AUC scores and accuracy-cost curves are reported as single point estimates with no confidence intervals, error bars, or standard deviations across runs.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used for any comparative claims; improvements are reported as raw AUC differences without p-values or hypothesis tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "AUC differences are reported in absolute terms throughout (e.g., federated 0.75 vs. client-local mean 0.69 for MLP on RouterBench), providing effect size context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of N=10 clients and 75/25 train-test split is not justified with any power analysis or sensitivity study.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance across random seeds or Dirichlet partition realizations is reported; all results appear to be single-run point estimates.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Client-local (no-FL) routers serve as the primary baseline; centralized training (oracle with full data pooled) is compared in Appendix D.1.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Client-local training is the natural competitive alternative, and centralized training is the appropriate oracle upper bound; both are competitive and contemporarily relevant.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The paper ablates adaptive personalization (Section 6.4), different sentence encoder models (Appendix E), and new-model/new-client extension scenarios (Sections 6.3, D.3).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both accuracy and inference cost are measured as primary metrics, reported jointly as accuracy-cost frontier curves with normalized AUC as a scalar summary.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "This is a routing systems paper; human evaluation of model outputs is not relevant to routing quality assessment.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Each client uses a 75/25 train-test split; global test set is the union of all client test splits (Appendix C).",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Per-client breakdowns are provided for all 10 clients in Figures 10–11 (RouterBench) and Figures 17–18 (ProxRouter).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.4 explicitly shows cases where federated MLP-Router underperforms client-local routers under extreme heterogeneity (α=0.03), e.g., Client 6 (federated 0.71 < local 0.73), motivating adaptive personalization.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper directly reports that federated training can hurt some clients under high heterogeneity, and Figure 5 shows specific cases where client-local outperforms federated.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "RouterBench-Data includes evaluations on specifically versioned models (GPT-3.5 Turbo 1106, GPT-4 1106 Preview, Claude v1/v2/Instant, Llama 2 70B Chat, etc.) shown in Figure 8; sentence encoder all-mpnet-base-v2 is also specified.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "No new LLM querying occurs during experiments; the paper uses pre-existing evaluation datasets from RouterBench and ProxRouter, so no prompts are generated.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C reports AdamW lr=10^-3, weight decay=3×10^-4, batch size=128, gradient clip norm=1.0, Klocal=15, Kglobal=20, 3 K-means restarts, participation rate=0.6, and λ sweep grid.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "FedAvg (Algorithm 1) and federated K-Means (Algorithm 2) are described step-by-step with full pseudocode covering all communication rounds, local update steps, and aggregation.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Dirichlet partitioning parameters (α=0.6 main, α=0.03 extreme, α=0.45 model assignment), 75/25 split, and the federated simulation protocol are documented in Appendices B and C.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "RouterBench-Data and ProxRouter-Data are publicly available benchmarks with references; raw evaluation data can be obtained from the original sources.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The federated simulation protocol is described in detail in Section 6 and Appendices B–C, including Dirichlet partitioning and model assignment procedures.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data comes from existing NLP evaluation benchmarks.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from benchmark data → Dirichlet client partitioning → federated training (Algorithms 1–2) → evaluation is documented across Section 6 and Appendices B–C.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper trains routing models (MLPs and K-Means) over pre-existing evaluation data, not LLM capabilities—contamination of the routing model itself is not a meaningful concern.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "N/A—the routing model is not an LLM evaluated on downstream benchmarks.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "N/A—the routing model is not an LLM; contamination of underlying LLM evaluations in RouterBench is out of scope.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Inference cost is one of the two primary metrics throughout all experiments; accuracy-cost frontier curves explicitly report average API cost per query in dollars.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "The acknowledgements mention PSC Bridges-2 GPU via ACCESS allocation CIS250087 but do not report total GPU-hours or compute cost for the experiments.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Federated training consistently improves the global accuracy-cost frontier over all client-local routers for both MLP and K-Means families.",
       "evidence": "Figure 2: federated AUC 0.75 vs. client-local range 0.63–0.72 (MLP) and 0.75 vs. 0.55–0.70 (K-Means) on RouterBench-Data; replicated on ProxRouter-Data (Figure 15).",
       "supported": "strong"
     },
     {
       "claim": "Federated learning improves in-distribution local performance via better effective model coverage under sparse, imbalanced evaluations.",
       "evidence": "Figures 3 and 10–11: all 10 clients show improved local-test AUC; mean AUC improves 0.69→0.74 (MLP) and 0.64→0.75 (K-Means) across clients.",
       "supported": "strong"
     },
     {
       "claim": "Federated routers match centralized (oracle) training performance despite operating under decentralized data constraints.",
       "evidence": "Figure 9: MLP-Federated AUC 0.75 = MLP-Centralized 0.75; K-Means-Federated 0.75 = K-Means-Centralized 0.75 on RouterBench-Data.",
       "supported": "strong"
     },
     {
       "claim": "Adaptive personalization improves robustness under extreme client heterogeneity (α=0.03) where global federated routing underperforms some clients.",
       "evidence": "Figure 5: personalized MLP improves over federated for Clients 4 (0.72 vs 0.69) and 6 (0.72 vs 0.71); mean personalized AUC 0.75 = federated 0.75 across all clients (Figure 13).",
       "supported": "moderate"
     },
     {
       "claim": "Both router types support lightweight extension to new models and new clients without full retraining.",
       "evidence": "Figure 4: adding 3 withheld models improves MLP AUC 0.732→0.748 and K-Means 0.732→0.749; Figure 12 shows similar gains after 3 new clients join.",
       "supported": "moderate"
     },
     {
       "claim": "Theoretical analysis proves federated routing reduces suboptimality from O(1/√Di) (local) to O(1/√D) (federated) where D >> Di.",
       "evidence": "Theorems 5.3 and 5.5 bound suboptimality with formal proofs in Appendix G under realizability, smoothness, and bounded heterogeneity assumptions.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "This paper introduces the first federated learning framework for LLM query routing, enabling decentralized clients to collaboratively train routing policies over private, heterogeneous query-model evaluation data without centralizing raw queries. Federated training achieves the accuracy-cost frontier of centralized (oracle) training while consistently outperforming all client-local baselines on RouterBench-Data and ProxRouter-Data across both MLP and K-Means router families. An adaptive personalization mechanism that interpolates between federated and local routers based on calibration error provides robustness when global collaboration misaligns with individual client distributions under extreme heterogeneity. Formal suboptimality bounds (Theorems 5.3, 5.5) confirm that federated aggregation reduces routing error at rate O(1/√D) versus O(1/√Di) for local training, with the gain proportional to the degree of complementary model coverage across clients.",
   "red_flags": [
     {
       "flag": "Simulated federation only",
       "detail": "The federated setting is entirely simulated via Dirichlet partitioning of existing benchmark datasets with N=10 clients; no real federated deployment is tested, leaving a material gap between simulation and practice that is never discussed."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims are made without confidence intervals, error bars, or p-values. AUC differences of 0.01–0.05 are treated as meaningful improvements without accounting for run-to-run variance."
     },
     {
       "flag": "Single-run results",
       "detail": "No variance across random seeds or Dirichlet partition realizations is reported; all results depend on a single experimental run, making stability of findings unclear."
     },
     {
       "flag": "Theory-experiment assumption mismatch",
       "detail": "The K-Means suboptimality bound (Theorem 5.5) requires uniform model logging (Assumption G.14), but experiments explicitly use highly non-uniform Dirichlet model assignment (α=0.45). The theoretical guarantee does not cover the actual experimental setting."
     },
     {
       "flag": "No limitations section",
       "detail": "No dedicated limitations or threats-to-validity section exists; the paper omits discussion of scalability beyond 10 clients, the simulation-reality gap, non-stationary query distributions, or communication overhead in real deployments."
     }
   ],
   "cited_papers": [
     {
       "title": "Communication-Efficient Learning of Deep Networks from Decentralized Data (FedAvg)",
       "relevance": "Foundational federated learning algorithm (McMahan et al., 2017) that this paper directly adapts as Algorithm 1 for federated MLP-Router training."
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "relevance": "Primary evaluation dataset providing query-model accuracy and cost evaluations for 11 LLMs across 8 datasets; the main experimental benchmark throughout the paper."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "relevance": "Key prior work on learning LLM routers from evaluation data; representative of the centralized routing paradigm this federated framework is designed to complement."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "relevance": "Related parametric routing approach; cited as prior centralized router work that assumes access to centralized evaluation data."
     },
     {
       "title": "ProxRouter: Proximity-Weighted LLM Query Routing for Improved Robustness to Outliers",
       "relevance": "Provides the second evaluation benchmark (ProxRouter-Data) with 14 LLMs over 10 datasets used for cross-benchmark validation."
     },
     {
       "title": "Universal LLM Routing with Correctness-Based Representation",
       "relevance": "Related nonparametric routing approach; cited as prior K-Means-style routing that this paper extends to the federated setting."
     },
     {
       "title": "Advances and Open Problems in Federated Learning",
       "relevance": "Comprehensive survey (Kairouz et al., 2021) cited to frame statistical heterogeneity and open challenges that motivate this work's federated formulation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses a real enterprise/edge deployment problem: privacy-preserving LLM routing where organizations cannot share sensitive query data centrally."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Applying federated learning to LLM routing is novel, but the result that FL helps when data is fragmented is expected; no surprising reversals or counterintuitive findings."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Privacy protection is a core motivation (client queries remain local), touching on data safety concerns, but no AI safety or broader risk framing."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward systems/theory paper with no controversy or conflict with prior work claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is not released, limiting immediate demonstrability; RouterBench data is public and the algorithm is well-specified, but reproduction requires non-trivial implementation effort."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Carnegie Mellon University is well-regarded for ML systems research, but this is not from a major AI lab (Google, OpenAI, Meta, etc.)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46925532",
         "title": "Convergent Discovery of Critical Phenomena Mathematics Across Disciplines",
         "points": 4,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=46925532",
         "created_at": "2026-02-07T17:16:44Z"
       }
     ],
     "top_points": 4,
     "total_points": 4,
     "total_comments": 3
   }
 }