ai-research-survey

scan-v5.json (18299B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Towards Fair and Comprehensive Evaluation of Routers in Collaborative LLM Systems",
     "authors": [
       "Wanxin Wu",
       "He Zhu",
       "Yixia Li",
       "Lei Yang",
       "Jie Zhao"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11877",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The 16.68% and 18.86% relative improvement claims are supported by Tables 1 and 2; generalization across model families is supported by Table 5 and Figure 5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'diverse training improves robustness' and 'hidden states outperform output-based signals' are supported by controlled ablation studies (Table 3, Table 4, Table 6) that vary one factor at a time.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'fair and comprehensive evaluation' but the main experiments use a single small-large model pair (Llama-3.1-8B + GPT-5); the limitations section acknowledges this but the abstract still claims generality across diverse settings.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes hidden-state superiority solely to capturing pre-generation uncertainty, but does not discuss alternative explanations such as the advantage being due to model-specific probing rather than a general principle.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses the conflation problem — separating intrinsic router ability (AUROC) from end-to-end system performance, which is the core methodological contribution of the framework.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears before the References, discussing the single model-pair constraint and single-run reporting.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats noted include single small-large model pair, single random seed (seed=42), single-run results due to compute, and the model-convergence failure mode analyzed in Appendix D.2.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states conclusions are bounded to a single model pair and warns that 'broader validation across diverse architectures, multiple seeds, and more complex OOD conditions would further strengthen the conclusions.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section appears anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header: Southern University of Science and Technology, Institut Polytechnique de Paris, Peking University, Deepexi Technology Co. Ltd., University of Edinburgh, Beihang University, and Chinese University of Hong Kong.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed, making this criterion not applicable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined with mathematical precision: router, AUROC, LPM, MPM, HCR, in-distribution, out-of-distribution, and the edge-cloud collaboration setting are all explicitly defined in Section 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states two contributions: (1) RouterXBench evaluation framework with three-dimensional metrics, and (2) ProbeDirichlet router using internal hidden states with Dirichlet aggregation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The related work section covers LLM routing, LLM collaboration, and uncertainty estimation, and Section 3.2 specifically analyzes limitations of prior metrics (FrugalGPT, HybridLLM, RouteLLM) as motivation.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly argues that AUROC measures routing ability independent of the large model's strength, and that LPM/MPM/HCR measure scenario-specific alignment — addressing the conflation problem in existing metrics with mathematical formalization.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions a 'difficulty gradient' conceptually ('simpler benchmarks such as Alpaca, Magpie, to more challenging ones like MMLU, Big-Math, and MATH') but provides no quantitative characterization of difficulty distribution within or across benchmarks.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not explicitly check for ceiling or floor effects; AUROC values ranging from ~47% to ~74% suggest adequate discrimination, but this is never explicitly verified or discussed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is included or discussed for any of the six benchmarks used in the evaluation framework.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "AUROC is well-justified as threshold-independent, but the specific thresholds for scenario alignment (25-30% call rate for LPM, 85-95% relative performance for HCR) are stated as deployment scenarios without principled empirical or domain-specific justification.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "RouterXBench builds on existing public benchmarks (MMLU, Alpaca, etc.) without any contamination resistance measures such as temporal splits, canary strings, or dynamic generation.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether the benchmark will remain useful as models improve or whether existing benchmarks will be saturated, nor is there a plan for updates.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.2 provides a case study where routing fails because both small and large models converge on the same wrong answer, and the limitations section discusses this as a fundamental gap in routing frameworks.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Code is publicly available at https://github.com/zhuchichi56/RouterXBench, and Appendix A and B provide implementation details including fixed random seed, training setup, and data preparation.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "Table 7 provides basic statistics for the six datasets, and Appendix B describes data preparation, but there is no formal data card; ground-truth construction methodology using xVerify and GPT-5-as-Judge is described but not validated.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub link is provided for the code but no license is specified; the underlying datasets are public but their licenses are not discussed in the context of the framework.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The three-dimensional evaluation structure (router ability vs. scenario alignment vs. cross-domain robustness) makes clear what should and should not be concluded from each metric, and the limitations section specifies what the benchmark does not cover.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "ProbeDirichlet achieves 16.68% relative improvement over the best baseline in router ability (AUROC) on in-domain and OOD scenarios.",
       "evidence": "Table 1 shows ProbeDirichlet AUROC averages of 68.70 (in-domain) and 65.46 (OOD) vs. EmbeddingMLP at 59.46 and 55.22.",
       "supported": "strong"
     },
     {
       "claim": "ProbeDirichlet achieves 18.86% relative improvement in high-accuracy (HCR) scenarios over the best baseline.",
       "evidence": "Table 2 HCR rows show ProbeDirichlet averages of 18.50 (in-domain) and 15.40 (OOD) vs. SemanticEntropy at 15.17 and 13.35.",
       "supported": "strong"
     },
     {
       "claim": "Internal hidden states outperform both output-based (logit/verbose) and external embedding-based routing signals.",
       "evidence": "Table 4 directly compares Longformer embeddings, LLM embeddings, and LLM hidden states using identical linear classifiers, showing hidden states outperform by large margins particularly on math tasks.",
       "supported": "strong"
     },
     {
       "claim": "Diverse multi-domain training yields additive gains without interference between domains.",
       "evidence": "Table 6 shows that adding BigMath training preserves Alpaca performance (71.85→71.96) while improving BigMath (49.19→66.49) and OOD tasks.",
       "supported": "strong"
     },
     {
       "claim": "Linear probe architecture is sufficient; adding hidden layers degrades generalization without improving performance.",
       "evidence": "Figure 3 shows MLP variants with 16-128 hidden dimensions do not improve AUROC over the linear baseline but exhibit larger train-validation loss gaps.",
       "supported": "moderate"
     },
     {
       "claim": "Existing routing metrics (static and curve-based) are inadequate for fair comparison because they conflate router ability with large model strength.",
       "evidence": "Figure 1 (right) illustrates that router rankings reverse under small threshold shifts, but this illustration uses a stylized example rather than real experimental data.",
       "supported": "moderate"
     },
     {
       "claim": "ProbeDirichlet generalizes across Llama and Qwen model families with consistent improvements over EmbeddingMLP baselines.",
       "evidence": "Table 5 shows ProbeDirichlet outperforms EmbeddingMLP across Llama-3.1-8B and Qwen2.5 (0.5B, 3B, 7B), with average improvements of ~10.5% in-domain and ~9.6% OOD.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "RouterXBench proposes a three-dimensional evaluation framework (router ability via AUROC, scenario alignment via LPM/MPM/HCR, cross-domain robustness) that disentangles intrinsic routing ability from end-to-end system performance, exposing limitations in prior single-metric evaluations. ProbeDirichlet, a lightweight router using internal hidden-state representations aggregated via Dirichlet distributions, achieves 16.68% and 18.86% relative improvements over the best baselines in router ability and high-accuracy scenarios. The primary driver of generalization is training data diversity rather than architectural complexity: diverse multi-domain training yields additive gains across domains without interference. Single-run experiments on one small-large model pair (Llama-3.1-8B + GPT-5) limit the strength of these conclusions.",
   "red_flags": [
     {
       "flag": "single model pair",
       "detail": "All main experiments use only Llama-3.1-8B as the small model and GPT-5 as the large model; the framework's claims about generality rest on a single routing configuration."
     },
     {
       "flag": "single run, no variance",
       "detail": "Appendix A explicitly states 'we report single-run results for all experiments' with a fixed seed=42, providing no statistical significance estimates or confidence intervals for any reported improvement."
     },
     {
       "flag": "circular evaluation: GPT-5 as both large model and judge",
       "detail": "GPT-5 serves as the large model being routed to AND as the LLM-as-a-Judge evaluator for open-ended tasks (Alpaca, Magpie). Footnote 3 acknowledges this but dismisses it, creating potential circular bias in ground-truth label construction."
     },
     {
       "flag": "arbitrary scenario thresholds",
       "detail": "The thresholds defining scenario alignment (25-30% call rate for LPM, 85-95% relative performance for HCR) are presented as deployment scenarios but are chosen without empirical or domain-specific justification."
     },
     {
       "flag": "no license for released code",
       "detail": "The GitHub repository is mentioned but no software license is specified, limiting the clarity of reuse rights."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "relevance": "Foundational paper on cost-aware LLM routing; RouterXBench's framework directly addresses limitations of FrugalGPT's fixed-accuracy metric."
     },
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "relevance": "State-of-the-art preference-based router and evaluation baseline; introduces curve-based AUC metric critiqued by this paper."
     },
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "relevance": "Direct predecessor benchmark for LLM routing evaluation; RouterXBench positions itself as more comprehensive."
     },
     {
       "title": "RouterEval: A comprehensive benchmark for routing LLMs",
       "relevance": "Contemporary routing benchmark; cited as related benchmarking effort that RouterXBench extends."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "relevance": "Key baseline routing system using fixed-cost metric; represents the static metric paradigm critiqued here."
     },
     {
       "title": "AutoMix: Automatically Mixing Language Models",
       "relevance": "Routing baseline using Incremental Benefit per Cost as a single-score metric; represents the static metric approach."
     },
     {
       "title": "Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation",
       "relevance": "Provides the SemanticEntropy baseline compared against ProbeDirichlet throughout the experiments."
     },
     {
       "title": "MMLU: Measuring massive multitask language understanding",
       "relevance": "Core in-domain benchmark used for both training and evaluation in the RouterXBench framework."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "LLM routing for edge-cloud cost reduction is an immediate deployment concern for practitioners; the framework directly guides router selection in production systems."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that data diversity matters more than architecture complexity (linear probe suffices) and that existing evaluation metrics produce misleading rankings challenges common assumptions."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Mentions safety-critical applications (healthcare) as a motivating scenario for high-accuracy routing, but does not raise broader AI risk concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Positions against existing evaluation frameworks (FrugalGPT, RouteLLM) but the critique is methodological rather than confrontational."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly available on GitHub with a fixed seed and documented setup, making reproduction straightforward for practitioners with the required models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "No top-tier lab affiliation; uses GPT-5 (OpenAI) as large model which adds some recognition, but the contributing institutions are not widely recognized in this context."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }