ai-research-survey

scan-v4.json (19691B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Towards Fair and Comprehensive Evaluation of Routers in Collaborative LLM Systems",
     "authors": [
       "Wanxing Wu",
       "He Zhu",
       "Yixia Li",
       "Lei Yang",
       "Jiehui Zhao",
       "Hongru Wang",
       "Jian Yang",
       "Benyou Wang",
       "Bingyi Jing",
       "Guanhua Chen"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11877",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '16.68% and 18.86% relative improvements' are supported by Tables 1 and 2. Claims of consistency across model families supported by Table 5. Agent scenario claim supported by Figure 5.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('data diversity yields additive gains,' 'Dirichlet aggregation prevents overfitting') are supported by controlled ablation studies. Table 6 varies training data composition while holding architecture fixed. Table 3 varies aggregation while holding data fixed. Single-variable manipulation is adequate.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Collaborative LLM Systems' but experiments test only two-model edge-cloud routing with one primary model pair (GPT-5 + Llama-3.1-8B). The Limitations section acknowledges 'single small-large model pair' but the title and framing remain broader than the evidence.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses why hidden states outperform embeddings (hierarchical vs surface-level information), why linear probes suffice (hidden states encode shared difficulty notion), and alternative data composition effects (interference vs additive gains, Table 6).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates router ability (AUROC, intrinsic discrimination) from end-to-end system accuracy, noting that 'end-to-end accuracy at a given cost reflects both' router skill and model strength (Section 3.2). The framework is designed to disentangle proxy from outcome.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing the assumption that the large model is always better, single model pair constraint, and single-run results.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: 'both models may perform similarly or converge on the same incorrect answer in certain domains' (with analysis in Appendix D.2), 'single small-large model pair,' and 'single-run results due to computational constraints.' These are specific to this study.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "States what was not tested: 'broader validation across diverse architectures, multiple seeds, and more complex OOD conditions would further strengthen the conclusions.' Also notes the assumption that large model capability exceeds small model's.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper. Authors include one from Deepexi Technology Co. Ltd. (industry) and several from universities.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Southern University of Science and Technology, Institut Polytechnique de Paris, Peking University, Deepexi Technology Co. Ltd., University of Edinburgh, Beihang University, Chinese University of Hong Kong (Shenzhen).",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. One co-author is from Deepexi Technology, an industry entity whose interest in the outcome is unknown.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 formally defines the routing problem with equations covering router score, decision threshold, call rate, and system performance. AUROC, LPM, HCR, and MPM are all defined with explicit mathematical expressions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Two contributions are clearly stated throughout: RouterXBench evaluation framework (three-dimensional) and ProbeDirichlet router (hidden-state based with Dirichlet aggregation), each described in dedicated sections.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Section 3.2 explicitly critique prior work (FrugalGPT, HybridLLM, RouterBench, RouteLLM, AutoMix) by demonstrating specific limitations of their metrics with Figure 1, not merely listing references.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 explicitly argues that AUROC isolates router discriminative ability from large model capability, and Section 3.2 argues why existing metrics conflate these — a specific, formal construct validity argument with mathematical support.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper characterizes difficulty only at the dataset level (Alpaca=easy to MATH=hard) but does not formally characterize the difficulty distribution within benchmark items or provide easy/medium/hard tier breakdowns.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "Baseline AUROC scores near 50% (SelfAsk at 49.99 in-domain average) suggest floor effects for weaker methods, but ceiling/floor effects are never explicitly checked or discussed as a benchmark design property.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is provided anywhere in the paper; the benchmark evaluates automated routing decisions without any human reference point for routing quality.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 provides detailed justification for AUROC over static and curve-based metrics. Section 3.3 defines band parameters for LPM (25-30% call rate) and HCR (85-95% relative performance) with deployment scenario rationale, though specific cutoff values are asserted rather than empirically motivated.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark uses widely known public datasets (MMLU, Alpaca, Big-Math) with no temporal splits, canary strings, or dynamic generation to prevent contamination of router training data from evaluation sets.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether RouterXBench will remain discriminative as models improve, whether chosen benchmarks will be saturated, or any plan for updating the evaluation suite over time.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 discusses failure modes of the metric design (threshold sensitivity, AUC scenario-blindness). Appendix D.2 provides a concrete case study where routing fundamentally fails because both models converge on the same wrong answer.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Footnote 1 states 'Our code is publicly available at https://github.com/zhuchichi56/RouterXBench,' enabling reproduction of all reported baselines and the ProbeDirichlet router.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B documents all six datasets with Table 7 statistics (domain, train/val/test sizes), ground truth construction methodology (xVerify for reasoning tasks, GPT-5-as-judge for open-ended), and evaluation protocols.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "While code is available on GitHub, the paper states no license for RouterXBench and does not reference the licenses of underlying benchmark datasets (MMLU, Alpaca, etc.) or clarify usage terms.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The paper specifies intended use as evaluating LLM routers in edge-cloud collaborative settings under three deployment scenarios (budget-sensitive, balanced, accuracy-critical), with clear definition of what each metric captures and does not capture.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "ProbeDirichlet achieves 16.68% relative improvement over best baseline in router ability (AUROC) on average across in-domain and OOD benchmarks",
       "evidence": "Table 1 shows ProbeDirichlet in-domain average AUROC 68.70 vs EmbeddingMLP 59.46 (~15.5% relative), OOD 65.46 vs 55.22 (~18.5% relative); paper reports the average as 16.68%",
       "supported": "strong"
     },
     {
       "claim": "ProbeDirichlet achieves 18.86% relative improvement over best baseline in accuracy-critical (HCR) scenarios",
       "evidence": "Table 2 HCR section: ProbeDirichlet in-domain 18.50 vs SemanticEntropy 15.17 (~21.9%); OOD 15.40 vs 13.35 (~15.4%); reported average improvement 18.86%",
       "supported": "moderate"
     },
     {
       "claim": "Internal hidden states significantly outperform output probabilities and external embeddings for routing decisions",
       "evidence": "Table 4 shows LLM Hidden (71.34 Alpaca, 62.39 BigMath) vs LLM Embedding (62.47, 56.21) vs Longformer (61.95, 43.10) with consistent, substantial margins especially for math tasks",
       "supported": "strong"
     },
     {
       "claim": "Data diversity yields additive gains without domain interference in cross-domain generalization",
       "evidence": "Table 6: adding BigMath to Alpaca training preserves Alpaca AUROC (71.85→71.63) while dramatically improving BigMath (49.19→66.49) and MMLU (49.35→51.06); full mix shows further gains",
       "supported": "strong"
     },
     {
       "claim": "Linear probes suffice for routing; MLP layers do not improve performance and increase overfitting",
       "evidence": "Figure 3 shows MLP variants with 16-128 hidden dimensions match or fall below linear probe AUROC (~78%) while widening train-validation loss gaps, with dropout only partially mitigating overfitting",
       "supported": "strong"
     },
     {
       "claim": "Existing routing metrics (static thresholds and AUC) are insufficient for fair evaluation due to threshold sensitivity and scenario blindness",
       "evidence": "Figure 1 illustrative example shows reversed router rankings in call-rate range 20-40%, motivating the triple-perspective framework — but this is a constructed illustration, not an empirical study of evaluation failures",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "RouterXBench proposes a three-dimensional evaluation framework for LLM routers—router ability (AUROC), scenario alignment (LPM/MPM/HCR), and cross-domain robustness—that disentangles intrinsic routing capability from large model performance, addressing fundamental limitations in existing single-metric evaluations. The ProbeDirichlet router, trained on multi-domain data using internal hidden states with Dirichlet layer aggregation, achieves 16.68% and 18.86% relative improvements over best baselines in router ability and accuracy-critical scenarios. Ablation analyses show that the key driver of performance is using internal hidden states rather than output probabilities or external embeddings, while data diversity is the primary driver of cross-domain generalization rather than architectural complexity. The Dirichlet aggregation itself contributes only marginally over simple mean pooling (68.70 vs 68.04 AUROC), with most gains attributable to the hidden-state signal choice rather than the architectural innovation.",
   "red_flags": [
     {
       "flag": "Single-run results throughout",
       "detail": "Appendix A states all experiments are conducted with a fixed seed and single run due to computational constraints; no variance estimates or confidence intervals are provided for any comparison, including the marginal Dirichlet vs. MeanPool gap (68.70 vs 68.04 AUROC)."
     },
     {
       "flag": "GPT-5 serves as both evaluated model and judge",
       "detail": "GPT-5 is simultaneously the large model being routed to AND the LLM-as-Judge evaluator for ground truth construction on open-ended generation tasks (Appendix B.1), creating circular evaluation where the oracle model defines its own routing ground truth."
     },
     {
       "flag": "Dirichlet contribution marginal relative to presented novelty",
       "detail": "Table 3 shows the core architectural innovation (Dirichlet vs. MeanPool aggregation) yields only 0.66 AUROC points (68.70 vs 68.04). The main gains come from using hidden states at all, undercutting the paper's framing of Dirichlet as a key contribution."
     },
     {
       "flag": "No funding disclosure with commercial co-author",
       "detail": "No funding acknowledgment despite two co-authors from Deepexi Technology Co. Ltd. and no competing interests statement anywhere in the paper."
     },
     {
       "flag": "No contamination resistance for public benchmark datasets",
       "detail": "RouterXBench uses MMLU, Alpaca, and Big-Math as both training and evaluation data with no measures to prevent router training data from overlapping with publicly known evaluation sets."
     },
     {
       "flag": "No human baseline",
       "detail": "No human performance baseline is provided for any of the routing tasks, making it impossible to assess whether AUROC differences are practically meaningful or how much headroom exists."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "relevance": "Direct predecessor benchmark using preference-based routing and curve-based AUC metric explicitly criticized by this paper"
     },
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "relevance": "Prior multi-LLM routing benchmark (ICML 2024 workshop), direct comparison context for RouterXBench"
     },
     {
       "title": "RouterEval: A comprehensive benchmark for routing LLMs to explore model-level scaling up",
       "relevance": "Concurrent comprehensive routing benchmark (EMNLP 2025 Findings) cited as prior work that RouterXBench extends"
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "relevance": "Key baseline paper for cost-efficient LLM routing using static threshold metrics; paradigmatic example of the limitations RouterXBench addresses"
     },
     {
       "title": "Hybrid LLM: Cost-efficient and quality-aware query routing",
       "relevance": "Direct experimental baseline; representative logit-based and cost-quality routing approach"
     },
     {
       "title": "AutoMix: Automatically mixing language models",
       "relevance": "Training-free routing baseline using Incremental Benefit per Cost single metric; motivates multi-dimensional evaluation"
     },
     {
       "title": "Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation",
       "relevance": "Provides SemanticEntropy baseline method for routing; represents uncertainty-based routing signal category"
     },
     {
       "title": "INSIDE: LLMs' internal states retain the power of hallucination detection",
       "relevance": "Prior work using LLM internal hidden states for quality assessment, directly motivates the hidden-state routing approach in ProbeDirichlet"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Cost-accuracy routing is a live operational concern for any organization deploying LLMs; code is publicly available and the framework directly guides deployment decisions."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Internal hidden states outperforming output logits aligns with growing literature; the multi-dimensional metric framework is incremental refinement, not a paradigm shift."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Healthcare reliability is cited as a motivating scenario but not experimentally studied; routing vulnerability is cited but not addressed."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward systems and evaluation paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly available on GitHub with the RouterXBench framework; practitioners could apply ProbeDirichlet to their own Llama or Qwen deployments."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Primarily academic affiliations (SUSTech, Peking University, CUHK-Shenzhen, Beihang); Deepexi Technology is not widely recognized; no major AI lab branding."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }