ai-research-survey

scan.json (27759B)

---

 {
   "paper": {
     "title": "Towards Fair and Comprehensive Evaluation of Routers in Collaborative LLM Systems",
     "authors": [
       "Wanxing Wu",
       "He Zhu",
       "Yixia Li",
       "Lei Yang",
       "Jiehui Zhao",
       "Hongru Wang",
       "Jian Yang",
       "Benyou Wang",
       "Bingyi Jing",
       "Guanhua Chen"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11877"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces RouterXBench, a three-dimensional evaluation framework (router ability, scenario alignment, cross-domain robustness) that disentangles intrinsic routing quality from model performance. ProbeDirichlet, a lightweight linear router using internal hidden states with Dirichlet-distributed layer weighting, achieves 16.68% and 18.86% relative improvements over best baselines on AUROC and HCR respectively. Analysis shows that signal provenance (internal hidden states vs. output logits or embeddings) matters more than aggregation architecture, and that training data diversity yields additive cross-domain gains without interference.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: 'Our code is publicly available at https://github.com/zhuchichi56/RouterXBench' (Section 1, footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six benchmarks used (Alpaca, MMLU, Big-Math, Magpie, MMLU Pro, MATH) are publicly available datasets. Table 7 lists them with references."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version listing found in the paper. Only model hidden state dimension (4096) and seed are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. A GitHub repo is referenced but no README content or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates. No confidence intervals, error bars, or ± notation anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of improvement ('16.68% relative improvement') are based solely on comparing point estimates. No statistical tests (p-values, t-tests, bootstrap) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context: '16.68% and 18.86% relative improvements over the best baselines' (Abstract, Section 5.2). Tables show absolute values for all methods, enabling readers to compute differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 12K training examples (4K per domain), 1K test examples for some benchmarks, or the specific benchmark sizes were chosen. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A explicitly states: 'we report single-run results for all experiments.' No variance, standard deviation, or spread measure reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines compared across three signal modalities: verbose-based (SelfAsk, SemanticEntropy), logit-based (ConfidenceMargin, Entropy, MaxLogits), and embedding-based (EmbeddingMLP). Section 5.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work: RouteLLM (Ong et al., 2025), SemanticEntropy (Kuhn et al., 2023), FusionFactory (Feng et al., 2025). Methods span 2022-2025 and represent state of the art across signal modalities."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies: Table 3 compares probe aggregation strategies (Final Layer vs Mean Pool vs Dirichlet), Table 4 compares input representations, Table 6 examines training data composition effects, Figure 3 studies probe architecture complexity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics: AUROC for router ability, LPM for cost-sensitive scenarios, MPM for balanced deployment, and HCR for accuracy-critical scenarios (Section 3.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. Open-ended task quality is judged by GPT-5 (LLM-as-a-Judge, Appendix B.1). No human evaluation of routing decisions or system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 shows explicit train/val and test splits. Additionally, three benchmarks (Magpie, MMLU Pro, MATH) serve as out-of-domain test sets with no training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide per-benchmark breakdowns for all methods across in-domain (Alpaca, BigMath, MMLU) and out-of-domain (Magpie, MATH, MMLU Pro subcategories) tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 'When Routing is Not Enough' shows a concrete failure case where both models converge on the same wrong answer, making routing ineffective. Includes specific example and analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows MLP probes provide no benefit over linear probes while increasing overfitting. Figure 4 shows diminishing returns with more training data. Appendix D.2 discusses fundamental limits of routing."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '16.68% and 18.86% relative improvements' are supported by Tables 1 and 2. Claims of consistency across model families supported by Table 5. Agent scenario claim supported by Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('data diversity yields additive gains,' 'Dirichlet aggregation prevents overfitting') are supported by controlled ablation studies. Table 6 varies training data composition while holding architecture fixed. Table 3 varies aggregation while holding data fixed. Single-variable manipulation is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Collaborative LLM Systems' but experiments test only two-model edge-cloud routing with one primary model pair (GPT-5 + Llama-3.1-8B). The Limitations section acknowledges 'single small-large model pair' but the title and framing remain broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses why hidden states outperform embeddings (hierarchical vs surface-level information), why linear probes suffice (hidden states encode shared difficulty notion), and alternative data composition effects (interference vs additive gains, Table 6)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates router ability (AUROC, intrinsic discrimination) from end-to-end system accuracy, noting that 'end-to-end accuracy at a given cost reflects both' router skill and model strength (Section 3.2). The framework is designed to disentangle proxy from outcome."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "'GPT-5' is stated without a version, snapshot date, or API version. 'Llama-3.1-8B-Instruct' and 'Qwen2.5-{0.5B,3B,7B}-Instruct' are more specific but GPT-5 lacks versioning. Per schema rules, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides the full LLM-as-a-Judge prompt template used for open-ended task evaluation. Exact reasoning tasks use xVerify (referenced with URL). Standard benchmark prompts are used for model evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Probe training hyperparameters reported: learning rate 1e-4, 50 epochs, seed=42, 12K training examples (Section 5.1). However, LLM generation hyperparameters (temperature, top-p, max tokens) for GPT-5 and Llama inference are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "ProbeDirichlet is a linear probe on hidden states, not an agentic scaffold. The HotpotQA agent experiment (Figure 5) is an evaluation scenario, not the paper's method."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes benchmark preparation, Table 7 lists train/val/test splits with sizes, and Appendix B.1 documents ground truth label construction for both exact reasoning (xVerify) and open-ended (LLM-as-Judge) tasks."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing the assumption that the large model is always better, single model pair constraint, and single-run results."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: 'both models may perform similarly or converge on the same incorrect answer in certain domains' (with analysis in Appendix D.2), 'single small-large model pair,' and 'single-run results due to computational constraints.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "States what was not tested: 'broader validation across diverse architectures, multiple seeds, and more complex OOD conditions would further strengthen the conclusions.' Also notes the assumption that large model capability exceeds small model's."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While benchmarks are public and code is released, the raw experimental outputs (model responses, hidden state extractions, routing decisions) are not provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes each benchmark's domain and purpose. Appendix B.1 details how ground truth labels are constructed using xVerify for exact tasks and LLM-as-a-Judge for open-ended tasks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark data to binary labels is documented: queries go through models, responses are evaluated by xVerify or LLM-as-Judge (Eq. 16-17), and labels are constructed. Training splits specified in Table 7."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper. Authors include one from Deepexi Technology Co. Ltd. (industry) and several from universities."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Southern University of Science and Technology, Institut Polytechnique de Paris, Peking University, Deepexi Technology Co. Ltd., University of Edinburgh, Beihang University, Chinese University of Hong Kong (Shenzhen)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. One co-author is from Deepexi Technology, an industry entity whose interest in the outcome is unknown."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for GPT-5 or Llama-3.1-8B-Instruct. These models are used to generate responses on benchmarks, and their training cutoffs are relevant to contamination risk."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (MMLU from 2021, MATH from 2021, Alpaca from 2023) appeared in GPT-5 or Llama training data. This could affect ground truth label quality."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2021) and MATH (2021) are well-known benchmarks likely in GPT-5 and Llama-3.1 training data. No contamination analysis is performed despite this risk affecting the validity of model performance comparisons."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency reported for ProbeDirichlet or any baseline method. For a paper about cost-efficient routing, the actual cost of running the router is notably absent."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or total compute budget stated. The Limitations section mentions 'computational constraints' but does not quantify them."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A: 'All experiments are conducted with a fixed random seed (seed=42)... we report single-run results for all experiments.' No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A explicitly states: 'we report single-run results for all experiments.' The number of runs (1) is clearly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Learning rate (1e-4), epochs (50), and training size (12K) appear fixed without explaining how these were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final configuration is presented without explaining how it was selected. No validation-based selection procedure described for the main hyperparameters."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so multiple comparison correction is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors compare their ProbeDirichlet against their own implementations of baselines without acknowledging author-evaluation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "ProbeDirichlet is claimed to be 'lightweight' but no compute comparison against baselines is provided. SemanticEntropy requires multiple generations while ProbeDirichlet uses hidden states, but the compute difference is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 extensively analyzes the construct validity of existing router metrics, demonstrating that AUC conflates router ability with model strength. The entire framework (RouterXBench) is designed to address validity gaps in router evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding comparisons between models. The router is a probe on hidden states. The HotpotQA agent experiment compares routing methods under the same setup."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmarks like MMLU (2021) and MATH (2021) predate GPT-5 and Llama-3.1 training, meaning solutions could be in training data, affecting ground truth labels."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, xVerify receives the query, model response, and gold answer — whether this introduces any artifacts is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test splits within benchmarks, or structural similarities between in-domain and out-of-domain test sets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied. No canary strings, membership inference, decontamination, or temporal split analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "ProbeDirichlet achieves 16.68% relative improvement over the best baseline in router ability (AUROC).",
       "evidence": "Table 1 shows ProbeDirichlet AUROC averages of 68.70 (in-domain) and 65.46 (out-of-domain) vs. best baseline EmbeddingMLP at 59.46 and 55.22. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "ProbeDirichlet achieves 18.86% relative improvement in high-accuracy scenarios (HCR).",
       "evidence": "Table 2 HCR section shows ProbeDirichlet at 18.50 (ID avg) and 15.40 (OOD avg) vs. best baseline SemanticEntropy at 15.17 and 13.35. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "Internal hidden states significantly outperform embedding-based and output-based methods for routing.",
       "evidence": "Table 4 compares identical linear models with different inputs: LLM hidden states (71.34/74.31) vs LLM embeddings (62.47/66.22) vs Longformer (61.95/66.19) on Alpaca/Magpie. Section 6.",
       "supported": "moderate"
     },
     {
       "claim": "Data diversity yields additive gains without interference across domains.",
       "evidence": "Table 6 shows adding domains preserves existing performance (Alpaca: 71.85→71.96→72.02) while independently boosting new domains (BigMath: 49.19→66.49→66.18). Section 6.",
       "supported": "strong"
     },
     {
       "claim": "Lightweight linear probes suffice; increased model complexity provides no benefit.",
       "evidence": "Figure 3 shows MLP probes with various hidden dimensions achieve comparable or worse AUROC than linear baseline while exhibiting larger train-validation gaps. Section 6.",
       "supported": "strong"
     },
     {
       "claim": "ProbeDirichlet generalizes consistently across model families (Llama, Qwen) and scales.",
       "evidence": "Table 5 shows ProbeDirichlet outperforms EmbeddingMLP across Llama-3.1-8B and Qwen2.5-{0.5B, 3B, 7B} with average improvements of 10.5% (ID) and 9.6% (OOD). Section 6.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run results with no uncertainty quantification",
       "detail": "All results are single-run with seed=42. For a paper proposing a benchmark evaluation framework, the lack of variance estimates across seeds undermines the reliability of claimed improvements. The 16.68% improvement could be within noise."
     },
     {
       "flag": "GPT-5 serves dual role as large model and LLM-as-a-Judge evaluator",
       "detail": "For open-ended tasks, GPT-5 scores are used to construct ground truth labels comparing small model vs. SOTA performance. But GPT-5 IS the large model. The judge is thus scoring its own outputs vs. the small model's, creating potential circularity in label construction. Footnote 3 acknowledges but does not address this."
     },
     {
       "flag": "No benchmark contamination analysis",
       "detail": "MMLU (2021), MATH (2021), and Alpaca (2023) are well-known benchmarks likely present in GPT-5 and Llama training data. If both models have memorized answers, routing ground truth labels become unreliable. This is not discussed."
     },
     {
       "flag": "Cost-efficiency paper without cost reporting",
       "detail": "The paper motivates routing as a cost reduction strategy ('reduces computational cost') but never reports actual inference costs, latency, or compute budgets for any method. SemanticEntropy requires multiple generations while ProbeDirichlet uses single-pass hidden states, but this advantage is never quantified."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "relevance": "Proposes preference-based LLM routing with curve-based evaluation metrics; directly compared as context for RouterXBench framework."
     },
     {
       "title": "AutoMix: Automatically Mixing Language Models",
       "authors": ["Pranjal Aggarwal", "Aman Madaan", "Ankit Anand"],
       "year": 2024,
       "relevance": "Training-free LLM mixing approach using incremental benefit per cost; represents cost-aware routing baseline."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Seminal work on cost-efficient LLM usage through routing and cascading strategies."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "arxiv_id": "2404.14618",
       "relevance": "Edge-cloud LLM routing with quality-aware routing; represents the hybrid deployment paradigm evaluated in this work."
     },
     {
       "title": "RouterEval: A Comprehensive Benchmark for Routing LLMs",
       "authors": ["Zhongzhan Huang", "Guoming Ling", "Yupei Lin"],
       "year": 2025,
       "relevance": "Concurrent routing benchmark exploring model-level scaling effects; complementary evaluation framework."
     },
     {
       "title": "EAGLE: Efficient Training-Free Router for Multi-LLM Inference",
       "authors": ["Zesen Zhao", "Shuowei Jin", "Z. Morley Mao"],
       "year": 2024,
       "arxiv_id": "2409.15518",
       "relevance": "Training-free multi-LLM routing using relative skill estimation."
     },
     {
       "title": "Semantic Uncertainty: Linguistic Invariances for Uncertainty Estimation in Natural Language Generation",
       "authors": ["Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar"],
       "year": 2023,
       "relevance": "Uncertainty estimation method used as a baseline routing signal; represents verbose-based routing approach."
     },
     {
       "title": "BEST-Route: Adaptive LLM Routing with Test-Time Optimal Compute",
       "authors": ["Dujian Ding", "Ankur Mallick", "Shaokun Zhang"],
       "year": 2025,
       "relevance": "Integrates routing with explicit test-time budget control for compute-aware LLM deployment."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Multi-agent collaboration framework relevant to agentic workflow evaluation of routers."
     },
     {
       "title": "RouterDC: Query-Based Router by Dual Contrastive Learning for Assembling Large Language Models",
       "authors": ["Shuhao Chen", "Weisen Jiang", "Baijiong Lin"],
       "year": 2024,
       "relevance": "Learning-based routing using contrastive query-model embedding alignment."
     },
     {
       "title": "How Robust Are Router-LLMs? Analysis of the Fragility of LLM Routing Capabilities",
       "authors": ["Aly M. Kassem", "Bernhard Schölkopf", "Zhijing Jin"],
       "year": 2025,
       "relevance": "Analyzes robustness of LLM routing systems; directly relevant to OOD robustness evaluation."
     },
     {
       "title": "Smoothie: Label Free Language Model Routing",
       "authors": ["Neel Guha", "Mayee F Chen", "Trevor Chow"],
       "year": 2024,
       "relevance": "Training-free routing leveraging weak agreement signals; represents unsupervised routing baseline."
     }
   ]
 }