ai-research-survey

scan.json (29213B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "PROTEUS: SLA-Aware Routing via Lagrangian RL for Multi-LLM Serving Systems",
     "authors": ["Amit Singh Bhatti", "Vishal Vaddina", "Dagnachew Birru"],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.19402",
     "doi": "10.48550/arXiv.2601.19402"
   },
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PROTEUS accepts accuracy targets τ as runtime input and achieves 100% floor compliance across evaluated τ values on RouterBench (405K queries, 11 models) and SPROUT (45K queries, 14 models), with τ-µ correlation of 0.97–0.98. A single trained model serves the full accuracy spectrum (τ ∈ [0.85, 0.95]) without retraining, achieving 90.1% accuracy on RouterBench (1.3pp below oracle) and 94.0% on SPROUT (4.6pp below oracle) with cost savings up to 89.8% versus the best fixed model. The closest baseline (OmniRouter) achieves only 22% floor compliance despite also using Lagrangian optimization.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository, GitHub link, or archive is provided anywhere in the paper. The implementation is described in detail (Section 3.1) but no code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available benchmark datasets: RouterBench (Hu et al., 2024) and SPROUT (Somerstep et al., 2025), both released by their respective authors."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions PyTorch, HuggingFace Transformers, DeBERTa-v3-small, and A100 GPU, but provides no requirements.txt, Dockerfile, or detailed dependency versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method description is detailed but there are no explicit commands or procedures to reproduce the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., '90.1% accuracy', '0.973 correlation') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PROTEUS outperforms all baselines (Table 2) and that ablation components matter (Table 4) based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, bootstrap) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '1.3 percentage points below oracle' (Section 4.2), 'cost savings of 90% versus GPT-4' (Section 4.2), ablation drops of '−0.27%' and '−0.53%' (Table 4), and cost ranges per tier (Table 1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 405K (RouterBench) and 45K (SPROUT) queries are sufficient, no power analysis, and no discussion of whether the sample supports the precision of claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The dynamic adaptation experiment uses '5 random seeds' (Section 4.1) but reports only aggregate floor satisfaction percentages without standard deviation or spread. Main results in Tables 1 and 2 show single-run numbers with no variance measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: 3 static (Random, Cheapest, Best Fixed), 4 learned routers (KNN, MLP, CARROT at 3 configurations, OmniRouter), and 3 ablated variants (Section 3.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CARROT (Somerstep et al., 2025), OmniRouter (Mei et al., 2025), and RouteLLM (Ong et al., 2024), all from within 1-2 years of this work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents a systematic ablation removing three components: Lagrangian constraint (λ), learnable γ, and critic network, with accuracy and cost impact on both datasets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy, cost, τ-µ correlation, SLA compliance (floor and tolerance-band), Routing Efficiency (RE), Routing Performance Index (RPI), and latency (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using benchmark correctness labels. No human evaluation of routing quality, user satisfaction, or real-world quality of selected model responses is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses '70/15/15 train/val/test splits for RouterBench and official splits for SPROUT' (Section 3.1) and reports results on held-out test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-tier breakdowns (Economy/Standard/Premium) with accuracy and cost for each tier on both datasets. Table 2 reports per-dataset results for all methods."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific routing failure examples, error analysis, or qualitative discussion of where routing decisions go wrong. The paper reports ±2% tolerance compliance is only 44%/11% (Table 1) but does not analyze why or show individual failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports the critic provides negligible benefit (<0.05% accuracy change), effectively a negative result on that component's value. SLA compliance at ±2% tolerance is low (44%/11%), and dynamic floor satisfaction is only 77-86% (Section 4.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are verified: 100% floor compliance (Table 1), τ-µ correlation 0.973/0.981 (Table 1), OmniRouter 22% compliance (Section 4.1/Figure 2a), 90.1%/94.0% accuracy (Table 2), 89.8% cost savings (Table 2 derived: $3.3 → $0.33)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 4) makes causal claims ('removing λ causes accuracy drops') through controlled single-variable manipulation, which is adequate for causal inference about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly bounds generalization: 'RouterBench's model pool includes... all of which predate current frontier models,' 'SPROUT covers narrower task distributions,' 'production systems lack such labels by default,' and 'model costs ci are fixed at training time' (Section 5)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for PROTEUS's performance advantages. No discussion of whether results could be explained by benchmark-specific artifacts, encoder choice effects, or training data characteristics rather than the Lagrangian constraint mechanism."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges the gap between benchmark accuracy and production quality: 'PROTEUS trains on ground-truth correctness labels... but production systems lack such labels by default, so deployment would require periodic sampling with human annotation or LLM-as-judge evaluation' (Section 5)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "DeBERTa-v3-small is specified precisely, but the LLM pool models are referred to by marketing names only: 'GPT-3.5, GPT-4, Claude-2' (RouterBench) and 'GPT-4o, Claude-3.5-Sonnet, o3-mini' (SPROUT) without exact version identifiers or snapshot dates."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. PROTEUS encodes queries via DeBERTa-v3-small embeddings and routes using a learned policy. No prompts are sent to any LLM as part of the method."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are reported in detail: batch size 32, AdamW lr=3×10⁻⁴, 10K steps, ηλ=0.4, gradient clipping max norm 1.0, γ learnable in [2,8], dual updates every 5 batches, 256-dim embeddings, 2-layer MLP policy (Section 2 and 3.1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. PROTEUS is a routing layer that selects among models, not an agent with tools, memory, or iterative execution."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Data splits are stated ('70/15/15 train/val/test' for RouterBench, 'official splits' for SPROUT) and cost normalization is described (EMA of percentile-based bounds), but no documentation of data filtering, query preprocessing, or transformation steps from raw benchmark data to training input."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A substantive 'Limitations' subsection is present in Section 5 (Conclusion), spanning approximately one full paragraph with multiple specific limitations discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: outdated model pools in RouterBench, compressed accuracy ranges making fine-grained τ distinctions difficult, reliance on ground-truth labels unavailable in production, and fixed costs becoming stale after API pricing changes (Section 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Specific scope boundaries are stated: 'RouterBench's model pool includes Llama-2, GPT-3.5, GPT-4, and Claude-2, all of which predate current frontier models,' 'SPROUT covers narrower task distributions,' and 'deployment would require periodic sampling with human annotation' (Section 5)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying benchmark data (RouterBench and SPROUT) is publicly released by their respective authors, enabling independent verification of the routing results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is described: 'RouterBench provides 405K inference outcomes across 11 models' spanning reasoning, mathematics, and coding tasks, and 'SPROUT complements RouterBench with 45K queries across 14 models' with instruction-following queries (Section 3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmark datasets (RouterBench, SPROUT)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes train/val/test splits and cost normalization but does not document the full pipeline from benchmark data to training input, including any filtering, transformation, or quality checks applied to the raw benchmark data."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section thanks open-source communities and dataset authors but does not mention any grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are clearly listed as affiliated with 'Phi Labs, Quantiphi, Boston, USA' with email addresses at quantiphi.com."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed despite all authors being from Quantiphi (a consulting/AI services company), making it impossible to assess funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The authors work at Quantiphi, an AI services company that could commercially benefit from routing technology, but no financial interests are declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a routing system, not a pre-trained model's capability on benchmarks. The LLM model performances come from pre-existing benchmark datasets (RouterBench, SPROUT), not from the authors querying models."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a routing tool's ability to select among models, not model knowledge or capability on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates routing policy quality, not the underlying LLM capabilities. Benchmark contamination of the LLM pool is the responsibility of the benchmark authors."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is a benchmark evaluation of a routing system."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost and latency are reported extensively: Table 3 provides per-query routing latency (2.6-8.7ms) and throughput (115-385 q/s). Table 1 reports per-tier costs ($0.20-$1.69/1K queries). Cost savings vs baselines are quantified (Table 2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training completes in '4 hours per dataset on one A100 GPU' (Section 2.3). Hardware specified as 'a single A100 GPU' (Sections 2.2, 3.1)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The dynamic adaptation experiment uses '5 random seeds' (Section 4.1) but does not report mean/std across seeds. Main results in Tables 1 and 2 show no evidence of multi-seed evaluation or seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "For dynamic adaptation: '5 random seeds' is stated. For the main results (Tables 1 and 2), the number of experimental runs is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states ηλ=0.4 was 'determined empirically' and mentions values >1.0 and <0.1 didn't work (Section 2.3), but reports no formal search budget, number of configurations tried, or search method."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'dual variable convergence as monitored on validation accuracy' (Section 2.3) but does not describe the selection criterion in detail or show validation performance across configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so there are no multiple comparisons to correct."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The '-Style' suffix on 'OmniRouter-Style' and 'RouteLLM-Style' (Section 3.1) indicates re-implementations rather than original code. No discussion of author-evaluation bias or whether these re-implementations faithfully represent the original methods."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'all learned routers use similar-sized encoders (22–125M params) with comparable latency' (Table 2 footnote) but does not report performance as a function of compute budget or provide matched-compute comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what RouterBench and SPROUT contain and notes complementary strengths, but does not discuss whether benchmark correctness labels actually measure real-world quality or whether the evaluation setup has validity gaps."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. PROTEUS is a routing layer, not an agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the DeBERTa encoder's pre-training data includes benchmark questions, or whether the temporal ordering of benchmark creation vs model training affects the validity of correctness labels used for routing."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the query encoding or routing features could leak information about model correctness, or whether the benchmark setup provides hints not available in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While train/test splits are used, there is no discussion of whether train and test queries share structural similarities (same topic domains, similar difficulty distributions) that could inflate generalization claims."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are described. No canary strings, membership inference, n-gram overlap analysis, or decontamination steps are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "PROTEUS achieves 100% floor compliance where accuracy meets or exceeds τ across all evaluated targets on both benchmarks.",
       "evidence": "Table 1 shows 100% floor compliance on both RouterBench and SPROUT. Figure 2(b) shows accuracy consistently exceeding the floor constraint across all τ values.",
       "supported": "strong"
     },
     {
       "claim": "The τ-µ correlation reaches 0.973 on RouterBench and 0.981 on SPROUT, demonstrating faithful translation of accuracy targets into routing behavior.",
       "evidence": "Table 1 reports these correlation values. Figure 2(a) shows PROTEUS adapting accuracy to match requested τ across both datasets.",
       "supported": "strong"
     },
     {
       "claim": "PROTEUS achieves 90.1% accuracy on RouterBench (1.3pp below oracle) and 94.0% on SPROUT (4.6pp below oracle) with significant cost savings.",
       "evidence": "Table 2 shows 90.1% (RB) and 94.0% (SP) accuracy versus oracle of 91.4% and 98.6%. Cost is $0.33/1K (RB) vs $3.3/1K for GPT-4, representing 90% savings.",
       "supported": "strong"
     },
     {
       "claim": "OmniRouter achieves only 22% floor compliance on RouterBench and 0% on SPROUT despite using Lagrangian optimization.",
       "evidence": "Section 4.1 and Figure 2(a) report these numbers. However, the comparison uses 'OmniRouter-Style' (a re-implementation), not the original code.",
       "supported": "moderate"
     },
     {
       "claim": "A single trained PROTEUS model serves the full accuracy spectrum τ ∈ [0.85, 0.95] without retraining.",
       "evidence": "Table 1 shows tiered performance from Economy to Premium tiers, and Figure 2(c) shows dynamic τ adaptation across four scenarios, all from a single model.",
       "supported": "strong"
     },
     {
       "claim": "The Lagrangian constraint mechanism (λ) is the critical component, causing 0.27% (RB) and 0.53% (SP) accuracy drops when removed.",
       "evidence": "Table 4 ablation study shows these drops alongside 8-25% cost reductions when λ is removed, indicating the unconstrained variant sacrifices accuracy for cost.",
       "supported": "moderate"
     },
     {
       "claim": "PROTEUS achieves highest routing efficiency with RE of 11.1 pp/ms (RB) and 9.5 pp/ms (SP), and RPI of 88.5 (RB) exceeding oracle RPI of 88.2.",
       "evidence": "Table 2 reports these metrics. The RPI exceeding oracle is explained by the cost-quality tradeoff: accepting 1.4% quality reduction saves 15% cost (Section 4.2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No code released",
       "detail": "The implementation is described in detail but no source code is provided, making independent verification of the results impossible."
     },
     {
       "flag": "Re-implemented baselines",
       "detail": "Key baselines are labeled 'OmniRouter-Style' and 'RouteLLM-Style' (Section 3.1), indicating re-implementations rather than original code. The authors' implementations may systematically underperform the originals (Lucic et al., 2018)."
     },
     {
       "flag": "No statistical tests or error bars",
       "detail": "All claims of superiority are based on comparing point estimates (Tables 1-2). With no significance tests, error bars, or multi-seed results for main experiments, it is unclear whether observed differences are statistically meaningful."
     },
     {
       "flag": "Novel metrics favor proposed method",
       "detail": "RE and RPI are metrics introduced by the authors (Section 3.1). RPI is defined such that PROTEUS exceeds even oracle performance (88.5 vs 88.2 in Table 2), which raises questions about metric design choices."
     },
     {
       "flag": "Missing funding disclosure from corporate authors",
       "detail": "All authors are from Quantiphi (an AI consulting company) but no funding source or competing interests are declared. Routing technology could be commercially relevant to the company."
     }
   ],
   "cited_papers": [
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "authors": ["Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li", "Nan Jiang", "Benjamin Keigwin", "Gaurav Ranganath", "Kurt Keutzer", "Shriyash Kaustubh Upadhyay"],
       "year": 2024,
       "arxiv_id": "2403.12031",
       "relevance": "Gold-standard benchmark for multi-LLM routing with 405K inference outcomes across 11 models, used as a primary evaluation dataset."
     },
     {
       "title": "CARROT: A Cost Aware Rate Optimal Router",
       "authors": ["Seamus Somerstep", "Felipe Maia Polo", "Allysson Flavio Melo de Oliveira"],
       "year": 2025,
       "arxiv_id": "2502.03261",
       "relevance": "Contemporary learned router baseline using RoBERTa encoder and cost-quality tradeoff scoring, also provides the SPROUT benchmark dataset."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Learned LLM router using preference data and threshold calibration, representative of the indirect parameter-tuning approach PROTEUS aims to replace."
     },
     {
       "title": "OmniRouter: Budget and Performance Controllable Multi-LLM Routing",
       "authors": ["Kai Mei", "Wujiang Xu", "Minghao Guo", "Shuhang Lin", "Yongfeng Zhang"],
       "year": 2025,
       "arxiv_id": "2502.20576",
       "relevance": "Closest baseline to PROTEUS, uses batch-level Lagrangian optimization for LLM routing but achieves only 22% floor compliance."
     },
     {
       "title": "Automix: Automatically Mixing Language Models",
       "authors": ["Pranjal Aggarwal", "Aman Madaan", "Ankit Anand"],
       "year": 2024,
       "arxiv_id": "2310.12963",
       "relevance": "Cascade-based LLM routing method that routes through models until confidence thresholds are met."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Early work on cost reduction through LLM cascading and selection, foundational to the routing literature."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "arxiv_id": "2404.14618",
       "relevance": "Learned LLM routing approach with quality-aware query routing, published at ICLR 2024."
     },
     {
       "title": "Router-R1: Teaching LLMs Multi-Round Routing and Aggregation via Reinforcement Learning",
       "authors": ["Haozhen Zhang", "Tao Feng", "Jiaxuan You"],
       "year": 2025,
       "arxiv_id": "2506.09033",
       "relevance": "RL-based multi-round routing method, contemporary work applying reinforcement learning to LLM routing."
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "authors": ["Dongfu Jiang", "Xiang Ren", "Bill Yuchen Lin"],
       "year": 2023,
       "relevance": "LLM ensembling approach using pairwise ranking, relevant to multi-model selection and routing."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2023,
       "relevance": "HELM benchmark framework establishing that LLM costs span four orders of magnitude, motivating cost-aware routing."
     },
     {
       "title": "Learning to Route LLMs from Bandit Feedback: One Policy, Many Trade-offs",
       "authors": ["Wei Wang", "Tiankai Yang", "Hongjie Chen"],
       "year": 2025,
       "arxiv_id": "2510.07429",
       "relevance": "Bandit feedback approach to LLM routing that reduces annotation cost, cited as future work direction."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "vLLM serving system that PROTEUS integrates with for production deployment."
     }
   ]
 }