ai-research-survey

scan.json (30342B)

---

 {
   "paper": {
     "title": "GreenServ: Energy-Efficient Context-Aware Dynamic Routing for Multi-Model LLM Inference",
     "authors": [
       "Thomas Ziller",
       "Shashikant Ilager",
       "Alessandro Tundo",
       "Ezio Bartocci",
       "Leonardo Mariani",
       "Ivona Brandic"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.17551",
     "doi": "10.48550/arXiv.2601.17551"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GreenServ, a contextual multi-armed bandit routing framework for LLM inference, achieves 22% higher accuracy and 31% lower energy consumption compared to random routing across 5 benchmarks and 16 open-source LLMs. Task type is the single most informative contextual feature for routing decisions, with full-feature contexts actually hurting convergence due to increased dimensionality. The system surpasses the static Pareto front of single-model deployments and adapts to new models at runtime without offline recalibration, with negligible overhead (6.7-7.8 ms per query).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All artifacts are open-source and available as an anonymous repository for review purposes' and provides a link to anonymous.4open.science. While this is an anonymous review link rather than a permanent public repository, a URL to the code is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five evaluation datasets are publicly available standard benchmarks: MMLU, HellaSwag, Winogrande, GSM8K, and CNN/Daily Mail, loaded through the HuggingFace datasets library (§5)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies Python 3.10, Ubuntu 22.04.5 LTS, CUDA 12.2, A100 80GB, bfloat16, and names libraries (FastAPI, sentence-transformers, scikit-learn, NumPy, PyTorch, transformers, zeus, textstat) but does not provide a requirements.txt, Dockerfile, or library version numbers sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The anonymous repository link is provided but the paper itself contains no README-equivalent or specific commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported throughout. §6.1.4: 'Results include 95% confidence intervals.' Figure 2a: 'Error bars represent 95% confidence intervals.' Figure 3 shows shaded areas representing 95% CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The paper claims GreenServ 'outperforms' baselines and notes '22% increase in accuracy' but relies on comparing point estimates and overlapping confidence intervals without formal hypothesis testing (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes with baseline context are reported: '22% increase in accuracy while reducing cumulative energy consumption by 31%' compared to random routing (§6.3.1). Additional comparisons: Smallest '+90-100% accuracy', accuracy-optimized '-10-12% accuracy, -75% energy.' These provide magnitude and baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "500 instances per dataset (2,500 total queries per run) are used with no justification for this sample size. No power analysis or rationale for why 500 was sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 50 independent runs (§6.3.1, §6.3.3) and 20 runs (§6.2.2) with 95% confidence intervals and shaded variance regions in figures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared (§6.1.6): Random, Largest (Yi-34B), Smallest (Qwen2.5-0.5B), Highest Accuracy (Gemma-3-27B), ε-Greedy, and Thompson Sampling."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary MAB algorithms (ε-Greedy, Contextual Thompson Sampling) and static strategies using current models (Gemma-3-27B, Yi-34B). The model pool includes recent models (Gemma 3, Qwen 2.5, Llama 3.1/3.2, Phi 4)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§6.2.3 and §6.3.3 present a thorough ablation study of contextual features: None (context-free), single features (Task, Cluster, Complexity), pairwise combinations, and Full features, with 50 runs per configuration."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation metrics are used: mean normalized accuracy, total energy consumption (Wh), cumulative regret, moving average regret, model selection frequency, overhead latency (ms), and AIQ from RouterBench."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated using Exact Match and ROUGE metrics. No human evaluation of routing quality or model output quality is performed. The paper acknowledges this limitation: 'our current evaluation focuses on tasks with objective ground truth' (§6.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "§6.1.2: '500 instances were uniformly sampled from the test set partition.' The task classifier uses separate training/validation splits (§4.2.1). RouterBench evaluation uses its own held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by algorithm type, by individual model (Figure 7, Table 3), by feature configuration (Figure 5, Figure 8), by λ value (Figure 4, Figure 9), and by overhead component (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific failure case examples are shown. The paper discusses system-level limitations (cold start, stationarity assumption) but does not present individual queries where routing failed or analyze error patterns qualitatively."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "§6.3.3 reports that including all features 'appears to raise regret levels notably' compared to task-only context, attributed to 'increased dimensionality which potentially slows convergence.' The Complexity feature slightly increases regret (+7). These are genuine negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '22% increase in accuracy while reducing cumulative energy consumption by 31%' are supported by §6.3.1 and Figure 2. RouterBench claims of '71.7% average accuracy with peak 75.7%' are confirmed in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made through controlled ablation (§6.3.3): removing/adding individual features demonstrates their contribution. The ablation design uses controlled single-variable manipulation across 50 runs. The model addition experiment (§6.3.4) uses controlled introduction at a specific timestep."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'five benchmark tasks and a pool of 16 contemporary open-access LLMs.' The discussion (§6.4) explicitly bounds generalization: controlled environments, single GPU hardware, objective ground truth tasks only, specific feature engineering choices."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The discussion (§6.4) covers limitations but does not consider alternative explanations for the observed improvements. For example, whether the specific benchmark selection or model pool composition could explain the routing advantages is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes its direct GPU energy measurements from proxy metrics used in prior work: 'Prior work often relies on proxy metrics (API costs, token budgets). We measure actual GPU power draw in watt-hours' (§5). They criticize others' use of 'synthetic proxies' (§1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact HuggingFace model handles are provided in Table 2 (Appendix A.1), e.g., 'Qwen/Qwen2.5-0.5B-Instruct', 'meta-llama/Llama-3.1-1B-Instruct', 'google/gemma-3-27b-it'. These are specific enough to identify exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide the actual prompts used to format benchmark queries for each model. §4.2.1 mentions 'the common structure of instruction-based tasks' but the actual prompt text or formatting templates used are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "MAB hyperparameters are well-documented (§6.1.5: LinUCB α=0.1, λ_reg=0.05, ε-Greedy ε₀=1.0, δ=0.98, ε_min=0.01, CTS σ=0.01, K=3, N_bins=3). However, critical LLM inference parameters (temperature, top_p, max_tokens) are not reported. batch_size=1 and bfloat16 are stated but sampling parameters are missing."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. GreenServ is a routing system that sends queries directly to models for single-pass inference without retry logic, tool use, or agent loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§6.1.2: '500 instances were uniformly sampled from the test set partition using a fixed random seed.' §4.2.1-4.2.3 describe feature extraction pipeline. §5 describes data loading, embedding computation, and evaluation metric computation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§6.4 Discussion contains substantial limitations discussion covering stationarity assumption, ground truth requirements, hardware generalization, feature sensitivity, and controlled vs. operational environments."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations in §6.4 are specific to this study: 'MAB algorithms assume stationary rewards and adapt slowly to drifts', 'our current evaluation focuses on tasks with objective ground truth', 'generalization across hardware requires latency profiles for various GPU architectures', 'sensitivity to specific feature engineering choices, such as the number of clusters K or the number of complexity bins N.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6.4 explicitly states scope boundaries: deterministic evaluation only (not open-ended generation), controlled environments (not operational conditions with concurrency and queuing), single GPU hardware. §3.1.1: 'we focus exclusively on tasks where accuracy can be measured objectively.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual query results, per-query energy measurements, routing decisions) is mentioned as being released. Only aggregate results are shown in figures and tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§6.1.2 describes dataset selection and sampling (500 instances per test partition with fixed random seed). §5 describes energy measurement via zeus library from GPU power monitoring, latency via Python's time module, accuracy via EM and ROUGE."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (MMLU, HellaSwag, Winogrande, GSM8K, CNN/DailyMail)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: datasets loaded from HuggingFace → 500 samples per dataset with fixed seed → feature extraction (embedding, clustering, complexity) → routing via MAB → inference → accuracy evaluation (EM/ROUGE) → energy/latency monitoring via zeus (§5, §6.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section, funding statement, or grant numbers appear in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: TU Wien, University of Amsterdam, University of Milano-Bicocca. All are academic institutions with no apparent conflict with the evaluated open-source models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of the funder cannot be assessed. The absence of funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 16 models evaluated. The models (Qwen 2.5, Gemma 3, Llama 3.1/3.2, Phi 4, etc.) are recent but their training data boundaries are not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 16 models may have been trained on the benchmark datasets. All five benchmarks (MMLU 2021, HellaSwag 2019, Winogrande 2021, GSM8K 2021, CNN/DailyMail 2015) predate the models and are widely available online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All five benchmarks were published years before the models' training. MMLU, HellaSwag, Winogrande, GSM8K, and CNN/DailyMail are all publicly available and likely in training data. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is entirely computational using benchmark datasets and automated metrics."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM routing performance on public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Energy consumption is reported in Wh throughout (e.g., ~165 Wh for GreenServ at λ=0.4). Overhead latency is reported per component (Table 4: 6.68-7.77 ms total). Per-model inference latency is detailed in Table 3 with min, P25, median, average, P75, max."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (A100 80GB, 512GB RAM, AMD EPYC 9354P) but total computational budget is not stated. With 50 runs for baseline comparison, 20 runs × 11 λ values for trade-off analysis, 50 runs × 8 configs for ablation, plus additional experiments, total GPU hours are never quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 50 independent runs for baseline comparison (§6.3.1) and ablation (§6.3.3), and 20 runs for λ sweep (§6.2.2), with 95% confidence intervals and variance shown in figures."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 50 runs for baseline comparison (§6.3.1), 20 runs for λ sweep (§6.2.2, 'we executed 20 runs'), 50 runs for ablation (§6.2.3, 'We executed 50 runs for each configuration'), single run for model addition (§6.3.4)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "§6.1.5 states 'we conducted preliminary experiments to tune hyperparameters' and §4.2.4 calls the search 'non-exhaustive tuning,' but the number of configurations tried, search method, and compute spent on tuning are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Final hyperparameter values are stated (§6.1.5) but the selection process is not described. No mention of selection on validation vs. test set, no reporting of all configurations tried, and no explanation of why these specific values were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons (GreenServ vs. 7 baselines across multiple λ values and feature configurations) but applies no correction for multiple comparisons (no Bonferroni, Holm, or Benjamini-Hochberg)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (ε-Greedy, Thompson Sampling, static strategies) themselves and compare against their own GreenServ system without acknowledging the bias of evaluating one's own system against self-implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is plotted directly against energy consumption (a compute proxy) in Figures 2b and 4. The accuracy-energy trade-off is the central analysis, with the Pareto front explicitly shown for comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses MMLU, HellaSwag, Winogrande, GSM8K, and CNN/DailyMail without discussing whether these benchmarks are appropriate proxies for real-world LLM routing scenarios or whether they measure the claimed capabilities."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. All models use the same direct inference pipeline (transformers + PyTorch with bfloat16), so there is no scaffold confound to address."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. All five benchmarks (MMLU 2021, HellaSwag 2019, Winogrande 2021, GSM8K 2021, CNN/DailyMail 2015) predate the training of the 16 models used (2024-2025 era), creating potential temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The paper does not consider whether the formatting or presentation of benchmark queries could leak answer information, or whether multiple-choice formatting provides hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether training data and benchmark data share structural similarities or overlapping sources."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "GreenServ achieves 22% higher accuracy and 31% lower energy consumption compared to random routing.",
       "evidence": "§6.3.1, Figure 2a: GreenServ accuracy ~0.65 vs. random ~0.51 with ~165 Wh vs. ~240 Wh, shown across 50 runs with 95% confidence intervals.",
       "supported": "strong"
     },
     {
       "claim": "GreenServ and contextual baselines surpass the static Pareto front, achieving accuracy-energy operating points unreachable by single-model deployments.",
       "evidence": "§6.3.1, Figure 2b: GreenServ and contextual baselines position in the upper-left region beyond the static Pareto front (red dashed line).",
       "supported": "strong"
     },
     {
       "claim": "Task type is the single most informative contextual feature for routing decisions, dropping median cumulative regret to approximately 400.",
       "evidence": "§6.3.3, Figure 5: Task-only context shows the largest regret reduction among single features across 50 runs. Adding all features raises regret due to increased dimensionality.",
       "supported": "strong"
     },
     {
       "claim": "GreenServ adapts to new models at runtime without offline recalibration.",
       "evidence": "§6.3.4, Figure 6: After Gemma-3-12b is introduced at query 1000, selection frequency rises to 20-25% within ~100 queries. Shown for a single run only.",
       "supported": "moderate"
     },
     {
       "claim": "GreenServ's total average overhead per query ranges between 6.68 and 7.77 ms.",
       "evidence": "§6.3.5, Table 4: Task classification 3.04ms, semantic clustering 3.37ms, complexity 0.15ms, LinUCB routing 0.86ms. Directly measured.",
       "supported": "strong"
     },
     {
       "claim": "GreenServ achieves an average accuracy of 71.7% with peak accuracy of 75.7% on RouterBench.",
       "evidence": "§6.3.6, Table 1: GreenServ peak accuracy 75.7%, average accuracy 71.7%, AIQ 0.607 across ~36k queries spanning 9 tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Contextual routing consistently outperforms non-contextual and static baselines across all λ configurations.",
       "evidence": "§6.3.2, Figures 4 and 9: Across λ from 0 to 1, contextual algorithms maintain higher accuracy and lower energy consumption than non-contextual ε-Greedy across 20 runs.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "All five benchmarks (MMLU, HellaSwag, Winogrande, GSM8K, CNN/DailyMail) were published in 2015-2021 and are widely available online. The 16 models tested are from the 2024-2025 era and almost certainly saw these benchmarks during training. This could inflate absolute accuracy numbers and distort routing decisions, yet contamination is never discussed."
     },
     {
       "flag": "No formal significance tests",
       "detail": "The paper claims GreenServ 'outperforms' baselines but notes that 'confidence intervals for both accuracy and energy consumption largely overlap across GreenServ and the contextual baselines' (§6.3.1). No formal hypothesis tests are conducted, making it unclear whether differences between contextual MAB variants are statistically significant."
     },
     {
       "flag": "Model addition experiment based on single run",
       "detail": "The adaptability claim (§6.3.4) relies on a single run, unlike baseline comparison (50 runs) and trade-off analysis (20 runs). This is insufficient evidence for a key contribution claim."
     },
     {
       "flag": "Missing LLM inference parameters",
       "detail": "Critical inference parameters (temperature, top_p, max_tokens) for the 16 models are not reported. These significantly affect model outputs and could influence routing behavior and accuracy measurements."
     },
     {
       "flag": "Anonymous repository only",
       "detail": "The code is provided via an anonymous review link (anonymous.4open.science) which is temporary and may not persist after review, undermining the reproducibility claim."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang", "Tianhao Wu", "Joseph E Gonzalez", "M Waleed Kadous", "Ion Stoica"],
       "year": 2025,
       "relevance": "Core related work on LLM routing using preference data, directly compared in the paper's related work section."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "doi": "10.48550/ARXIV.2305.05176",
       "relevance": "Foundational work on cost-efficient LLM inference through cascading, motivates the routing-based alternative."
     },
     {
       "title": "MixLLM: Dynamic Routing in Mixed Large Language Models",
       "authors": ["Xinyuan Wang", "Yanchi Liu", "Wei Cheng"],
       "year": 2025,
       "relevance": "Contextual-bandit-based LLM routing with tag-enhanced embeddings, closest comparison to GreenServ's approach."
     },
     {
       "title": "LLM Bandit: Cost-Efficient LLM Generation via Preference-Conditioned Dynamic Routing",
       "authors": ["Yang Li"],
       "year": 2025,
       "arxiv_id": "2502.02743",
       "relevance": "Multi-armed bandit approach to LLM routing with preference conditioning, directly related methodology."
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "authors": ["Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li", "Nan Jiang"],
       "year": 2024,
       "relevance": "Evaluation framework used for external validation of GreenServ, establishes routing benchmark methodology."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Major LLM reference relevant to the survey's coverage of foundational model capabilities and deployment costs."
     },
     {
       "title": "Eagle: Efficient Training-Free Router for Multi-LLM Inference",
       "authors": ["Zesen Zhao", "Shuowei Jin", "Z. Morley Mao"],
       "year": 2024,
       "arxiv_id": "2409.15518",
       "relevance": "Training-free multi-LLM routing approach, represents the zero-calibration end of the routing spectrum."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "relevance": "Meta-learning approach to LLM routing that reduces large model calls by 40%, relevant to cost-efficiency claims."
     },
     {
       "title": "Smoothie: Label Free Language Model Routing",
       "authors": ["Neel Guha", "Mayee Chen", "Trevor Chow"],
       "year": 2024,
       "relevance": "Label-free routing via embedding-based similarity, addresses calibration overhead problem."
     },
     {
       "title": "EmbedLLM: Learning Compact Representations of Large Language Models",
       "authors": ["Richard Zhuang", "Tianhao Wu", "Zhaojin Wen"],
       "year": 2025,
       "relevance": "Compact model representations for routing, relevant to efficient model selection approaches."
     },
     {
       "title": "GraphRouter: A Graph-Based Router for LLM Selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Graph neural network approach to LLM routing, models task-query-LLM interactions."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "relevance": "MMLU benchmark used as one of five evaluation datasets, major LLM capability benchmark."
     }
   ]
 }