ai-research-survey

scan.json (25909B)

---

 {
   "paper": {
     "title": "BOUTE: Cost-Efficient LLM Serving with Heterogeneous LLMs and GPUs via Multi-Objective Bayesian Optimization",
     "authors": ["Youhe Jiang", "Fangcheng Fu", "Eiko Yoneki"],
     "year": 2026,
     "venue": "MLSys 2026",
     "arxiv_id": "2602.10729"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The system implementation section (§6) describes the components but does not release code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available datasets: GSM8K (Cobbe et al., 2021) and MTBench (Zheng et al., 2023), both standard public benchmarks. The ETH EASL Scratchpad simulator is also publicly available at https://github.com/eth-easl/Scratchpad."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 specifies the GPU hardware in detail: NVIDIA H100-80G, RTX PRO 6000-96G, RTX 5090-32G, and RTX 4090-24G, with interconnect bandwidths (NVLink 300GB/s, PCIe 60GB/s). Section 6 specifies GPyTorch and BoTorch frameworks. However, no requirements.txt or Dockerfile is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts for replicating the main experiments are provided. The paper describes the system architecture but not how to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals, error bars, or ± notation are reported for the main results in Figure 4 or Tables 1-3. All results are presented as point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'up to 157% improvement') but no statistical significance tests (p-values, t-tests, etc.) are used to validate these differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports percentage improvements with baseline context. For example, 'BOUTE reduces P95 latency by up to 66% (50% on average)' and 'reducing costs by 15%-61% (38% on average)' in §7.2 and Table 3, with absolute values provided in the figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of workload traces, the system load level (100 req/s), or the number of quality requirement thresholds tested. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or spread measures are reported. The simulator accuracy in Table 4 shows absolute percentage errors but no variance across multiple runs of the full system."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 describes four baselines: Stand-Alone-Homo (vLLM with homogeneous GPUs), Stand-Alone-Hetero (vLLM with heterogeneous GPUs), RouteLLM, and BOUTE-Homo (BOUTE without heterogeneous GPUs)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include RouteLLM (2024), vLLM (2023), and systems like Helix (2025) and ThunderServe (2025) are discussed in related work. The baselines are recent and represent state-of-the-art open-source LLM serving systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents ablations: BOUTE vs. BOUTE (w/o structural info) and BOUTE (w/o offline prep). Figure 5 also compares BOUTE, BOUTE-Homo, and BOUTE (w/o structural info) across quality requirements. §3 provides a step-by-step workload characterization showing the contribution of each component (routing, resource allocation, heterogeneous GPUs)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on multiple metrics: P95 latency, average latency, P96-P99 latencies, throughput (req/s), quality (accuracy on GSM8K, score on MTBench), and cost ($/h) in Table 3."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper about LLM serving infrastructure. Human evaluation of the system's outputs is not relevant — the paper evaluates system-level metrics (latency, throughput, cost) using automated measurements."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning modeling paper. The evaluation uses workload traces for system benchmarking, not a train/dev/test split. The concept of a held-out test set does not structurally apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by workload (GSM8K vs. MTBench), by quality requirement levels (87/89/91 for GSM8K, 8.1/8.3/8.5 for MTBench), and by percentile latency metrics (AVG, P95-P99) in Figure 4. Table 1 provides per-model routing and resource allocation details."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 discusses failure cases: Approach 1 shows that naive routing with uniform resource allocation actually increases latency by 10% over the baseline. §7.3 discusses how BOUTE (w/o structural info) frequently converges to local optima, resulting in inferior performance. Figure 5 shows cases where the ablated system underperforms even BOUTE-Homo."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3, Approach 1 reports that model routing with uniform resource allocation increases P95 latency by 10% compared to the baseline. The paper also notes that BOUTE (w/o structural info) can perform worse than BOUTE-Homo (§7.4), and that RouteLLM achieves even lower throughput than standalone deployment in some cases (§7.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 157% and 59% on average' throughput improvement and '15%-61% (38% on average)' cost reduction. These are supported by Figure 4 and Table 3 in §7.2 and §7.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about component contributions via controlled ablations. The workload characterization (§3) systematically adds components one at a time (routing, resource allocation, heterogeneous GPUs), and §7.3/Table 2 ablates structural information and offline preparation. These are controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'cost-efficient LLM serving' generally, but results are only shown for Llama3.1-8B and Llama3.1-70B on two benchmarks (GSM8K, MTBench) with a single router. The paper does not bound its claims to these specific models or workloads, nor acknowledge that results may differ with other model families, sizes, or workload patterns."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the improvements are due to the specific model pairing (8B vs 70B having a large capability gap), the particular GPU price ratios tested, or the workload characteristics of GSM8K/MTBench. No threats-to-validity or alternative explanations section is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 specifies 'Llama3.1-8B' and 'Llama3.1-70B' which are specific model versions with known architectures and parameter counts. The router is from RouteLLM (Ong et al., 2024) trained on specific data."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper evaluating serving infrastructure. The evaluation uses benchmark datasets (GSM8K, MTBench) directly, not prompting. No custom prompts are designed as part of the methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports system-level hyperparameters: system load of 100 req/s, budget of $30/h, quality requirements (86/88/90 for GSM8K, 8.1/8.3/8.5 for MTBench), GPU costs per type (§7.1), MOBO convergence criterion (stable for 20 consecutive iterations, Table 2), and kernel specifications (Matérn-5/2, Matérn-3/2, §6)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. BOUTE is a scheduling/serving system, not an agentic LLM system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 states workload traces are 'subsampled from GSM8K and MTBench' following prior work (Peng et al., 2025; Zhong et al., 2024). The routing score computation and system load configuration (100 req/s) are specified. Appendix A describes simulator inputs including workload-specific characteristics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section anywhere in the paper. The paper moves directly from evaluation results to the conclusion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. There is no mention of potential limitations such as simulator accuracy affecting conclusions, generalizability to other model families, or sensitivity to GPU pricing."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not clarify that results are specific to the tested model pair, GPU types, benchmarks, or routing strategy. The conclusion makes broad claims about 'democratization of LLM serving' without qualifying scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, performance logs, or simulation outputs are made available. Only aggregated results in figures and tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 describes the experimental setup: GPU server configurations, GPU types and costs, interconnect bandwidths, models used, router design, and workload trace generation methodology referencing prior work."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses computational benchmarks and simulations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: offline preparation (§5.1) profiles GPUs and simulates deployments, building a performance database. Online optimization (§5.2) generates deployment candidates via Pareto-skimming and knapsack optimization, then runs MOBO. The coordinator (§6, Appendix C) dispatches queries based on optimal configurations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Cambridge (Youhe Jiang, Eiko Yoneki) and Shanghai Jiao Tong University (Fangcheng Fu). These are academic institutions, not vendors of the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of the funder cannot be assessed. The absence of any funding or acknowledgments statement means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper. First author Youhe Jiang has multiple related papers on heterogeneous LLM serving (HexGen, ThunderServe, etc.) which could indicate a pattern, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate model capabilities on benchmarks to measure model knowledge. It uses benchmark datasets (GSM8K, MTBench) to generate workload traces for system-level performance evaluation (latency, throughput). The quality metric measures how well the routing system preserves the models' existing capabilities, not the models' raw benchmark performance."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates serving system performance (latency, throughput, cost), not model benchmark performance. Contamination of the benchmark in model training data would not affect the validity of the systems-level claims."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper's claims are about system-level serving efficiency, not about model accuracy on benchmarks. Benchmark contamination is irrelevant to the systems claims."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost is a central focus of the paper. Table 3 reports the hourly cost for each system to meet specific quality and latency targets (e.g., BOUTE at $32.25/h vs. Stand-Alone-Homo at $61.83/h for GSM8K with quality 87 and latency 8s). GPU per-hour costs are specified in §7.1."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies the budget constraint ($30/h for main experiments, §7.1), GPU costs per type ($2.64-3.07/h for H100, $0.89/h for RTX 5090, etc.), and scheduling convergence time (23.5s to 1.2min for BOUTE, Table 2). The offline preparation completes 'within several dozen minutes' (§5.1)."
       }
     }
   },
   "claims": [
     {
       "claim": "BOUTE outperforms state-of-the-art LLM serving systems by up to 157% and 59% on average in throughput under identical cost budgets and quality requirements.",
       "evidence": "Figure 4 shows throughput comparisons across GSM8K and MTBench workloads with varying quality requirements. The 157% improvement is against Stand-Alone-Homo on MTBench-8.1. The 59% average is across all baselines and settings (§7.2).",
       "supported": "strong"
     },
     {
       "claim": "BOUTE reduces serving costs by 15%-61% (38% on average) while maintaining the same performance targets.",
       "evidence": "Table 3 shows hourly costs under identical quality and latency requirements. BOUTE achieves $30.98-$32.25/h vs. $37.50-$79.93/h for baselines across four test scenarios (§7.4).",
       "supported": "strong"
     },
     {
       "claim": "Heterogeneous model deployment naturally complements heterogeneous model routing, creating a synergistic system.",
       "evidence": "Section 3 provides a step-by-step workload characterization showing that combining routing with heterogeneous GPU deployment reduces P95 latency by 33% compared to single-model deployment. Figure 1 and the comparison between BOUTE and BOUTE-Homo (14% average improvement, §7.2) support this.",
       "supported": "strong"
     },
     {
       "claim": "Incorporating structural information into the MOBO framework reduces scheduling time by approximately 8x.",
       "evidence": "Table 2 compares BOUTE vs. BOUTE (w/o structural info): 23.5s vs. 3.6min for 3 GPU types, 32.5s vs. 5.3min for 4 GPU types (§7.3). However, this is a single measurement with no variance reported.",
       "supported": "moderate"
     },
     {
       "claim": "Different models favor different GPU types for cost-efficient serving.",
       "evidence": "Section 3 shows that Llama3.1-8B achieves 1.5x lower P95 latency on RTX 5090 vs H100 under equal cost, while Llama3.1-70B achieves 2x lower P95 latency on H100 vs RTX 5090. Demonstrated with one model pair only.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "BOUTE is a quality-aware scheduling system that jointly optimizes heterogeneous model routing and GPU deployment for cost-efficient LLM serving using multi-objective Bayesian optimization (MOBO). The system achieves up to 2.57x lower latency or 15-61% cost reduction compared to state-of-the-art baselines (vLLM, RouteLLM) on GSM8K and MTBench workloads with Llama3.1-8B/70B. The key insight is that heterogeneous model deployment (matching GPU types to model characteristics) naturally complements heterogeneous model routing, and co-optimizing both dimensions through MOBO with structural information achieves superior Pareto-optimal solutions. The offline preparation phase (simulator-based performance profiling) enables the online optimization to converge in under a minute.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are presented as single point estimates with no confidence intervals, error bars, or variance across experimental runs. For a systems paper, run-to-run variance in latency measurements can be significant due to queuing effects, network contention, and GPU thermal throttling."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or scope-bounding discussion. Given the narrow experimental scope (one model pair, two benchmarks, one router), this is a significant omission."
     },
     {
       "flag": "Narrow model diversity",
       "detail": "All experiments use only Llama3.1-8B and Llama3.1-70B. The approach's effectiveness with other model families (e.g., Mistral, Qwen), more than two model types, or models with different architectural characteristics is unknown. The abstract and conclusion make general claims about 'LLM serving' without this caveat."
     },
     {
       "flag": "Simulator-based evaluation concerns",
       "detail": "The offline preparation relies on a simulator (ETH EASL Scratchpad) with 2-7% estimation errors (Table 4). While simulator accuracy is validated for throughput, its accuracy for latency percentiles (P95-P99) under realistic conditions is not independently verified. The end-to-end results mix real GPU execution with simulator-optimized configurations."
     },
     {
       "flag": "Self-citation density",
       "detail": "Several of the cited system works (HexGen, ThunderServe, HexGen-2, Hexiscale, etc.) share the first author (Youhe Jiang), suggesting the evaluation ecosystem may be somewhat self-referential. No potential conflicts are declared."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu", "W.-L. Chiang", "T. Wu", "J. E. Gonzalez", "M. W. Kadous", "I. Stoica"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Core baseline for heterogeneous model routing, directly compared against BOUTE in the evaluation."
     },
     {
       "title": "HybridLLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["D. Ding", "A. Mallick", "C. Wang", "R. Sim", "S. Mukherjee", "V. Ruhle", "L. V. Lakshmanan", "A. H. Awadallah"],
       "year": 2024,
       "arxiv_id": "2404.14618",
       "relevance": "Adaptive LLM routing system that switches between large and small models based on query complexity."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "relevance": "Cascading approach for cost-efficient LLM usage that routes through models from weakest to strongest."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["W. Kwon", "Z. Li", "S. Zhuang", "Y. Sheng", "L. Zheng", "C. H. Yu", "J. Gonzalez", "H. Zhang", "I. Stoica"],
       "year": 2023,
       "relevance": "vLLM serving system used as the primary baseline for standalone LLM serving in the evaluation."
     },
     {
       "title": "Helix: Serving Large Language Models over Heterogeneous GPUs and Network via Max-Flow",
       "authors": ["Y. Mei", "Y. Zhuang", "X. Miao", "J. Yang", "Z. Jia", "R. Vinayak"],
       "year": 2025,
       "relevance": "Heterogeneous GPU deployment system using max-flow algorithm, directly relevant to the LLM serving cost-efficiency research."
     },
     {
       "title": "ThunderServe: High-Performance and Cost-Efficient LLM Serving in Cloud Environments",
       "authors": ["Y. Jiang", "F. Fu", "X. Yao", "T. Wang", "B. Cui", "A. Klimovic", "E. Yoneki"],
       "year": 2025,
       "arxiv_id": "2502.09334",
       "relevance": "Heterogeneous GPU deployment for LLM serving that separates prefill and decoding phases across different GPU types."
     },
     {
       "title": "Mélange: Cost Efficient Large Language Model Serving by Exploiting GPU Heterogeneity",
       "authors": ["T. Griggs", "X. Liu", "J. Yu", "D. Kim", "W.-L. Chiang", "A. Cheung", "I. Stoica"],
       "year": 2024,
       "arxiv_id": "2404.14527",
       "relevance": "Explores GPU heterogeneity for cost-efficient LLM serving, closely related to the deployment optimization in BOUTE."
     },
     {
       "title": "Rerouting LLM Routers",
       "authors": ["A. Shafran", "R. Schuster", "T. Ristenpart", "V. Shmatikov"],
       "year": 2025,
       "arxiv_id": "2501.01818",
       "relevance": "Examines security and adversarial aspects of LLM routing systems, relevant to the robustness of routing-based serving approaches."
     },
     {
       "title": "Learning to Route LLMs with Confidence Tokens",
       "authors": ["Y.-N. Chuang", "P. K. Sarma", "P. Gopalan", "J. Boccio", "S. Bolouki", "X. Hu", "H. Zhou"],
       "year": 2024,
       "arxiv_id": "2410.13284",
       "relevance": "Alternative approach to LLM routing using confidence tokens, directly related to the heterogeneous model routing paradigm."
     },
     {
       "title": "Adaptive LLM Routing under Budget Constraints",
       "authors": ["P. Panda", "R. Magazine", "C. Devaguptapu", "S. Takemori", "V. Sharma"],
       "year": 2025,
       "arxiv_id": "2508.21141",
       "relevance": "Addresses the same problem space of budget-constrained LLM routing, directly relevant to cost-efficient serving research."
     },
     {
       "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
       "authors": ["Y. Zhong", "S. Liu", "J. Chen", "J. Hu", "Y. Zhu", "X. Liu", "X. Jin", "H. Zhang"],
       "year": 2024,
       "relevance": "Key LLM serving system that disaggregates prefill and decoding phases, influencing the serving architecture design space."
     }
   ]
 }