ai-research-survey

scan.json (19448B)

---

 {
   "paper": {
     "title": "Aegaeon: Effective GPU Pooling for Concurrent LLM Serving on the Market",
     "authors": ["Yuxing Xiang", "Xue Li", "Kun Qian", "Yufan Yang", "Diwen Zhu", "Wenyuan Yu", "Ennan Zhai", "Xuanzhe Liu", "Xin Jin", "Jingren Zhou"],
     "year": 2025,
     "venue": "SOSP '25",
     "doi": "10.1145/3731569.3764815"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The system is described as deployed at Alibaba Cloud but no open-source release is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The production workload data from Alibaba Cloud Model Studio is not released. ShareGPT is public, but the paper's proprietary workload traces are not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (H800 GPUs, DDR5, Intel Xeon 8469C CPUs) but no software environment specifications (requirements.txt, library versions, CUDA version) are provided beyond mentioning vLLM and Ray."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. The implementation details are described at the design level but not enough for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as point estimates (e.g., '2-2.5x higher request arrival rates', '82% GPU resource saving') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Aegaeon outperforms baselines based on comparing numbers directly without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., '2-2.5x higher request arrival rates', '1.5-9x more goodput', 'GPU utilization from 13.3%~33.9% to 48.1%', 'GPUs reduced from 1,192 to 213 (82% saving)'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of models, workload configurations, or experimental runs chosen. The evaluation covers specific setups but does not discuss why these are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance across runs is reported. Results appear to be single-run. CDFs are shown for auto-scaling latency but no variance across repeated experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "ServerlessLLM, ServerlessLLM+ (with oracle SJF scheduling), and MuxServe are used as baselines (§7.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ServerlessLLM (OSDI 2024), MuxServe (ICML 2024), and BlitzScale (OSDI 2025) are all recent works. The baselines are state-of-the-art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§7.3 breaks down effectiveness of individual components: latency breakdown (Figure 14), auto-scaling speed (Figure 15), memory fragmentation (Figure 16). §5 shows progressive optimization stages (T0→T1→T2→T3) with component reuse removing 80%, then further reductions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "SLO attainment, goodput, GPU utilization, auto-scaling latency, KV cache transfer overhead, and memory fragmentation are all reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper about GPU resource management; human evaluation of system outputs is not relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper, not a machine learning evaluation. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by number of models, RPS, dataset type (ShareGPT, ShareGPT-ix2, ShareGPT-ox2), SLO strictness levels, hardware configurations (H800 vs A10), and model sizes (7B-72B)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 13(c) shows that under the strictest SLO (0.2x), Aegaeon no longer outperforms MuxServe. The paper acknowledges 'stricter SLOs reduce slack time and limit GPU pooling opportunities.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that under 0.2x SLO strictness, Aegaeon no longer outperforms MuxServe (Figure 13(c)), and that ServerlessLLM+ can actually underperform ServerlessLLM when SJF causes excessive auto-scaling."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '2-2.5x higher request arrival rates', '1.5-9x more goodput', '97% auto-scaling overhead reduction', and '82% GPU resource saving (1,192→213)' are all supported by results in §7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about token-level auto-scaling improving performance. These are supported by ablation-style analysis (progressive optimization stages) and controlled comparisons against baselines with the same workloads."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests on specific hardware (H800, A10), model sizes (6B-72B), and datasets (ShareGPT variants). §7.4 explicitly tests generalization to different hardware and model sizes. Claims are generally scoped to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. The paper does not consider confounds such as whether the improvements are specific to the workload distribution or vLLM backend."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model families are listed (Qwen, Llama, InternLM, Yi) with parameter counts but no specific model versions or checkpoint identifiers are provided."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper about serving infrastructure; it does not use prompting as part of its methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: MAX_GPSIZE=8, QMAX=4s, TTFT=10s, TBT=100ms, correction factor β=0.625, prefill/decode instance split (6/10). Latency model constants (C1-C5) are described as profiled."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a systems paper about GPU resource management."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset construction is described: ShareGPT is used as base, ShareGPT-ix2 and ShareGPT-ox2 are created by scaling input/output lengths by 2x. Workloads are synthesized with scaled Poisson processes and random sampling (§7.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges scope boundaries: §7.2 notes MuxServe has advantages 'in extremely latency-sensitive scenarios' (Figure 13(c)). The system targets sporadic multi-model workloads specifically, not single-model high-throughput scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, logs, or production traces are made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Production workload statistics are described: 779 models, 167.6M requests, 30K GPUs, arrival rate distributions (Figure 1). Experimental setup details hardware, models, and datasets (§7.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from production workloads and standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The experimental pipeline is documented: workload synthesis from ShareGPT with Poisson processes, dataset variants created by scaling lengths, and the production deployment setup (213 H20 GPUs, 47 models) is described in §7.5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists National Key Research and Development Program of China (Grant 2022YFB4500700), Fundamental Research Funds for Central Universities, and NSFC grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University and Alibaba Group. The system is deployed at Alibaba Cloud Model Studio."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are from Alibaba Group, and the system is deployed at Alibaba Cloud. Alibaba has a financial interest in demonstrating GPU cost savings for their cloud platform. Government grants are independent but the corporate affiliation creates a non-independent interest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Alibaba employees evaluating a system deployed at Alibaba Cloud represents an undeclared potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate pre-trained model capabilities on benchmarks. It evaluates a serving system's resource efficiency."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests serving infrastructure, not model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model capability benchmarks are evaluated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systems paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports GPU reduction from 1,192 to 213 GPUs (82% saving) in production. Auto-scaling latencies are reported via CDF (Figure 15). GPU utilization improved from 13.3-33.9% to 48.1%."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Testbed is specified: 2 nodes with 16 NVIDIA H800 80GB GPUs, 2TB DDR5, 192 Intel Xeon CPUs. Production deployment uses 213 H20 GPUs. Implementation is 5,700 lines of Python and CUDA/C++."
       }
     }
   },
   "claims": [
     {
       "claim": "Aegaeon sustains 2-2.5x higher request arrival rates compared to ServerlessLLM",
       "evidence": "Figures 11 and 12 show SLO attainment across varying model counts and RPS. At RPS=0.1, Aegaeon supports up to 70 models vs ~35 for ServerlessLLM. At RPS=0.5, the advantage is 2.5x (§7.2).",
       "supported": "strong"
     },
     {
       "claim": "Aegaeon achieves 1.5-9x more goodput compared to existing solutions",
       "evidence": "Figures 11(c) and 12 show goodput comparisons with vertical lines indicating maximum sustainable load at 90% SLO target (§7.2).",
       "supported": "strong"
     },
     {
       "claim": "Aegaeon reduces auto-scaling overhead by 97%",
       "evidence": "§5 details progressive optimizations from T0 (26.9s for 13B model) to T3, with component reuse removing 80% and explicit memory management + KV cache synchronization removing the remainder. Figure 15 shows sub-second scaling in practice.",
       "supported": "strong"
     },
     {
       "claim": "Production deployment reduces GPU count from 1,192 to 213 (82% saving)",
       "evidence": "§7.5 describes beta deployment at Alibaba Cloud Model Studio serving 47 models (1.8B-72B). Figure 18 shows GPU utilization improvement from 13.3-33.9% to 48.1% over 70 hours.",
       "supported": "strong"
     },
     {
       "claim": "Token-level auto-scaling supports up to 7 models per GPU",
       "evidence": "Figure 11(a) shows Aegaeon supporting 70 models on 10 decoding instances at RPS=0.1 with >90% SLO attainment (§7.2).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Aegaeon introduces token-level auto-scaling for multi-model LLM serving, enabling preemptive model switching at per-token granularity rather than per-request. Through component reuse, explicit memory management, and fine-grained KV cache synchronization, it reduces auto-scaling overhead by 97%. In controlled experiments, Aegaeon sustains 2-2.5x higher request rates and 1.5-9x more goodput than ServerlessLLM and MuxServe. Production deployment at Alibaba Cloud serving 47 models reduced GPU requirements from 1,192 to 213, an 82% saving.",
   "red_flags": [
     {
       "flag": "Company evaluating own production system",
       "detail": "Alibaba Group employees evaluate Aegaeon, which is deployed at Alibaba Cloud Model Studio. The production deployment results (§7.5) come from their own infrastructure with no independent verification possible."
     },
     {
       "flag": "No variance or repeated trials",
       "detail": "All experimental results appear to be single runs with no error bars, confidence intervals, or variance across repeated experiments."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section despite being a systems paper with production claims."
     },
     {
       "flag": "No artifact release",
       "detail": "Neither code nor production workload traces are released, making independent reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "Foundational LLM serving system (vLLM) used as Aegaeon's execution backend; central to LLM inference infrastructure."
     },
     {
       "title": "ServerlessLLM: Low-Latency Serverless Inference for Large Language Models",
       "authors": ["Yao Fu", "Leyang Xue"],
       "year": 2024,
       "relevance": "Primary baseline for auto-scaling LLM serving; represents state-of-the-art in serverless LLM inference."
     },
     {
       "title": "MuxServe: Flexible Spatial-Temporal Multiplexing for Multiple LLM Serving",
       "authors": ["Jiangfei Duan", "Runyu Lu"],
       "year": 2024,
       "relevance": "Primary baseline for multiplexing-based multi-model serving on GPUs."
     },
     {
       "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
       "authors": ["Yinmin Zhong", "Shengyu Liu"],
       "year": 2024,
       "relevance": "Pioneered prefill-decoding disaggregation adopted by Aegaeon; key architectural influence."
     },
     {
       "title": "BlitzScale: Fast and Live Large Model Autoscaling with O(1) Host Caching",
       "authors": ["Dingyan Zhang", "Haotian Wang"],
       "year": 2025,
       "relevance": "Contemporary auto-scaling system for LLM serving, addressing cold start optimization."
     },
     {
       "title": "Splitwise: Efficient Generative LLM Inference Using Phase Splitting",
       "authors": ["Pratyush Patel", "Esha Choukse"],
       "year": 2024,
       "relevance": "Another approach to disaggregating prefill and decoding phases in LLM inference."
     },
     {
       "title": "AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving",
       "authors": ["Zhuohan Li", "Lianmin Zheng"],
       "year": 2023,
       "relevance": "Statistical multiplexing approach to multi-model serving that Aegaeon improves upon."
     },
     {
       "title": "Mooncake: Trading More Storage for Less Computation — A KVCache-centric Architecture for Serving LLM Chatbot",
       "authors": ["Ruoyu Qin", "Zheming Li"],
       "year": 2025,
       "relevance": "KV cache management architecture for LLM serving; related industrial deployment at scale."
     }
   ]
 }