ai-research-survey

scan.json (26627B)

---

 {
   "paper": {
     "title": "Apt-Serve: Adaptive Request Scheduling on Hybrid Cache for Scalable LLM Inference Serving",
     "authors": ["Shihong Gao", "Xin Zhang", "Yanyan Shen", "Lei Chen"],
     "year": 2025,
     "venue": "Proc. ACM Manag. Data (SIGMOD)",
     "arxiv_id": "2504.07494",
     "doi": "10.1145/3725394"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/eddiegaoo/Apt-Serve (footnote 4, Section 6.1). The appendix with the greedy scheduling algorithm is also linked from the repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three publicly available datasets: ShareGPT, HumanEval, and LongBench. These are standard public benchmarks referenced with citations. The ultra-long context experiments use WikiText, Arxiv, and BookCorpus, also public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies hardware (NVIDIA A100 40GB GPUs with NVLink) and mentions FP16 precision, FlashAttention, NCCL, and building on vLLM, but does not provide a requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While a GitHub link is given, the paper itself does not contain a 'Reproducing Results' section or specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (SLO attainment percentages, throughput multipliers). No confidence intervals, error bars, or ± notation appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '2.3× higher average request rates') but uses no statistical significance tests. Comparisons are based solely on comparing numbers from single experimental runs."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports effect sizes as multipliers with baseline context, e.g., 'Apt-Serve achieves up to 8.8× improvement in effective throughput compared to the state-of-the-art' and detailed per-dataset, per-model comparisons (Section 6.3) such as '2.3×, 2.0×, and 1.9× higher request rates on average compared to vLLM, Sarathi-Serve, and DeepSpeed-FastGen.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 500 or 1,000 sampled requests per dataset but never justifies why these specific numbers were chosen or discusses whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. It is unclear whether experiments were run multiple times. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: vLLM, Sarathi-Serve, and DeepSpeed-FastGen (Section 6.2). These are described as 'representative state-of-the-art LLM inference serving systems.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All three baselines are from 2023-2024 and represent recent state-of-the-art systems: vLLM (SOSP 2023), Sarathi-Serve (OSDI 2024), and DeepSpeed-FastGen (2024). These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 presents ablation studies: (1) Apt-Serve with KV-only cache vs. hybrid cache (Table 4), and (2) FCFS scheduling vs. adaptive scheduling (Table 5). Both isolate the contribution of individual components across multiple settings."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports SLO attainment rate (%), effective throughput (maximum sustainable request rate), TTFT and P99 TBT distributions (CDFs in Figure 10), and scheduling algorithm execution time (Table 6)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper evaluating infrastructure performance (throughput, latency). Human evaluation of system outputs is not relevant to the claims about scheduling and cache management."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning model evaluation paper. The system is evaluated on serving workloads, not on train/test splits. The concept of held-out test sets does not apply to systems benchmarking."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (ShareGPT, HumanEval, LongBench), per model size (13B, 30B, 66B), per SLO attainment threshold (90% and 60%), and per metric (TTFT vs. TBT) in Figures 8-12 and Tables 4-5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.6 discusses that Apt-Serve may cause 'a small fraction of requests (10%) experience starvation' due to the SLO-aware fallback mechanism, with high tail latency shown in Figure 10. Section 6.7 acknowledges challenges in ultra-long context scenarios where both systems struggle with TBT SLO."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that on HumanEval, improvement is 'moderate' compared to other datasets (Section 6.3), explains why (shorter cache lifetimes, lower variance), and notes that in ultra-long context scenarios on BookCorpus with Yi-6B-200K, 'both systems struggle to exceed 60% TBT SLO attainment' (Section 6.7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 8.8× improvement in effective throughput.' Section 6.3 confirms: 'At 60% SLO attainment, Apt-Serve can handle 4.9×, 4.4×, and 4.3× higher request rates on average, with maximum values up to 8.8×, 7.9×, and 7.5×.' The 8.8× is the peak value at 60% SLO attainment comparing to vLLM."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about the two factors (KV cache memory and FCFS scheduling) causing TTFT SLO violations are supported by controlled ablation studies. Section 6.5 isolates each component: Table 4 shows the effect of hybrid cache with adaptive scheduling held constant, and Table 5 shows the effect of adaptive scheduling with hybrid cache held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds claims to the tested settings: specific model families (OPT 13B/30B/66B, LLaMA3-8B, Yi-6B), specific datasets, and specific hardware configurations (A100 GPUs). Section 6.7 frames generalization experiments as 'further investigate Apt-Serve's generalization capabilities' rather than claiming universal applicability. Future work explicitly mentions extending to multi-instance scenarios."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the observed improvements. For example, it does not consider whether the gains might be partially attributable to differences in implementation quality, tuning of baselines, or workload-specific properties beyond what is analyzed. There is no threats-to-validity discussion."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: OPT-13B, OPT-30B, OPT-66B (Section 6.2, Table 2), LLaMA3-8B-Instruct262K, and Yi-6B-200K (Section 6.7). These are well-defined open-source model checkpoints, not API-based models where version ambiguity is a concern."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting in the sense of prompt engineering for LLM capabilities. The workloads consist of request datasets (ShareGPT conversations, HumanEval code completions, LongBench summaries) fed directly to models for inference serving evaluation. No prompt design is involved."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: FP16 precision (Section 6.2), SLO thresholds (Table 3), Poisson/Gamma distribution parameters for request arrivals, coefficient of variation values, request rates, memory pool block sizes, and the preprocessing cost of ~30 seconds for the coefficient ρ (Section 4.2). Hardware configurations are in Table 2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use agentic scaffolding. It is a systems paper about inference serving infrastructure, not an agentic AI system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 describes: 1,000 requests randomly sampled from each dataset, Poisson distribution for arrival generation, Figure 7 shows input/output length distributions. LongBench sequence lengths are limited due to OPT's 2048-token positional embedding constraint (footnote 5). Ultra-long dataset sampling is described in Section 6.7 with statistics in Table 7."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. The conclusion mentions future work (multi-instance scenario, integrating with disaggregated architectures) but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential concerns such as the limited model family (mostly OPT), synthetic workload generation, or the gap between simulated and production workloads."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While future work mentions multi-instance scenarios and disaggregated architectures, there is no explicit articulation of boundaries such as 'our results do not address production-scale deployment with heterogeneous GPUs' or similar scoping statements."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (e.g., per-request latency traces, scheduling decisions, memory utilization logs) are not made available. Only aggregated results in figures and tables are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 describes data collection: 1,000 requests randomly sampled from each public dataset (ShareGPT, HumanEval, LongBench), request arrivals generated via Poisson distribution at various rates. Section 6.7 describes ultra-long context dataset sampling with statistics in Table 7."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (ShareGPT, HumanEval, LongBench, WikiText, Arxiv, BookCorpus)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from datasets to experimental results is documented: sampling requests from public datasets → generating arrival patterns via Poisson/Gamma distributions → running on specified hardware with specified models → measuring SLO attainment. Distribution statistics are shown in Figure 7 and Table 7."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive funding disclosure in the Acknowledgments section, including NSFC, Hong Kong RGC, Guangdong Province grants, Microsoft Research Asia Collaborative Research Grant, HKUST-Webank joint research lab, and multiple other sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: HKUST, HKUST(GZ), and Shanghai Jiao Tong University. This is an academic paper evaluating open-source systems, not a product evaluation by its creators."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding sources include government grants (NSFC, Hong Kong RGC, Guangdong Province), university funds, and Microsoft Research Asia. The paper evaluates open-source inference serving systems (vLLM, Sarathi-Serve, DeepSpeed-FastGen) and none of the funders have a direct financial stake in the specific outcome of which system performs better."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. There is no declaration regarding patents, equity, or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It evaluates inference serving system performance (throughput, latency) using models as workload generators. The models' knowledge of benchmark content is irrelevant to the claims."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for the same reason: the paper evaluates systems infrastructure performance, not model knowledge or capability. Whether the model has seen HumanEval problems does not affect the serving throughput measurements."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The benchmarks (ShareGPT, HumanEval, LongBench) are used to generate request workloads with representative length distributions, not to evaluate model quality. Contamination of model training data is irrelevant to the serving performance claims."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. This is a systems evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports the execution time of the scheduling algorithm (0.3ms to 10.8ms for 50-1600 requests), and compares it to the practical computation time ('a single decode iteration with 50 requests using the OPT-13B model takes approximately 120 milliseconds'). The ~30-second preprocessing cost for coefficient ρ is also stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies hardware (A100 GPUs, configurations in Table 2) but does not state the total computational budget, such as total GPU hours spent on experiments, wall-clock time for the full experimental suite, or total energy/cost."
       }
     }
   },
   "claims": [
     {
       "claim": "Apt-Serve achieves up to 8.8× improvement in effective throughput compared to state-of-the-art inference serving systems.",
       "evidence": "Section 6.3: At 60% SLO attainment, Apt-Serve handles up to 8.8× higher request rates than vLLM on ShareGPT with OPT-13B. Average improvements are 4.9×, 4.4×, and 4.3× over vLLM, Sarathi-Serve, and DeepSpeed-FastGen.",
       "supported": "strong"
     },
     {
       "claim": "KV cache memory consumption limits batch size and is a primary bottleneck for TTFT SLO attainment.",
       "evidence": "Section 3.1, Figure 2: At 3 req/s, the system hits batch size limit for over 80% of serving time, causing SLO attainment to drop sharply. TTFT SLO attainment declines correlate with time spent at batch size limit.",
       "supported": "strong"
     },
     {
       "claim": "FCFS scheduling produces suboptimal batch compositions and even random scheduling outperforms it.",
       "evidence": "Section 3.2, Figure 4: Random scheduling consistently achieves higher SLO attainment than FCFS across all request rates. Per-request analysis (Figures 4b, 4c) shows FCFS causes clustered TTFT violations.",
       "supported": "strong"
     },
     {
       "claim": "The hybrid-cache-based scheduling problem is NP-hard and the greedy solution has an approximation ratio of 2.",
       "evidence": "Section 5 establishes NP-hardness by reduction from 0-1 knapsack (Definition 1). The proof of the approximation ratio is deferred to an online appendix at the GitHub repository.",
       "supported": "moderate"
     },
     {
       "claim": "Apt-Serve's hybrid cache consistently improves SLO attainment over KV-cache-only under the same adaptive scheduling.",
       "evidence": "Section 6.5, Table 4: Across all tested request rates and burstiness levels on ShareGPT and LongBench, hybrid cache achieves equal or higher SLO attainment. Gains are more prominent at higher request rates and burstier loads.",
       "supported": "strong"
     },
     {
       "claim": "Apt-Serve achieves up to 7.5× higher SLO attainment under bursty request conditions compared to baselines.",
       "evidence": "Section 6.4, Figure 9: Under varying burstiness (Gamma distribution with different CV values), Apt-Serve consistently outperforms baselines with widening gaps as burstiness increases.",
       "supported": "strong"
     },
     {
       "claim": "Apt-Serve's optimizations are complementary to other techniques like Sarathi-Serve's chunked prefill.",
       "evidence": "Section 6.7, Figure 11: Apt-Serve-S (Apt-Serve atop Sarathi-Serve) outperforms both standalone Apt-Serve and standalone Sarathi-Serve, demonstrating complementary benefits.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Apt-Serve introduces a hybrid cache scheme combining KV cache with a memory-efficient hidden cache (storing input hidden state vectors at half the memory cost) and an adaptive runtime scheduling mechanism that dynamically optimizes batch composition. The hybrid-cache-based scheduling problem is formulated as an NP-hard optimization with a greedy 2-approximation solution. Extensive experiments on three datasets (ShareGPT, HumanEval, LongBench) and three OPT model sizes (13B-66B) show Apt-Serve achieves up to 8.8× higher effective throughput than vLLM, with consistent gains across varying request patterns and model scales. The approach is demonstrated to be complementary to existing optimizations like Sarathi-Serve's chunked prefill.",
   "red_flags": [
     {
       "flag": "No error bars or repeated runs",
       "detail": "All experimental results appear to be from single runs with no confidence intervals, standard deviations, or indications of variance across repetitions. For a systems paper with stochastic workloads (Poisson/Gamma arrivals), variance across runs could be substantial."
     },
     {
       "flag": "No limitations or threats-to-validity section",
       "detail": "The paper lacks any discussion of limitations. Potential concerns include: reliance primarily on the OPT model family, synthetic workload generation vs. production traces, limited GPU types tested (only A100), and the gap between controlled benchmarks and real deployment scenarios."
     },
     {
       "flag": "Peak improvement number in abstract",
       "detail": "The abstract highlights 'up to 8.8× improvement' which is the peak value at 60% SLO attainment on one dataset (ShareGPT) with OPT-13B vs. vLLM. Average improvements are more modest (2.3× at 90% SLO attainment). While supported by data, leading with the peak number overstates typical gains."
     }
   ],
   "cited_papers": [
     {
       "title": "Efficient memory management for large language model serving with pagedattention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang", "Ying Sheng", "Lianmin Zheng", "Cody Hao Yu", "Joseph Gonzalez", "Hao Zhang", "Ion Stoica"],
       "year": 2023,
       "relevance": "vLLM is the primary baseline and the system Apt-Serve is built upon; foundational work on PagedAttention for KV cache management in LLM serving."
     },
     {
       "title": "Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve",
       "authors": ["Amey Agrawal", "Nitin Kedia", "Ashish Panwar", "Jayashree Mohan", "Nipun Kwatra", "Bhargav Gulavani", "Alexey Tumanov", "Ramachandran Ramjee"],
       "year": 2024,
       "relevance": "State-of-the-art baseline system for LLM inference serving with chunked prefill and prefill-decode coalescing batching."
     },
     {
       "title": "Deepspeed-fastgen: High-throughput text generation for llms via mii and deepspeed-inference",
       "authors": ["Connor Holmes", "Masahiro Tanaka", "Michael Wyatt"],
       "year": 2024,
       "arxiv_id": "2401.08671",
       "relevance": "State-of-the-art baseline for LLM inference serving with prefill-decode coalescing batching techniques."
     },
     {
       "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
       "authors": ["Yinmin Zhong", "Shengyu Liu", "Junda Chen", "Jianbo Hu", "Yibo Zhu", "Xuanzhe Liu", "Xin Jin", "Hao Zhang"],
       "year": 2024,
       "relevance": "Disaggregated serving approach that distributes prefill and decode across separate GPUs; complementary direction to Apt-Serve."
     },
     {
       "title": "Orca: A distributed serving system for Transformer-Based generative models",
       "authors": ["Gyeong-In Yu", "Joo Seong Jeong", "Geon-Woo Kim", "Soojeong Kim", "Byung-Gon Chun"],
       "year": 2022,
       "relevance": "Introduced iteration-level batching for LLM serving, a foundational technique used by all systems in this paper."
     },
     {
       "title": "Towards efficient generative large language model serving: A survey from algorithms to systems",
       "authors": ["Xupeng Miao", "Gabriele Oliaro", "Zhihao Zhang"],
       "year": 2023,
       "arxiv_id": "2312.15234",
       "relevance": "Comprehensive survey of LLM inference serving optimizations covering the algorithmic and systems landscape."
     },
     {
       "title": "Mooncake: Kimi's KVCache-centric Architecture for LLM Serving",
       "authors": ["Ruoyu Qin", "Zheming Li", "Weiran He"],
       "year": 2024,
       "arxiv_id": "2407.00079",
       "relevance": "KV-cache-centric serving architecture that addresses effective throughput optimization, directly related to the cache management problem."
     },
     {
       "title": "Llumnix: Dynamic Scheduling for Large Language Model Serving",
       "authors": ["Biao Sun", "Ziming Huang", "Hanyu Zhao"],
       "year": 2024,
       "relevance": "Dynamic scheduling for LLM serving using KV cache migration across instances, addressing similar scheduling challenges."
     },
     {
       "title": "Splitwise: Efficient generative llm inference using phase splitting",
       "authors": ["Pratyush Patel", "Esha Choukse", "Chaojie Zhang"],
       "year": 2024,
       "relevance": "Phase-splitting approach to LLM serving that separates prefill and decode, a complementary optimization direction."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark used as one of the three evaluation datasets in Apt-Serve's experiments."
     },
     {
       "title": "Efficient LLM Scheduling by Learning to Rank",
       "authors": ["Yichao Fu", "Siqi Zhu", "Runlong Su", "Aurick Qiao", "Ion Stoica", "Hao Zhang"],
       "year": 2024,
       "relevance": "Learning-based LLM scheduling that predicts output length ranks; discussed as complementary to Apt-Serve's scheduling approach."
     }
   ]
 }