ai-research-survey

scan.json (25627B)

---

 {
   "paper": {
     "title": "Aladdin: Joint Placement and Scaling for SLO-Aware LLM Serving",
     "authors": ["Chengyi Nie", "Rodrigo Fonseca", "Zhenhua Liu"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2405.06856"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or any other code repository URL is provided in the paper. No mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available ShareGPT_V3_unfiltered_cleaned_split dataset (reference [27]) and publicly available Llama2 models (reference [28]). The datasets and models used are all public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware (A100 80GB GPUs with PCIe, V100 32GB GPUs with NVLink, Intel Xeon processors, RAM amounts) in Section 6.1 but provides no software environment details — no Python version, no library versions, no requirements.txt, no Dockerfile."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. The algorithms are described but there are no step-by-step instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes prediction intervals (shaded areas) in Figures 6a, 6b, 7a, 7b, and 8 for the performance model validation. Section 6.2 states 'The shaded area is the prediction interval, which represents the estimation of the range in which future observations are likely to fall.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Aladdin outperforms baselines (up to 71% cost reduction, 51% SLO improvement) but uses no statistical significance tests — comparisons are based solely on comparing point estimates from single simulation/experiment runs."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout. For example, Section 6.3: 'Aladdin reduces the SLO violation rate by up to 3.5X,' 'improves the SLO attainment rate by up to 51% compared with JSQ,' 'reduces the P99 ATGT by up to 40%.' Section 6.4: 'reduces the LLM serving cost by up to 71% and 40% compared with the default vLLM and JSQ.' Baseline and treatment values are visible in figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why specific workload sizes, arrival rates, or numbers of requests were chosen. The experiments use ShareGPT prompts and Poisson arrival distributions but no rationale for why these quantities are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The end-to-end experiments (Sections 6.3 and 6.4) report single-run results with no standard deviation, no variance across multiple runs, and no indication of how many times experiments were repeated. Only the performance model validation includes prediction intervals."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: default vLLM, Aladdin optimal worker + JSQ (ablation), Power-of-Two algorithm, and JSQ. These are described in Section 6.1 under 'Baselines.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include vLLM (2023), DistServe (2024), Splitwise (2023), and the Power-of-Two algorithm from TetriInfer (2024). These are contemporary systems at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 includes an ablation: 'Aladdin best worker + JSQ' isolates the contribution of optimal worker configuration from request placement. The comparison between full Aladdin and this ablated version shows the value of the placement algorithm specifically."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: SLO attainment rate, P99 ATGT (average token generation time), GPU count required, TTFT, and scheduling overhead (ms). See Sections 6.3, 6.4, and 6.5."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper about GPU resource scheduling for LLM inference. Human evaluation of system outputs is not relevant — the claims are about latency, cost, and SLO attainment."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems paper, not a machine learning model evaluation. There is no train/test split concept applicable to the scheduling system evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size (7b, 13b, 70b), testbed (A100, V100), and inference setting (default continuous batching vs split-phase). Figures 9-12 show per-model, per-testbed results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses prediction errors and how the system handles them. Section 6.3 discusses when baselines outperform (low arrival rates on V100: 'when the arrival rate is low, the P99 ATGT of baseline default vLLM outperforms the performance with optimal worker configuration'). The paper also discusses scheduling overhead limitations for high-demand scenarios."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that at low arrival rates, the default vLLM outperforms optimal worker configuration on V100 (Section 6.3). It also acknowledges the scheduling overhead becomes a problem at high arrival rates (Section 6.5) and discusses output length prediction errors as an inherent limitation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Aladdin 'reduces the serving cost of a single model by up to 71% for the same SLO level compared with the baselines.' This is supported by Section 6.4/Figure 11, which shows up to 71% GPU reduction compared to default vLLM. The 'millions of dollars per year' claim is not quantified in the paper body but the GPU reduction percentages are substantiated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation design — comparing full Aladdin against 'Aladdin best worker + JSQ' isolates the contribution of request placement. The performance improvements are attributed to specific components (worker configuration, request placement, re-balancing) with controlled comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only Llama2 models (7b, 13b, 70b) on two testbeds but the title and abstract make general claims about 'LLM Serving' without bounding to Llama2. Section 5.3 notes 'Aladdin is specifically designed for single-model serving' but the broader framing does not adequately bound to the tested models and hardware."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, the improvements could be partially due to the specific workload characteristics of ShareGPT rather than general properties of LLM inference, but this is not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Llama2-chat 70b,' 'Llama2-chat 13b,' 'Llama2-chat 7b' (Table 2), and 'Llama2-70b-chat-hf,' 'Llama2-13b-chat-hf,' 'Llama2-7b-chat-hf' (Table 3, Figure 8). These are specific HuggingFace model identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not prompt LLMs for any task — it uses ShareGPT prompts as workload input to test the scheduling system. The prompts are the workload, not part of the methodology that needs reproducing."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0 (Section 6.1), maximum output token limit is 2048, SLO values are specified in Table 2, gamma=0.5 for the scheduling bound, and theta is discussed as a hyperparameter. Poisson distribution parameters for arrival rates are varied and shown in figures."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an agentic AI system. It is a scheduling system for LLM inference, with no agentic scaffolding involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 documents workload preparation: 'we first collect the prompts from users of ShareGPT_V3_unfiltered_cleaned_split dataset, then submit the prompts follows a Poisson distribution. The outputs are generated by each evaluated model with a temperature of 0 and a maximum output token limit of 2048.' For simulation: prompt lengths from ShareGPT with output length predicted from CDF."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Some limitations are mentioned inline (e.g., scheduling overhead at high arrival rates, no request migration support, no cold start consideration) but they are scattered, not collected in a dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper mentions some inline limitations but does not systematically analyze threats — e.g., the reliance on Poisson arrival patterns, single workload dataset (ShareGPT), homogeneous GPU assumption, and single-model serving limitation are not framed as threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Several scope boundaries are stated: 'Aladdin is specifically designed for single-model serving' (Section 5.3), 'we consider homogeneous GPUs in this paper' (Section 4.1), 'we focus on predicting the minimal GPU required for the varying arrival rate without considering the cold start problem and the switching cost' (Section 5.2), and 'we consider the tensor parallelism distributed inference' (Section 4.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (latency measurements, trace logs, simulation outputs) are made available. The ShareGPT dataset is public but the experimental results themselves are not independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes the workload generation process: prompts from ShareGPT, Poisson arrival distribution, temperature 0, max output 2048 tokens. Section 6.2 describes how performance model traces are collected on the testbeds."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public benchmark dataset (ShareGPT) and hardware testbed measurements."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: collect ShareGPT prompts -> generate outputs with Llama2 models -> record input/output lengths -> use as workload for Poisson arrival simulation. For performance modeling: collect batch inference traces -> fit linear models -> validate predictions against observations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. One author is from Azure Research - Systems (Microsoft), suggesting potential corporate funding, but this is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Chengyi Nie (Stony Brook University), Rodrigo Fonseca (Azure Research - Systems), Zhenhua Liu (Stony Brook University). The Microsoft/Azure affiliation is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "An author is from Azure Research - Systems (Microsoft), a major cloud provider that sells GPU compute for LLM inference. A system reducing GPU costs could be commercially valuable to Azure. No statement is made about funder independence. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It evaluates a scheduling system for LLM inference. The LLMs are used as workload generators, not evaluated for their knowledge or capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates a scheduling system, not a model's benchmark performance. There is no concern about train/test overlap for the scheduling algorithm."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate pre-trained model capability on benchmarks. The ShareGPT data is used as workload, not as a capability benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The primary metric of the paper is GPU count required for SLO-guaranteed serving, which directly measures inference cost. Section 6.4 reports GPU numbers needed at various arrival rates. Section 6.5 reports scheduling overhead in milliseconds (Figure 13)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified: 4 A100 80GB GPUs per machine with Intel Xeon Platinum 8380 and 512GB RAM, and 4 V100 32GB GPUs per machine with Intel Xeon Gold 6230 and 128GB RAM (Section 6.1). The simulation scales up to 25 req/s arrival rates."
       }
     }
   },
   "claims": [
     {
       "claim": "Aladdin reduces the serving cost of a single model by up to 71% for the same SLO level compared with baselines.",
       "evidence": "Section 6.4, Figure 11 shows GPU reduction of up to 71% compared to default vLLM with continuous batching across different models and testbeds.",
       "supported": "strong"
     },
     {
       "claim": "Aladdin reduces GPU number required for the decode phase by up to 60% compared with JSQ in split-phase inference.",
       "evidence": "Section 6.4, Figure 12 shows decode-phase GPU reduction of up to 60% vs JSQ and 49% vs Power-of-Two across models and testbeds.",
       "supported": "strong"
     },
     {
       "claim": "Aladdin improves the SLO attainment rate by up to 51% compared with JSQ with optimal workers.",
       "evidence": "Section 6.3, Figure 9a shows 51% improvement for LlaMa2-13b on A100 testbed. However, the improvement for 70b is only up to 19% due to having only two workers.",
       "supported": "moderate"
     },
     {
       "claim": "The performance models predict prefill and decode latency with maximum error less than 10%.",
       "evidence": "Section 6.2: prefill latency prediction error less than 4% (Figure 6), decode latency prediction error less than 5% (Figure 7), KV cache prediction error less than 1% (Figure 8).",
       "supported": "strong"
     },
     {
       "claim": "The scheduling overhead is acceptable for request arrival rates up to about 25 requests per second.",
       "evidence": "Section 6.5, Figure 13 shows scheduling overhead under 50ms at 25 req/s. However, overhead grows significantly beyond this point, reaching ~500ms at 100 req/s.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Aladdin is a co-adaptive scheduler for LLM inference that jointly optimizes request placement and GPU resource scaling with SLO awareness. It models prefill and decode latency as linear functions of input length, batch size, and context length, achieving prediction errors under 10%. By formulating request placement as a multi-dimensional bin packing problem with a best-fit heuristic, and dynamically configuring worker GPU counts, Aladdin reduces serving costs by up to 71% compared to default vLLM and 60% compared to split-phase inference baselines for SLO-guaranteed service across Llama2 models of varying sizes on A100 and V100 testbeds.",
   "red_flags": [
     {
       "flag": "No code release",
       "detail": "The system is not open-sourced and no code repository is provided, making independent verification or reproduction of results impossible."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "End-to-end experiments (Sections 6.3, 6.4) report single-run results with no standard deviation, variance, or confidence intervals. It is unclear whether the improvements are consistent across multiple runs with different random seeds for Poisson arrivals."
     },
     {
       "flag": "Simulated arrival patterns only",
       "detail": "All experiments use Poisson arrival distributions. The paper acknowledges there is 'no available trace of LLM inference that includes the arrival time of each request.' Real-world arrival patterns may differ significantly from Poisson (e.g., bursty traffic), and the system's performance under such conditions is unknown."
     },
     {
       "flag": "Cherry-picked 'up to' numbers",
       "detail": "Claims use 'up to X%' phrasing (up to 71%, up to 60%, up to 51%), which reports best-case improvements. The improvements vary substantially by model size and testbed — for example, the SLO attainment improvement for 70b is only 19% vs the 51% reported for 13b."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section, despite having several notable constraints: Llama2-only evaluation, homogeneous GPU assumption, single-model serving only, no cold start consideration, Poisson-only arrival patterns."
     },
     {
       "flag": "Azure affiliation without COI disclosure",
       "detail": "One author is from Azure Research (Microsoft), a major cloud compute provider. A system that reduces GPU requirements for LLM serving has direct commercial relevance to Azure, but no conflict-of-interest statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "vLLM is a foundational LLM serving system with PagedAttention that Aladdin builds on top of, directly relevant to LLM inference efficiency."
     },
     {
       "title": "Splitwise: Efficient Generative LLM Inference Using Phase Splitting",
       "authors": ["Pratyush Patel", "Esha Choukse", "Chaojie Zhang"],
       "year": 2023,
       "relevance": "Split-phase inference approach that separates prefill and decode, a key baseline and complementary technique for LLM serving."
     },
     {
       "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
       "authors": ["Yinmin Zhong", "Shengyu Liu", "Junda Chen"],
       "year": 2024,
       "relevance": "Goodput-optimized LLM serving system that disaggregates prefill and decoding phases, a direct baseline for Aladdin."
     },
     {
       "title": "Orca: A Distributed Serving System for Transformer-Based Generative Models",
       "authors": ["Gyeong-In Yu", "Joo Seong Jeong", "Geon-Woo Kim"],
       "year": 2022,
       "relevance": "Introduced continuous batching for LLM serving, foundational to all modern LLM inference systems."
     },
     {
       "title": "Sarathi: Efficient LLM Inference by Piggybacking Decodes with Chunked Prefills",
       "authors": ["Amey Agrawal", "Ashish Panwar", "Jayashree Mohan"],
       "year": 2023,
       "relevance": "Chunked prefill technique for balancing prefill and decode in LLM inference, complementary to cluster-level scheduling."
     },
     {
       "title": "Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve",
       "authors": ["Amey Agrawal", "Nitin Kedia", "Ashish Panwar"],
       "year": 2024,
       "relevance": "Addresses throughput-latency tradeoffs in LLM inference serving, closely related to SLO-aware scheduling."
     },
     {
       "title": "Inference without Interference: Disaggregate LLM Inference for Mixed Downstream Workloads",
       "authors": ["Cunchen Hu", "Heyang Huang", "Liangliang Xu"],
       "year": 2024,
       "relevance": "Disaggregated LLM inference for mixed workloads with Power-of-Two scheduling, a direct baseline for request placement."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "relevance": "The Llama2 model family used as the primary evaluation workload across all Aladdin experiments."
     },
     {
       "title": "Efficient Interactive LLM Serving with Proxy Model-Based Sequence Length Prediction",
       "authors": ["Haoran Qiu", "Weichao Mao", "Archit Patke"],
       "year": 2024,
       "relevance": "Proxy model-based output length prediction for LLM serving, related to Aladdin's output length prediction component."
     },
     {
       "title": "ExeGPT: Constraint-Aware Resource Scheduling for LLM Inference",
       "authors": ["Hyungjun Oh", "Kihong Kim", "Jaemin Kim"],
       "year": 2024,
       "relevance": "Constraint-aware resource scheduling for LLM inference at ASPLOS, directly related to resource management for LLM serving."
     },
     {
       "title": "Andes: Defining and Enhancing Quality-of-Experience in LLM-Based Text Streaming Services",
       "authors": ["Jiachen Liu", "Zhiyu Wu", "Jae-Won Chung"],
       "year": 2024,
       "relevance": "QoE-focused LLM serving system, related to Aladdin's ATGT SLO metric for quality of experience."
     },
     {
       "title": "FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU",
       "authors": ["Ying Sheng", "Lianmin Zheng", "Binhang Yuan"],
       "year": 2023,
       "relevance": "CPU-GPU offloading approach for LLM inference throughput, an alternative approach to efficient LLM serving."
     }
   ]
 }