ai-research-survey

scan.json (25645B)

---

 {
   "paper": {
     "title": "Equinox: Holistic Fair Scheduling in Serving Large Language Models",
     "authors": [
       "Zhixiang Wei",
       "James Yen",
       "Jingyi Chen",
       "Ziyang Zhang",
       "Zhibai Huang",
       "Chen Chen",
       "Xingzi Yu",
       "Yicheng Gu",
       "Chenggang Wu",
       "Yun Wang",
       "Mingyuan Xia",
       "Jie Wu",
       "Hao Wang",
       "Zhengwei Qi"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.16646",
     "doi": "10.48550/arXiv.2508.16646"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Equinox proposes a dual-counter fairness framework (UFC + RFC) for LLM serving that separates user-facing metrics (latency, weighted tokens) from operator-facing metrics (GPU utilization, throughput). A Mixture of Prediction Experts (MoPE) with 3 specialized experts reduces L1 token prediction error from 80 to 33 versus single proxy models. Evaluations on ShareGPT and LMSYS traces across S-LoRA, vLLM, and SGLang show up to 1.3x throughput improvement, 60% lower TTFT, and 13% higher Jain's fairness index versus VTC.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes Equinox as 'open-source' (Section 1, Section 4) but provides no repository URL, GitHub link, or archive link anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: LMSYS Chatbot Arena (lmsys-chat-1m) [47] and ShareGPT [38]. These are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (A100-80GB, 8×A100-40GB cluster, Intel Xeon Gold 5218, 256GB DRAM), but no software environment details are given — no Python version, no library versions, no requirements.txt or Dockerfile."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. The paper mentions ~1000 lines of Python atop existing systems but provides no way to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported on any of the main results. Figures show point estimates only (e.g., Figures 9-13)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like '1.3× higher throughput' and '60% lower TTFT' are made by comparing raw numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context throughout: '1.3× higher throughput', '60% lower TTFT', '13% higher fairness', '42% worst-case service gap reduction', '86% average gap reduction' (Section 1, Section 7)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for workload sizes (e.g., 1280 prompts for SGLang, 1000 requests per client for vLLM). No discussion of whether these are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. It is unclear whether experiments were run multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "FCFS and VTC are used as baselines throughout all experiments (Section 7.1). For prediction, a single proxy model [31] is the baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "VTC (Sheng et al., OSDI 2024 [39]) is the state-of-the-art fair scheduling approach for LLM serving. The proxy model baseline is from 2024 [31]. These are recent and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 presents an ablation study (Table 1) isolating MoPE's contribution by comparing Equinox and VTC with different predictors (Single, MoPE, Oracle)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: service rate, service difference, TTFT (P50/P90), end-to-end latency, Jain's fairness index, throughput, GPU utilization (Section 7.1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems/scheduling paper; human evaluation of system outputs is not relevant to the claims about fairness and throughput."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MoPE is trained on LMSYS dataset and tested on the unseen ShareGPT dataset (Section 7.3), demonstrating generalization across datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-client (Figures 9c, 10d, 12c), per-system (S-LoRA, vLLM, SGLang in Figure 13), per-metric (UFC/RFC components in Figure 5), and per-scenario (balanced, stochastic, overload, dynamic)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or scenarios where Equinox underperforms are discussed. All results show improvements."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 15 shows the fairness-throughput tradeoff: at α=0.9, throughput drops 20%. The paper also notes VTC+Single performs worse than baseline VTC (Table 1), and acknowledges multi-node deployment needs 'additional engineering efforts' (Section 7.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 1.3× throughput (Figure 9/17), 60% lower TTFT (Figure 9a), 13% higher fairness (Figure 13), and 94% GPU utilization (Figure 9b) are supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about Equinox improving fairness. The ablation study (Table 1) with controlled single-variable manipulation (same predictor, different scheduler; same scheduler, different predictor) adequately supports the causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to 'redefine fairness in multi-tenant LLM serving' (Section 9) broadly, but evaluates only on Llama-2-7b and Llama-2-70b models. No evaluation on other model families (e.g., Mistral, GPT) or model sizes. The title and abstract make broad claims not bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, could the improvements be due to the specific workload characteristics rather than the general framework design? No robustness checks against different workload distributions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the gap between token count as a proxy and actual fairness (Section 1, Figure 1), arguing that token-level metrics are inadequate proxies for the multi-dimensional fairness they measure. The HF metric is clearly defined in terms of what it measures (Section 3.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: Llama-2-7b for synthetic traces, Llama-2-70b for real-world traces (Section 7.1). BERT-base for MoPE experts (Section 6)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems/scheduling paper that does not use prompting as part of its method. The LLM requests come from existing trace datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: α=0.7, β=0.3 (Section 7.6), δ=0.1 (Section 3.1), 3 MoPE experts with boundaries at 33rd/66th/99th percentiles (<53, 53-210, >210 tokens), BF16 precision (Section 6), TP=8 for multi-GPU (Section 7.1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a scheduling system, not an agentic AI system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "MoPE training data partitioning is documented: router trained on up to 120k samples from LMSYS, data split by output length boundaries at 33rd/66th/99th percentiles (Section 6, Figure 7c). Workload configurations are described in detail for each experiment (Sections 7.2-7.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The paper briefly mentions multi-node deployment needing 'additional engineering efforts' (Section 7.5) but this is not a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. No mention of specific concerns about generalizability beyond the tested models, workloads, or hardware."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what settings or configurations the results do NOT apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (logs, traces, measurements) is made available. Only aggregated results in figures and tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Synthetic workload parameters are fully specified (client rates, input/output lengths). Real-world traces are from public datasets (LMSYS, ShareGPT) with setup described in Section 7.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmark traces."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The MoPE training pipeline is documented: training dataset → router training → dataset splitting → expert training (Figure 8). Online prediction pipeline is also documented. Workload generation is specified per experiment."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Shanghai Jiao Tong University, UltraRISC Shanghai, China Telecom Cloud Computing Research Institute, Stevens Institute of Technology."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. One author is from UltraRISC and another from China Telecom, which could have commercial interest in LLM serving."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. One author is affiliated with UltraRISC (a commercial entity), which may have financial interest in LLM serving technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a scheduling system, not a pre-trained model's capability on a benchmark. The LLMs (Llama-2) are used as workload generators, not as the subject of evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates scheduling fairness, not model knowledge. No benchmark contamination concern exists for the scheduling evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — scheduling system evaluation, not model capability evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "MoPE overhead is reported: 0.02ms router overhead, 4.5ms total MoPE inference, less than 1% of average prompt latency (Section 6, Figure 7d). Memory usage is reported in Figure 7b."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed but total compute budget (GPU hours for training MoPE, total experiment time) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity. Stochastic arrival experiments use Poisson processes but it's unclear if multiple seeds were tested."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "α/β sensitivity is studied (Figure 15) but the search budget for other hyperparameters (δ, MoPE expert boundaries, training parameters) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The selection of α=0.7, β=0.3 is justified by the sensitivity analysis in Section 7.6/Figure 15, showing the fairness-throughput tradeoff. The 3-expert MoPE configuration is justified by Figure 7a-b."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite many comparisons across scenarios, metrics, and systems."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement Equinox and compare it against their own implementations of FCFS and VTC. No acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of performance at matched compute budgets. MoPE adds prediction overhead (reported as <1%) but the training cost of MoPE vs. simpler predictors is not compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 and Section 2 provide extensive analysis of why token count is an inadequate proxy for fairness (Figures 1-2), directly addressing construct validity of the VTC metric. The paper argues for multi-dimensional fairness as the construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The system evaluates scheduling, not model capability through scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this evaluates a scheduling system, not model knowledge. The LLM is a workload generator, not the evaluation subject."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — scheduling evaluation does not involve prediction tasks where feature leakage is a concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "MoPE is trained on LMSYS data and tested on the unseen ShareGPT dataset (Section 7.1, 7.3), ensuring train-test independence for the prediction component."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model capability benchmark where leakage detection would be relevant."
       }
     }
   },
   "claims": [
     {
       "claim": "Equinox achieves up to 1.3× higher throughput than FCFS and VTC",
       "evidence": "Figure 9c and Appendix Figure 17a show per-client service rates 1.3× higher than baselines in the balanced load scenario (Section 7.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "Equinox reduces time-to-first-token latency by up to 60% versus VTC",
       "evidence": "Figure 9a shows response time comparison. The 60% figure appears to come from the balanced load scenario (Section 7.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "Equinox achieves 13% higher Jain's fairness index compared to FCFS and VTC",
       "evidence": "Figure 13 shows cross-system fairness comparison: Equinox achieves 0.80-0.90 vs FCFS 0.60-0.72 and VTC 0.66-0.76 across S-LoRA, vLLM, and SGLang.",
       "supported": "moderate"
     },
     {
       "claim": "MoPE reduces L1 prediction error for output tokens from 80 to 33 versus single proxy models",
       "evidence": "Figure 7a shows expert specialization results with 3 experts achieving mean L1 error of 33 vs baseline of 80 (Section 6).",
       "supported": "moderate"
     },
     {
       "claim": "Equinox cuts worst-case service gaps by 42% and average gaps by 86% versus VTC",
       "evidence": "Table 1 ablation study: Equinox+MoPE max diff 865.62 vs VTC 1505.13 (42% reduction), avg diff 150.64 vs 1106.31 (86% reduction).",
       "supported": "strong"
     },
     {
       "claim": "MoPE introduces less than 1% latency overhead",
       "evidence": "Figure 7d shows MoPE total inference 4.5ms vs average prompt latency 2400ms. Router overhead is 0.02ms (Section 6).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "None of the main results include error bars, confidence intervals, or variance across runs. For a systems paper with stochastic workloads (Poisson arrivals), this is a significant omission — it's impossible to assess result stability."
     },
     {
       "flag": "Open-source claim without URL",
       "detail": "The paper repeatedly describes Equinox as 'open-source' (abstract, Section 1, Section 4) but provides no repository URL or archive link. This makes the claim unverifiable."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section despite making broad claims about 'redefining fairness in multi-tenant LLM serving.'"
     },
     {
       "flag": "Narrow model evaluation for broad claims",
       "detail": "All evaluations use only Llama-2 models (7b and 70b). The paper claims general applicability to LLM serving but does not test on other model families, architectures, or sizes."
     },
     {
       "flag": "Cherry-picked improvement numbers",
       "detail": "The abstract highlights 'up to 1.3×' throughput and 'up to 60%' TTFT reduction. These are best-case numbers from specific scenarios. The SGLang results show more modest improvements (up to 30% TTFT, ~25% throughput at high RPS only)."
     }
   ],
   "cited_papers": [
     {
       "title": "Fairness in Serving Large Language Models",
       "authors": ["Ying Sheng", "Shiyi Cao", "Dacheng Li"],
       "year": 2024,
       "relevance": "VTC paper — the primary baseline for fair scheduling in LLM serving, published at OSDI 2024."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "doi": "10.1145/3600006.3613165",
       "relevance": "vLLM paper — foundational LLM serving system with PagedAttention, one of the platforms Equinox is evaluated on."
     },
     {
       "title": "SGLang: Efficient Execution of Structured Language Model Programs",
       "authors": ["Lianmin Zheng", "Liangsheng Yin", "Zhiqiang Xie"],
       "year": 2024,
       "relevance": "SGLang serving system — another platform Equinox is evaluated on, published at NeurIPS 2024."
     },
     {
       "title": "Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve",
       "authors": ["Amey Agrawal", "Nitin Kedia", "Ashish Panwar"],
       "year": 2024,
       "relevance": "Introduces chunked prefills for LLM serving, addressing prefill-decode interference — a key related system optimization."
     },
     {
       "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
       "authors": ["Yinmin Zhong", "Shengyu Liu", "Junda Chen"],
       "year": 2024,
       "arxiv_id": "2401.09670",
       "relevance": "Proposes disaggregated prefill/decode serving — directly relevant to the prefill-decode bifurcation problem Equinox addresses."
     },
     {
       "title": "Orca: A Distributed Serving System for Transformer-Based Generative Models",
       "authors": ["Gyeong-In Yu", "Joo Seong Jeong", "Geon-Woo Kim"],
       "year": 2022,
       "relevance": "Introduced continuous batching for LLM serving at OSDI 2022 — foundational technique Equinox builds upon."
     },
     {
       "title": "Efficient Interactive LLM Serving with Proxy Model-based Sequence Length Prediction",
       "authors": ["Haoran Qiu", "Weichao Mao", "Archit Patke"],
       "year": 2024,
       "arxiv_id": "2404.08509",
       "relevance": "Proxy model prediction baseline that MoPE is compared against for token length prediction."
     },
     {
       "title": "LoongServe: Efficiently Serving Long-Context Large Language Models with Elastic Sequence Parallelism",
       "authors": ["Bingyang Wu", "Shengyu Liu", "Yinmin Zhong"],
       "year": 2024,
       "doi": "10.1145/3694715.3695948",
       "relevance": "Addresses long-context LLM serving challenges, relevant to the workload diversity that Equinox's fairness framework handles."
     },
     {
       "title": "LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2024,
       "arxiv_id": "2309.11998",
       "relevance": "Primary dataset used for MoPE training and evaluation traces in Equinox experiments."
     },
     {
       "title": "Past-Future Scheduler for LLM Serving under SLA Guarantees",
       "authors": ["Ruihao Gong", "Shihao Bai", "Siyu Wu"],
       "year": 2025,
       "doi": "10.1145/3676641.3716011",
       "relevance": "SLA-aware LLM scheduling at ASPLOS 2025 — related approach to quality-of-service in LLM serving."
     }
   ]
 }