ai-research-survey

scan.json (19888B)

---

 {
   "scan_version": 2,
   "active_modules": [],
   "paper": {
     "title": "Routing, Cascades, and User Choice for LLMs",
     "authors": ["Rafid Mahmood"],
     "year": 2026,
     "venue": "ICLR 2026",
     "arxiv_id": "2602.09902"
   },
   "methodology_tags": ["theoretical"],
   "key_findings": "In a Stackelberg game between an LLM provider (two models) and a user, optimal routing policies almost always collapse to simple threshold-based static routing with no cascading. Cascading is rarely optimal except in a narrow regime where models are sufficiently differentiated in net value. Provider-user misalignment arises when their model rankings by utility versus cost-of-pass disagree. When churn penalties are low, providers are incentivized to throttle latency to reduce costs, harming user utility.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or plotting scripts are provided. The paper has computational figures (heatmaps in Figures 3-5) but no code is released."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. There is no dataset — all results are derived from closed-form mathematical expressions."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. Numerical examples are deterministic computations from closed-form equations, requiring no special computational environment."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. The proofs are self-contained in the paper and appendices. Figures can be reproduced from the stated equations."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper with no experiments. All results are mathematical proofs, not empirical measurements."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or comparative empirical claims. Results are proven theorems."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments. Theoretical characterizations are exact, not estimated."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or samples. All results are analytical."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs. Results are deterministic mathematical derivations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No system is evaluated against baselines. The paper characterizes equilibria of a game-theoretic model."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation with baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. The paper is a mathematical analysis of a game."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation. The paper characterizes user utility and provider cost as optimization objectives within the model."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. Human evaluation is irrelevant to the mathematical claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No data or test sets. All results are analytical."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation to break down by category. The paper does characterize results across parameter regimes (Theorems 2-5), but this is part of the theoretical analysis, not evaluation design."
       },
       "failure_cases_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No system is being evaluated. The paper does characterize misalignment and throttling as adverse outcomes within the model, but this is part of the theoretical contribution, not failure analysis of an evaluated system."
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments. The paper's finding that cascading is rarely optimal is part of the theoretical characterization, not a negative experimental result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported by formal proofs: static routing optimality (Theorems 3-5), misalignment gap (Proposition 1), and throttling incentives (Proposition 2). Each claim maps directly to a proven result."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about incentives and optimal strategies (e.g., 'providers are incentivized to throttle latency'). These are derived rigorously from the formal game-theoretic model via mathematical proof, which is adequate for causal inference within the model's assumptions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'single-provider, single-user interactions.' Section 7 explicitly bounds the scope: two models only, subscription pricing (not pay-per-call), observable provider strategy, stationary abandonment policy, i.i.d. success probabilities, and known parameters."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses what happens under imperfect information ('beliefs drive the same threshold logic'). Section 4.2 discusses extension to heterogeneous tasks with task-dependent parameters. Section 7 discusses how different modeling choices (hidden strategies, non-stationary policies, non-i.i.d. success) might affect results."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper with no empirical measurements. The paper defines its quantities (utility, cost, net value) precisely within the model."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No LLM models are used in experiments. The paper is a mathematical analysis. LLM usage for writing assistance (Appendix A) is not part of the experimental setup."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its methodology. It is a theoretical analysis."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters. The model parameters (pi, ci, ti, V, P) are symbolic and their ranges are analyzed theoretically."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used. Purely theoretical paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data collected or preprocessed. All results are derived analytically."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 contains a substantive limitations paragraph: 'We note several limitations and next steps' followed by four specific limitations — two-model restriction, subscription-only pricing, observable strategy assumption, and i.i.d. success assumption."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'our analysis is of two models,' 'our subscription framing abstracts away per-query monetary prices,' 'we assume that a user can observe the provider cascade strategy,' 'we treat the per-pass success as i.i.d.' These are not generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what the results do NOT show: does not cover more than two models, pay-per-call pricing, hidden routing strategies, adaptive abandonment, non-i.i.d. success, or unknown parameters requiring online learning."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No empirical data to verify."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection. All results are mathematical derivations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants and no data source beyond the mathematical model."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline. Results are analytical."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was funded partially by the Natural Sciences and Engineering Research Council of Canada.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is listed as 'University of Ottawa & NVIDIA.' NVIDIA is a major LLM infrastructure provider, and this is transparently disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a public Canadian funding agency with no commercial interest in LLM routing outcomes. While the author is affiliated with NVIDIA, NVIDIA is not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The author is affiliated with NVIDIA, which has commercial interests in LLM inference infrastructure, but this potential conflict is not explicitly declared beyond the affiliation line."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on a benchmark. It is a purely theoretical game-theoretic analysis."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on any benchmark. Purely theoretical."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation. Purely theoretical."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely theoretical paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No system with inference costs to report."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No significant computation beyond plotting."
       }
     }
   },
   "claims": [
     {
       "claim": "User patience is governed by the sign of net value ξi = Vpi − ti: when both models are value-dominated or both latency-dominated, user behavior is static regardless of routing policy.",
       "evidence": "Theorems 1 and 2 (Section 4.1) provide formal proofs. Theorem 1 shows q*(2,s) = 1{ξ2 < 0}. Theorem 2 shows q*(1,s) = 0 when ξ1, ξ2 ≥ 0 and q*(1,s) = 1 when ξ1, ξ2 ≤ 0.",
       "supported": "strong"
     },
     {
       "claim": "Optimal routing almost always collapses to simple threshold-based static policies with no cascading.",
       "evidence": "Theorems 3-5 (Section 4.2) characterize the provider-optimal policy. Theorem 3 shows no cascading is optimal when both models share the same sign. Theorem 4 shows cascading is optimal only in specific regimes for ξ1 < 0 < ξ2. Theorem 5 provides sufficient conditions for static policies when ξ1 > 0 > ξ2. Figure 4 visualizes that cascading is only optimal in narrow parameter regions.",
       "supported": "strong"
     },
     {
       "claim": "Misalignment between provider and user arises when the user's model utility ranking and the provider's cost-of-pass ranking disagree.",
       "evidence": "Proposition 1 (Section 5) formally characterizes when ΔU = 0 for each regime. Condition 1 shows alignment requires sign(c1/p1 − c2/p2) = sign(ξ2/p2 − ξ1/p1). Figure 5 visualizes misalignment gaps.",
       "supported": "strong"
     },
     {
       "claim": "When churn penalties are sufficiently low (P ≤ min{c1/p1, c2/p2}), providers benefit from throttling latency, reducing user utility.",
       "evidence": "Proposition 2 (Section 6) proves that setting ĥt_i > Vpi yields J*_post ≤ J*_pre. Full proof in Appendix C.4. Figure 5 (Right) visualizes the gain from throttling as a function of P and cost-of-pass difference.",
       "supported": "strong"
     },
     {
       "claim": "Cascading is valuable only when the two models are sufficiently differentiated in user net value.",
       "evidence": "Theorem 4 shows cascading is optimal only in the regime ξ1 < 0 < ξ2 with c1/p1 < c2/p2 and P > P2. Theorem 5 shows cascading when ξ1 > 0 > ξ2 only for low P. When models are undifferentiated (both value- or latency-dominated), Theorem 3 shows cascading is never optimal.",
       "supported": "strong"
     }
   ],
   "red_flags": [],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational work on LLM routing to reduce cost while maintaining performance, directly extended by this paper's game-theoretic analysis."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "relevance": "Proposes routing by predicting task difficulty and allocating to cheaper models; this paper adds the user-behavior dimension missing from such algorithms."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "relevance": "Learns routing policies from preference data; relevant to the survey's coverage of LLM inference optimization and routing."
     },
     {
       "title": "A Unified Approach to Routing and Cascading for LLMs",
       "authors": ["Jasper Dekoninck", "Maximilian Baader", "Martin Vechev"],
       "year": 2025,
       "relevance": "Proposes unified optimization for routing and cascading; this paper's finding that cascading is rarely optimal contrasts with that approach."
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "authors": ["Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li"],
       "year": 2024,
       "arxiv_id": "2403.12031",
       "relevance": "Benchmark for evaluating LLM routing systems, relevant to the survey's coverage of evaluation methodology for routing."
     },
     {
       "title": "Cost-of-Pass: An Economic Framework for Evaluating Language Models",
       "authors": ["Mehmet Hamza Erol", "Batu El", "Mirac Suzgun"],
       "year": 2025,
       "arxiv_id": "2504.13359",
       "relevance": "Formalizes cost-of-pass metric for LLM evaluation, directly used and extended in this paper's provider cost model."
     },
     {
       "title": "Pricing and Competition for Generative AI",
       "authors": ["Rafid Mahmood"],
       "year": 2024,
       "relevance": "Prior work by the same author on LLM pricing economics; this paper extends the single-model framework to routing between two models."
     },
     {
       "title": "The Economics of Large Language Models: Token Allocation, Fine-tuning, and Optimal Pricing",
       "authors": ["Dirk Bergemann", "Alessandro Bonatti", "Alex Smolin"],
       "year": 2025,
       "relevance": "Economic analysis of LLM deployment including user engagement and pricing, relevant to the survey's coverage of LLM operational economics."
     },
     {
       "title": "Human-AI Interactions and Societal Pitfalls",
       "authors": ["Francisco Castro", "Jian Gao", "Sébastien Martin"],
       "year": 2023,
       "arxiv_id": "2309.10448",
       "relevance": "Models multi-round human-AI interaction dynamics; foundational for this paper's user abandonment model."
     },
     {
       "title": "The Burden of Interactive Alignment with Inconsistent Preferences",
       "authors": ["Ali Shirali"],
       "year": 2025,
       "relevance": "Studies interactive alignment as a Stackelberg game with costly user signals; closely related game-theoretic approach to human-AI interaction."
     },
     {
       "title": "Large Language Model Cascades with Mixture of Thoughts Representations for Cost-efficient Reasoning",
       "authors": ["Murong Yue", "Jie Zhao", "Min Zhang"],
       "year": 2023,
       "arxiv_id": "2310.03094",
       "relevance": "Proposes LLM cascade architectures for cost-efficient reasoning, relevant to the cascading mechanisms analyzed in this paper."
     },
     {
       "title": "Game Theory Meets Large Language Models: A Systematic Survey",
       "authors": ["Haoran Sun", "Yusen Wu", "Yukun Cheng"],
       "year": 2025,
       "relevance": "Survey of game theory applied to LLMs; contextualizes this paper within the broader game-theoretic LLM literature."
     }
   ]
 }