ai-research-survey

scan.json (27488B)

---

 {
   "paper": {
     "title": "Budget-Aware Agentic Routing via Boundary-Guided Training",
     "authors": [
       "Caiqi Zhang",
       "Menglin Xia",
       "Xuchao Zhang",
       "Daniel Madrigal",
       "Ankur Mallick",
       "Samuel Kessler",
       "Victor Rühle",
       "Saravan Rajmohan"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.21227"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D states 'Codes will be released upon acceptance.' This is a promise of future release, not actual release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three publicly available benchmarks: SciWorld, ALFWorld (via AgentGym framework), and AppWorld. All are established public datasets with citations and available implementations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using Qwen2.5-1.5B and vLLM on an A100 GPU, and OpenAI APIs, but does not provide a reproducible environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix D provides pipeline details (hyperparameters, training stages), there are no step-by-step reproduction instructions, no README, and no scripts. The code is not yet released."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'All experiments are averaged over three random seeds' (Section 5.1) but does not report confidence intervals, error bars, or standard deviations in any table or figure. Only point estimates are presented."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes claims like 'BoPO consistently improves the efficiency frontier' and 'BoPO generally outperforms baselines' but provides no statistical significance tests (no p-values, no bootstrap tests, no paired comparisons)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute success rates and costs for all methods across benchmarks (e.g., 'reaches the high-performance regime (> 63%) at a cost of $0.125'), and Table 1 provides per-setting breakdowns. Results are presented in context with baseline performance, allowing effect magnitude assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test set sizes are reported in Table 3 (200 for ALFWorld and SciWorld, 168 for AppWorld) and 3 random seeds are used, but there is no justification for why these sizes are sufficient for the claims made, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running experiments over 3 random seeds (Section 5.1), no standard deviations, variance, or any spread measures are reported in the tables or figures. Only averaged point estimates are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes seven baselines: Always-small/large, Random routing, Zero-shot GPT-5 router, Cascading (FrugalGPT-style), Single-turn (RouteLLM-style), SFT-only, and Vanilla RL. Additional baselines (First-Large) are included for hard-budget evaluation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work: RouteLLM (Ong et al., 2025), FrugalGPT (Chen et al., 2024), xRouter/Router-R1 (Qian et al., 2025; Zhang et al., 2025). The Zero-shot GPT-5 router is also included as an upper bound. These are all contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 and Figure 2 present a component-wise ablation study on SciWorld, testing: SFT-only, Vanilla RL, no stratified sampling, no reference advantage, and no difficulty bonus. Each component's contribution to the efficiency frontier is isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are reported: Success Rate (SR) and Average Cost per task ($). Additionally, Large Model Usage percentage (Use%) is reported in Table 1 for hard-budget experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper about model routing. The system's outputs are routing decisions evaluated against task success metrics in automated environments. Human evaluation is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 explicitly separates train and test sizes: ALFWorld (2420 train / 200 test), SciWorld (2120 train / 200 test), AppWorld (105 train / 168 test). Results are reported on these held-out test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (SciWorld, ALFWorld, AppWorld) in Figure 1 and Table 1. Figure 3 provides a per-difficulty-category breakdown (Easy/Hard/Intractable) of budget allocation. Table 1 provides per-K breakdowns for hard-budget experiments."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses the limitation of static trade-off parameters, noting 'a small λ is too aggressive for low budgets (leading to early exhaustion), while a large λ is overly conservative for high budgets.' Section 6 discusses Vanilla RL's failure to identify high-leverage steps and policy collapse. Appendix F analyzes cost inversions where the small model performs worse."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that BoPO with large λ shows a 'K=15 drop in AppWorld' (Section 5.2), that 'policies trained with a fixed soft-budget trade-off do not fully adapt to different hard caps' (Section 1), and that Vanilla RL collapses. Appendix A explicitly discusses limitations of static risk profiles."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims BoPO 'improves the efficiency frontier, matching strong routing baselines at substantially lower cost while demonstrating generalization to strict inference-time budget constraints.' These are supported by Figure 1 (Pareto frontiers) and Table 1 (hard-budget results). The abstract also honestly notes the generalization is partial."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through its ablation study (Section 6, Figure 2), which removes components one at a time (stratified sampling, reference advantage, difficulty bonus) to show each component's causal contribution. The ablation design uses controlled single-variable manipulation, which is adequate for causal inference about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper scopes claims to three specific benchmarks, discusses testing with two model pairs (GPT-4.1 pair and Llama-3.1 pair), and Appendix A ('Model Scope') explicitly states they focused on binary choice and acknowledges the need for N-model generalization. The limitations section discusses static risk profiles and incomplete hard-budget adaptation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses that GPT-5's strong performance 'likely stems from GPT-5's strong instruction-following.' The paper considers why Vanilla RL fails (sparsity of reward signal) and why First-Large is competitive (importance of early-stage trajectory correctness). Section 6 discusses why BoPO may not fully generalize to hard budgets (static λ)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names: 'GPT-4.1 mini' as Msmall, 'GPT-4.1' as Mlarge, and 'Qwen2.5-1.5B (instruction-tuned)' as the router. However, no API snapshot dates or specific version strings (e.g., gpt-4.1-2025-04-14) are provided. 'GPT-4.1' and 'GPT-4.1 mini' are marketing names without version identifiers. For the generalization experiment, 'Llama-3.1-8B-Instruct' and 'Llama-3.1-72B-Instruct' are specified, which are versioned model names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G (Tables 4 and 5) provides the full prompt templates for both soft-budget and hard-budget routing modes, including the system prompt configuration, history formatting, and decision format. The templates include placeholders but the fill values are defined (task description, model name, action/observation previews, budget state)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides detailed hyperparameters: global batch size 64, BoSFT learning rate 2×10⁻⁵ with cosine decay and 3% warmup, BoPO learning rate 1×10⁻⁶, GRPO group size G=8, KL coefficient β=0.04, reward weights rsuccess=1.0, rhard=0.5, λ∈{0.1,0.3,0.5,0.7,0.9}, N=20 stratified trajectories, K=5 profiling trials."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the agent scaffolding: 'Our agent is instantiated with ReAct-style prompting (Yao et al., 2022)' (Section 5.1). The routing mechanism, decision-token generation, budget-constrained decoding, and history truncation are all described in detail. The router's role in the agentic pipeline is specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D documents the pipeline: difficulty profiling (K=5 trials per task), task partitioning into Easy/Hard/Intractable, stratified trajectory synthesis (N=20), expert trajectory selection, batch composition (70% Hard task oversampling), router input truncation (most recent 2k tokens), and decision-token masking."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A is titled 'Limitations and Future Work' and provides substantive discussion across three subsections: Training Overhead, Model Scope, and Static Risk Profiles."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A discusses specific limitations: (1) training overhead is a one-time cost amortized over inference, (2) the binary model choice is a simplification of N-model selection, (3) static λ does not adapt to varying hard-budget caps. Section 5.2 also identifies specific cases where the approach underperforms (BoPO Large λ at K=15 in AppWorld)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A ('Model Scope') explicitly states: 'We intentionally focused this study on the binary choice between a cost-efficient model (Msmall) and a high-capability model (Mlarge) to isolate the fundamental economic trade-off.' The paper also states BoPO 'is not the final solution for hard-budget routing' (Section 5.2) and that future work should condition on budget state."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw trajectory data, routing decisions, or per-task results are released. Only aggregated metrics are reported in figures and tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix D describe how data is collected: profiling with K=5 trials per task for difficulty taxonomy, N=20 stratified trajectories for SFT data synthesis, and evaluation on held-out test sets from three benchmarks. The evaluation protocol (max steps, termination conditions) is specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (SciWorld, ALFWorld, AppWorld)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides the full pipeline: (i) difficulty profiling with K=5 trials, (ii) Boundary-Guided SFT with stratified sampling (N=20), expert trajectory selection, Hard task oversampling (~70%), (iii) BoPO online optimization, (iv) Budget-Constrained Decoding at inference. Dataset statistics with train/test splits are in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The affiliations show Microsoft and University of Cambridge, but no funding disclosures are made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Caiqi Zhang at University of Cambridge, and the remaining authors (Menglin Xia, Xuchao Zhang, Daniel Madrigal, Ankur Mallick, Samuel Kessler, Victor Rühle, Saravan Rajmohan) at M365 Research, Microsoft."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Seven of eight authors are Microsoft employees (M365 Research). The paper evaluates routing using OpenAI models (GPT-4.1, GPT-4.1 mini) which are integrated into Microsoft products. While the paper does not evaluate Microsoft-specific products, Microsoft has a financial interest in efficient LLM routing for its agentic AI platforms. No funding disclosure is made to assess independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are provided. The paper lacks a conflicts of interest declaration."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for GPT-4.1, GPT-4.1 mini, Qwen2.5-1.5B, or Llama-3.1 models. While the router is fine-tuned, the underlying agent models' training cutoffs are not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark tasks (SciWorld, ALFWorld, AppWorld) could have appeared in the training data of GPT-4.1 or other models. These are public benchmarks that could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SciWorld (2022), ALFWorld (2021), and AppWorld (2024) are all publicly available benchmarks. Models like GPT-4.1 may have been trained on data containing these benchmarks. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average cost per task in USD is a primary metric, reported in Figure 1 and Table 1. Section 5.1 provides OpenAI pricing: GPT-4.1 is $2.00/$8.00 per 1M tokens (in/out), GPT-4.1 mini is $0.40/$1.60. The router overhead is analyzed at <0.2 seconds and <2% of total latency (Section 6)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for training. While the paper mentions using A100 GPUs for vLLM inference of the router, the total GPU hours for difficulty profiling (K=5 trials per task), stratified sampling (N=20 trajectories per hard task), SFT training, and BoPO training are not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "BoPO consistently improves the cost-success efficiency frontier across all three benchmarks, matching the performance of always using the large model at a fraction of the cost.",
       "evidence": "Figure 1 shows Pareto efficiency frontiers for SciWorld, ALFWorld, and AppWorld. BoPO curves are consistently closer to the top-left (high success, low cost) compared to all baselines. In ALFWorld, BoPO reaches >63% success at $0.125.",
       "supported": "strong"
     },
     {
       "claim": "Boundary-Guided Training prevents the 'always-small' policy collapse that occurs with vanilla RL under sparse rewards.",
       "evidence": "Figure 1 and Section 5.2 show that Vanilla RL baseline performance is comparable to Random routing in ALFWorld and AppWorld. The ablation study (Figure 2) shows removing the reference-guided advantage causes destabilization and catastrophic forgetting.",
       "supported": "strong"
     },
     {
       "claim": "BoPO generalizes to hard-budget constraints via Budget-Constrained Decoding without hard-budget-specific training.",
       "evidence": "Table 1 shows BoPO performance under strict K∈{5,10,15} constraints. BoPO generally outperforms baselines, though the paper acknowledges First-Large is 'a remarkably strong baseline' and that static λ limits adaptability.",
       "supported": "moderate"
     },
     {
       "claim": "BoPO strategically allocates budget to Hard tasks where compute is most productive.",
       "evidence": "Figure 3 shows BoPO concentrates 52.2% of cost on Hard tasks vs. 35.7% for Random and 37.3% for Vanilla RL, while reducing Easy task spending to 21.8%.",
       "supported": "strong"
     },
     {
       "claim": "The approach generalizes to different model pairs (open-source Llama-3.1 models).",
       "evidence": "Figure 4 in the appendix shows BoPO maintains its effectiveness on SciWorld with Llama-3.1-8B-Instruct / Llama-3.1-72B-Instruct. However, this is shown on only one benchmark.",
       "supported": "moderate"
     },
     {
       "claim": "The routing overhead is negligible (<2% of total trajectory latency).",
       "evidence": "Section 6 provides a latency analysis estimating the router (Qwen2.5-1.5B generating single-token decisions) adds <0.2 seconds vs. 11-12 seconds baseline per task, based on standardized throughput figures.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes Budget-Aware Agentic Routing (BAAR), which formulates step-wise model selection in long-horizon agentic workflows as a sequential decision process. The Boundary-Guided Training approach (BoSFT + BoPO) uses task difficulty taxonomies and boundary reference policies to stabilize RL under sparse rewards. Across SciWorld, ALFWorld, and AppWorld, BoPO consistently dominates the cost-success Pareto frontier, matching expensive always-large baselines at substantially reduced costs. The method also transfers to hard-budget constraints via Budget-Constrained Decoding, though static trade-off parameters limit full adaptability to varying budget caps.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reported despite multi-seed experiments",
       "detail": "The paper runs all experiments over 3 random seeds but reports only averaged point estimates. No standard deviations, confidence intervals, or error bars are shown in any table or figure. This makes it impossible to assess result stability or whether differences between methods are statistically meaningful."
     },
     {
       "flag": "No benchmark contamination analysis",
       "detail": "SciWorld (2022) and ALFWorld (2021) are publicly available benchmarks that may be in the training data of GPT-4.1. No contamination analysis is performed, and model training cutoff dates are not stated."
     },
     {
       "flag": "Code not released",
       "detail": "The paper states 'Codes will be released upon acceptance' but no code is available at the time of evaluation. This prevents independent verification of the results."
     },
     {
       "flag": "Router cost excluded from metrics",
       "detail": "Section 5.1 and Appendix E acknowledge that the computational cost of the routing module is excluded from reported metrics. While the paper argues this is justified for the lightweight trained router (<2% overhead), the Zero-shot GPT-5 router baseline — which incurs substantial cost — is compared on the same cost axis, making the comparison asymmetric."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Foundational work on cost-efficient LLM routing via cascading strategies, a key baseline in this paper."
     },
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang", "Tianhao Wu", "Joseph E. Gonzalez", "M. Waleed Kadous", "Ion Stoica"],
       "year": 2025,
       "relevance": "Trains a preference classifier for single-turn LLM routing, used as a baseline in this paper."
     },
     {
       "title": "xRouter: Training cost-aware LLMs orchestration system via reinforcement learning",
       "authors": ["Cheng Qian", "Zhengming Liu", "Srinivas Kokane", "Anurag Prabhakar", "Junwei Qiu", "Hao Chen", "Zhaoyang Liu", "Heng Ji", "Weizhu Yao", "Shelby Heinecke", "Silvio Savarese", "Caiming Xiong", "Huan Wang"],
       "year": 2025,
       "arxiv_id": "2510.08439",
       "relevance": "RL-based multi-round LLM orchestration system, closely related work for agentic routing."
     },
     {
       "title": "Router-R1: Teaching LLMs multi-round routing and aggregation via reinforcement learning",
       "authors": ["Hao Zhang", "Tao Feng", "Jiaxuan You"],
       "year": 2025,
       "relevance": "RL-trained router for multi-round model routing and aggregation, directly comparable approach."
     },
     {
       "title": "MasRouter: Learning to route LLMs for multi-agent systems",
       "authors": ["Yanbo Yue", "Guoxin Zhang", "Biao Liu", "Guowei Wan", "Kai Wang", "Deheng Cheng", "Yuhui Qi"],
       "year": 2025,
       "doi": "10.18653/v1/2025.acl-long.757",
       "relevance": "Routing for multi-agent systems, addressing sub-task assignment to specialized agent roles."
     },
     {
       "title": "A unified approach to routing and cascading for LLMs",
       "authors": ["Jasper Dekoninck", "Maximilian Baader", "Martin Vechev"],
       "year": 2025,
       "relevance": "Unified routing-cascading framework for LLMs, advancing the routing paradigm."
     },
     {
       "title": "Hybrid LLM: Cost-efficient and quality-aware query routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang", "Robert Sim", "Subhabrata Mukherjee", "Victor Rühle", "Laks V. S. Lakshmanan", "Ahmed H. Awadallah"],
       "year": 2024,
       "relevance": "Cost-quality routing framework for LLMs, prior work from some of the same authors."
     },
     {
       "title": "OmniRouter: Budget and performance controllable multi-LLM routing",
       "authors": ["Kai Mei", "Wenxuan Xu", "Mingyi Guo", "Shiwen Lin", "Yang Zhang"],
       "year": 2026,
       "doi": "10.1145/3787470.3787480",
       "relevance": "Casts routing as global constrained optimization over query distributions, relevant budget-aware approach."
     },
     {
       "title": "CARROT: A cost aware rate optimal router",
       "authors": ["Samuel Somerstep", "Felipe Maia Polo"],
       "year": 2025,
       "arxiv_id": "2502.03261",
       "relevance": "Lightweight cost-quality trade-off router with theoretical guarantees."
     },
     {
       "title": "Adaptive LLM routing under budget constraints",
       "authors": ["Pratik Panda", "R. Magazine", "C. Devaguptapu", "S. Takemori", "V. Sharma"],
       "year": 2025,
       "doi": "10.18653/v1/2025.findings-emnlp.1301",
       "relevance": "Budget-constrained adaptive routing policies for LLMs, directly related to the budget-aware paradigm."
     },
     {
       "title": "Best-route: Adaptive LLM routing with test-time optimal compute",
       "authors": ["Dujian Ding", "Ankur Mallick"],
       "year": 2025,
       "relevance": "Test-time compute optimization for LLM routing, related budget-constrained approach from overlapping authors."
     },
     {
       "title": "AppWorld: A controllable world of apps and people for benchmarking interactive coding agents",
       "authors": ["Harsh Trivedi", "Tushar Khot", "Marc Hartmann"],
       "year": 2024,
       "doi": "10.18653/v1/2024.acl-long.850",
       "relevance": "Primary benchmark for tool-use and coding complexity evaluation of agentic systems."
     },
     {
       "title": "AgentGym: Evolving large language model-based agents across diverse environments",
       "authors": ["Zhiheng Xi"],
       "year": 2024,
       "relevance": "Framework used for standardized evaluation on ALFWorld and SciWorld benchmarks."
     }
   ]
 }