ai-research-survey

scan.json (29369B)

---

 {
   "paper": {
     "title": "SkillOrchestra: Learning to Route Agents via Skill Transfer",
     "authors": [
       "Jiayu Wang",
       "Yifei Ming",
       "Zixuan Ke",
       "Shafiq Joty",
       "Aws Albarghouthi",
       "Frederic Sala"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.19672"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SkillOrchestra proposes skill-aware orchestration using a learned Skill Handbook that captures fine-grained competence and cost profiles of agents. Across 10 benchmarks (QA, math, FRAMES), it outperforms RL-based orchestrators like Router-R1 by up to 22.5 percentage points while requiring 700× less training data. The method alleviates routing collapse seen in RL-based approaches (Router-R1 routes 98% to one model vs. SkillOrchestra's balanced distribution) and transfers the learned handbook across orchestrator backbones without retraining.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at: https://github.com/jiayuww/SkillOrchestra' providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks are publicly available standard datasets: NQ, TriviaQA, PopQA, HotpotQA, 2WikiMultiHopQA, Musique, Bamboogle, MATH, AMC23, and FRAMES. Referenced with citations throughout Section 5."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists models used (Appendix A.1, A.2) but provides no environment specifications such as requirements.txt, Dockerfile, library versions, or dependency lists."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Appendix A describes experimental setup details but not how to run the code to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., 54.2, 71.6) with no confidence intervals, error bars, or ± notation. Figure 5 likewise shows only single values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SkillOrchestra 'outperforms' all baselines based solely on comparing raw accuracy numbers. No statistical significance tests (p-values, bootstrap tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: 'SkillOrchestra reaches 47.4 (+5.8)' relative to Router-R1's 41.6, 'up to +22.5 percentage-point improvement' on MATH (73.6 vs 55.8), and '+8.0 points' over ToolOrchestra (84.3 vs 76.3). Cost reductions are also quantified (38.4¢ vs 51.8¢)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Training uses 'k (k < 50) samples from each dataset' with no justification for why this number is sufficient. The ablation uses '100 randomly sampled FRAMES tasks' with no power analysis or sample size justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any experiment. All results in Table 1 and Figure 5 appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines across three categories: no routing (Vanilla, SFT, RAG, CoT, Search-R1), heuristic/discriminative (9 methods including FrugalGPT, RouterDC, GraphRouter), and RL-based (Router-R1, ToolOrchestra). Table 1 and Figure 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent work: Router-R1 (2025), ToolOrchestra (2025), GraphRouter (2025), RouteLLM (2025). GPT-5, Claude Opus 4.5, and Gemini 3 Pro are also used as orchestrator comparisons."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents a controlled ablation removing components incrementally: No Handbook, No Refinement+Selection, No Selection, No Fine-Grained Skills, and Full System. Each row isolates the contribution of one component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are reported: Exact Match accuracy and total inference cost (USD). Figure 1 plots both jointly, and Figure 5 and Table 2 report both."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. QA benchmarks use automated Exact Match, and FRAMES uses GPT-5-mini as a judge (Section A.2). All evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes evaluation on 'a held-out validation set D_val' for handbook selection. Appendix A.1 states 'k additional samples for validation.' The main results use standard benchmark test splits separate from handbook training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results across all 7 QA benchmarks individually. Figure 5 shows per-benchmark results for MATH and AMC. Figure 6 shows per-model routing distributions. Table 2 breaks down component contributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses failures of baseline methods (e.g., routing collapse in Router-R1) but does not analyze where SkillOrchestra itself fails or show qualitative failure examples of its own method."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows SkillOrchestra improving over baselines. No approaches that were tried and abandoned, configurations that failed, or ablations where adding a component hurt performance are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 22.5%' improvement supported by MATH results in Figure 5 (73.6 vs 55.8 = 17.8pp; AMC 52.5 vs 25.0 = 27.5pp — the 22.5% likely refers to AMC or rounding). The '700×' cost reduction is implied by k<50 training samples vs Router-R1's 14k (14000/20≈700). Results across 10 benchmarks are shown in Table 1 and Figure 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily supported through ablation studies (Table 2): 'removing the Skill Handbook causes a large drop in accuracy (85.0% → 71.0%).' Ablation design is controlled single-variable manipulation, which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'scalable, interpretable, and sample-efficient orchestration' and the conclusion states 'a principled alternative to data-intensive RL-based approaches' without bounding to the tested QA/math/FRAMES settings. The title 'Learning to Route Agents' suggests general applicability beyond the specific benchmarks tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for the performance gains. For example, the LLM-based skill discovery (using GPT-5 as the 'reflector') could itself be a confound — the improvement might stem from the LLM's reasoning rather than the skill abstraction framework. No robustness checks against alternative explanations are provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Exact Match accuracy and USD cost, and claims are about accuracy-cost tradeoffs. These measurements directly match the claims — no significant proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are specified with family and size (e.g., 'Qwen2.5-3B', 'LLaMA-3.1-70B') but proprietary models use only marketing names: 'GPT-5', 'GPT-5-mini', 'Claude Opus 4.5', 'Gemini 3 Pro' without API versions or snapshot dates. Per schema guidance, marketing names without versions do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 7-10 in Appendix D provide full orchestration instruction templates with complete actual prompt text, including skill definitions, model performance data, routing instructions, and format specifications. This is exceptionally transparent."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Max turns are stated (4 for routing, 50 for orchestration) and k < 50 training samples. However, critical hyperparameters are missing: temperature and sampling settings for all LLM calls, the value of λ (cost tradeoff weight) and λc, and the number k of training samples per dataset."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The orchestration framework is described in extensive detail: Section 3 formalizes modes, model pools, tools, and agent instantiation. Section 4 details the Skill Handbook structure, mode selection, and skill-grounded routing. Figure 3 provides a complete system overview. Algorithm 1 gives pseudocode."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We assume an exploratory dataset D_train' without describing how trajectories were collected in practice. Appendix A references 'the same evaluation protocol as Router-R1' without detailing preprocessing steps. How benchmark examples were sampled for training/validation splits is not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. Section 6 (Conclusion) is a brief forward-looking paragraph with no discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no analysis of what could threaten the validity of the results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do not show. The conclusion says 'We hope this work serves as a springboard for scalable orchestration' without stating boundaries or non-claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While benchmarks are public, the collected execution traces, learned skill handbooks, agent profiles, and trajectory data generated during experiments are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The data collection procedure for the exploratory dataset is only described abstractly: 'trajectories obtained by varying the agent choice at specific modes' (Section 4.2). How many trajectories, which specific queries, and what the success/failure criteria were are not specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (NQ, TriviaQA, PopQA, HotpotQA, etc.)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The skill handbook learning pipeline is described algorithmically (Phase 1: Discovery, Phase 2: Refinement) but the specific data transformation steps — how many skills were discovered, how many were merged/split, what thresholds were used — are not documented with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section is present in the paper despite having authors from both a university and a major AI company (Salesforce)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: 'University of Wisconsin-Madison' and 'Salesforce AI Research' with superscript numbering identifying which authors belong to each institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. Salesforce AI Research, which employs three of the six authors, has commercial interest in AI orchestration and agent technology."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses models like Qwen2.5, LLaMA-3.1, Mistral-7B, GPT-5, and Gemini 3 Pro without stating any of their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap despite using benchmarks published years before model training (NQ 2019, TriviaQA 2017, HotpotQA 2018, MATH 2021). These could be in the training data of all models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Most benchmarks (NQ, TriviaQA, PopQA, HotpotQA, 2Wiki, Musique, Bamboogle, MATH) were published years before the models' training cutoffs. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference costs are reported throughout: Figure 1 plots accuracy vs cost, Figure 5 compares cost in US cents (6.5¢ vs 3.6¢ for MATH), Table 2 reports cost in USD ($9.3 for full system), and specific per-query costs are given (38.4¢ vs 51.8¢ for Router-R1)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget for training the Skill Handbook is not quantified. The paper mentions '700× cost reduction' relative to baselines but does not state the absolute training cost, GPU hours, or total API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results (Table 1, Figure 5, Table 2) appear to be single-run numbers with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The trade-off hyperparameters λ and λc are mentioned but their values and how they were tuned are not reported. No hyperparameter search budget is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes Pareto-optimal handbook selection on a held-out validation set D_val. The selection criterion is formalized as maximizing the expected reward minus cost. SkillOrchestra+ selects the best orchestrator via the same validation-based Pareto criterion."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Comparisons are made across 10 benchmarks and against 15+ baselines with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own system against baselines without acknowledging self-comparison bias. While they use 'the same evaluation protocol as Router-R1,' the SkillOrchestra implementation, including LLM-based skill discovery and refinement, is their own."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 explicitly plots performance (accuracy) as a function of inference cost for all methods. Figure 5 directly compares accuracy and cost side by side. The paper's central thesis is about performance-cost tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Standard QA and math benchmarks are used without any discussion of whether they actually measure the orchestration capabilities claimed. No analysis of construct validity for using Exact Match on factoid QA as a measure of 'orchestration quality.'"
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "In model routing (Section 5.1): 'We use the same configuration as Router-R1 for controlled comparison.' In agent orchestration (Section 5.2): 'We follow the same evaluation protocol and experimental setup as ToolOrchestra' with identical modes, model pools, tools, and execution environments. The scaffold is controlled across comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Models trained on data collected well after the benchmark creation dates (e.g., TriviaQA 2017, HotpotQA 2018) are used without addressing temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the evaluation setup (multi-turn routing with intermediate model calls) leaks answer information through context."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No verification that training samples used for skill handbook learning are independent from test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "SkillOrchestra outperforms SoTA RL-based orchestrators by up to 22.5 percentage points with 700× and 300× learning cost reduction compared to Router-R1 and ToolOrchestra respectively.",
       "evidence": "Table 1 shows SkillOrchestra at 47.4 avg EM vs Router-R1 at 41.6. Figure 5 shows MATH: 73.6 vs 55.8 and AMC: 52.5 vs 25.0. Training uses k<50 samples vs Router-R1's 14k samples (Section A.1). Figure 1 (right) shows 84.3% vs ToolOrchestra's 76.3%.",
       "supported": "moderate"
     },
     {
       "claim": "Skill-based routing alleviates routing collapse seen in RL-based routing.",
       "evidence": "Figure 6 (left) shows Router-R1 selects LLaMA-3.1-70B for 98.02% of calls, while SkillOrchestra distributes: Mixtral-8x22B 44.53%, Qwen2.5-7B 25.99%, LLaMA-3.1-70B 15.38%, Qwen2.5-3B 11.50% (Section 5.1, Observation 3).",
       "supported": "strong"
     },
     {
       "claim": "The learned Skill Handbook transfers across orchestrator backbones without retraining.",
       "evidence": "Figure 6 (right) shows a handbook learned from Qwen2.5-3B improving all tested models: Qwen2.5-7B +24.3, LLaMA-3.1-8B +22.5, Mistral-7B +23.3, Mixtral-8x22B +14.8 (Section 5.1, Observation 4).",
       "supported": "moderate"
     },
     {
       "claim": "SkillOrchestra achieves the best performance-cost trade-off in full agent orchestration, outperforming GPT-5, Claude Opus 4.5, and Gemini 3 Pro as orchestrators.",
       "evidence": "Figure 1 (right) shows SkillOrchestra at 84.3% accuracy, $72.7 cost vs ToolOrchestra 76.3%, $92.7; GPT-5 74.6%, $120.4; Claude Opus 4.5 77.9%, $758.1; Gemini 3 Pro 78.9%, $1729.3 (Section 5.2, Observation 5).",
       "supported": "moderate"
     },
     {
       "claim": "Optimal skill granularity depends on orchestrator capacity; more skills are not always better.",
       "evidence": "Table 2 ablation shows: No Fine-Grained Skills achieves 80.4% at $15.1 cost vs Full System at 85.0% at $9.3. Disabling fine-grained skills degrades both accuracy and efficiency (Section 5.2, Observation 6).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across all experiments (Tables 1-2, Figures 1, 5) are single-run point estimates with no standard deviations, confidence intervals, or multi-seed results. Given known high variance in LLM-based systems, the reported differences could be within noise."
     },
     {
       "flag": "No contamination discussion despite using old benchmarks",
       "detail": "Benchmarks like TriviaQA (2017), HotpotQA (2018), NQ (2019), and MATH (2021) are evaluated with models trained years later (Qwen2.5, LLaMA-3.1, GPT-5). These benchmarks are almost certainly in the training data, yet contamination is never discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundaries discussion. This is a significant omission for a paper making broad claims about 'scalable, interpretable, and sample-efficient orchestration.'"
     },
     {
       "flag": "LLM-based components as confound",
       "detail": "The Skill Handbook is discovered and refined by an LLM-based discoverer and reflector (e.g., GPT-5). The improvement could partially stem from GPT-5's reasoning ability rather than the skill abstraction framework itself. This confound is not addressed."
     },
     {
       "flag": "Salesforce affiliation without funding disclosure",
       "detail": "Three of six authors are from Salesforce AI Research, a company with commercial interest in AI orchestration. No funding or competing interests statement is provided."
     },
     {
       "flag": "Every experiment shows improvement",
       "detail": "Across all 10 benchmarks, all metrics, and all ablations, SkillOrchestra always improves over baselines. No negative results, failed configurations, or settings where the method underperforms are reported, which is suspicious."
     }
   ],
   "cited_papers": [
     {
       "title": "Router-R1: Teaching LLMs Multi-Round Routing and Aggregation via Reinforcement Learning",
       "authors": ["Haozhen Zhang", "Tao Feng", "Jiaxuan You"],
       "year": 2025,
       "relevance": "SoTA RL-based multi-turn routing approach that SkillOrchestra directly outperforms; demonstrates routing collapse problem."
     },
     {
       "title": "ToolOrchestra: Elevating Intelligence via Efficient Model and Tool Orchestration",
       "authors": ["Hongjin Su", "Shizhe Diao", "Ximing Lu"],
       "year": 2025,
       "arxiv_id": "2511.21689",
       "relevance": "RL-trained agent orchestrator using GRPO for multi-tool coordination; primary baseline in agent orchestration experiments."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Cascade-based LLM routing approach for cost-efficient model selection; representative of heuristic routing methods."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "relevance": "Discriminative routing approach using preference data; BERT Router baseline in the evaluation."
     },
     {
       "title": "RouterDC: Query-Based Router by Dual Contrastive Learning for Assembling Large Language Models",
       "authors": ["Shuhao Chen", "Weisen Jiang", "Baijiong Lin"],
       "year": 2024,
       "relevance": "Contrastive learning-based router for LLM selection; discriminative routing baseline."
     },
     {
       "title": "GraphRouter: A Graph-Based Router for LLM Selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Graph-based formulation for LLM routing decisions; discriminative routing baseline."
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "authors": ["Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li"],
       "year": 2024,
       "arxiv_id": "2403.12031",
       "relevance": "Benchmark and KNN/MLP router baselines for multi-LLM routing evaluation."
     },
     {
       "title": "Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning",
       "authors": ["Bowen Jin", "Hansi Zeng", "Zhenrui Yue"],
       "year": 2025,
       "relevance": "RL-trained LLM for search-augmented reasoning; no-routing baseline in evaluation."
     },
     {
       "title": "A Survey of Frontiers in LLM Reasoning: Inference Scaling, Learning to Reason, and Agentic Systems",
       "authors": ["Zixuan Ke", "Fangkai Jiao", "Yifei Ming"],
       "year": 2025,
       "relevance": "Survey of LLM reasoning and agentic systems; motivates the need for effective orchestration in compound AI systems."
     },
     {
       "title": "Towards an AI Co-Scientist",
       "authors": ["Juraj Gottweis", "Wei-Hung Weng", "Alexander Daryin"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "Compound AI system for scientific discovery illustrating the need for multi-model coordination."
     },
     {
       "title": "FACT, FETCH, AND REASON: A Unified Evaluation of Retrieval-Augmented Generation",
       "authors": ["Satyapriya Krishna", "Kalpesh Krishna", "Anhad Mohananey"],
       "year": 2024,
       "arxiv_id": "2409.12941",
       "relevance": "FRAMES benchmark used in agent orchestration evaluation."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO algorithm used for training RL-based routing baselines like Router-R1."
     }
   ]
 }