ai-research-survey

scan.json (27401B)

---

 {
   "paper": {
     "title": "Orchestrating Intelligence: Confidence-Aware Routing for Efficient Multi-Agent Collaboration across Multi-Scale Models",
     "authors": [
       "Jingbo Wang",
       "Sendong Zhao",
       "Jiatong Liu",
       "Haochun Wang",
       "Wanting Li",
       "Bing Qin",
       "Ting Liu"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.04861",
     "doi": "10.48550/arXiv.2601.04861"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets (GSM8K, MATH, MedQA, GPQA, MBPP, HumanEval) are standard public benchmarks with published citations. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA A100 80G GPUs with vLLM for accelerated inference' (Section 4.1) but provides no requirements.txt, library versions, Dockerfile, or detailed environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described at a conceptual level but lacks the specifics needed to replicate."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 and Figures 3-7 are reported as point estimates only. No confidence intervals, error bars, or ± notation appears anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'OI-MAS consistently outperforms all vanilla backbone models' and 'average improvement of 7.68%' (Section 4.2) are based entirely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: 'accuracy gains ranging from 9.52% to 21.13%' over vanilla models, 'average improvement of 7.68%' over MasRouter (Section 4.2), and cost reduction '17.05%–78.47%' (Section 4.2). Absolute values in tables allow computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark subset sizes or power analysis. The paper states a 4:1 train/test split (Appendix A) but does not justify the resulting test set sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across runs. Temperature is set to 0 for LLM decoding, but the RL-trained routing policy has stochastic training. All results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five multi-agent baselines are included: LLM-Debate, GPTSwarm, AFlow, MaAS, and MasRouter. Vanilla single-model results are also shown (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MasRouter (Yue et al., 2025), MaAS (Zhang et al., 2025b), and AFlow (Zhang et al., 2025c) are all from 2025, representing recent and competitive approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents ablation of three core components: model router (w/o Gψ), cost term (w/o C(·)), and confidence weighting (w/o Conf(·)), evaluated on MedQA and MBPP."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy/Pass@1, inference cost (in dollars), and wall-clock latency (Figure 4). Cost-accuracy trade-offs are analyzed in Figure 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is fully automated through benchmark accuracy and pass@1 metrics. Human evaluation of reasoning quality or output quality could have been informative."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states 'For all datasets except HumanEval, we adopt a train/test split with a ratio of 4:1.' HumanEval is used exclusively for out-of-distribution testing."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark in Table 1. Figure 5 shows per-difficulty-level model selection on MATH. Figure 6 shows per-role model allocation. The case study (Appendix D) provides per-turn detail."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis or systematic failure case discussion. The case study (Figure 8) shows a success case only. No examples of where OI-MAS fails or makes poor routing decisions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 3) shows that removing the model router slightly improves accuracy on MedQA/MBPP but at substantially higher cost. Hyperparameter sensitivity (Figure 7) shows overly large λ degrades accuracy."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'improving accuracy by up to 12.88% while reducing cost by up to 79.78%.' Table 1 shows OI-MAS avg (78.23) vs AFlow Large avg (65.35), a 12.88% gap. Figure 3 shows cost reductions of 17.05%–78.47%. Claims are supported by the reported numbers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation (Table 3), where single components are removed in isolation. The ablation design constitutes controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims 'Multi-Scale Models' generally, but only tests four models from two families (Qwen2.5, Llama3.1). Claims like 'more reliable and efficient multi-agent reasoning systems' (Conclusion) extend beyond the five benchmarks and four models tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses memory, scalability, and safety concerns, but does not address alternative explanations for the observed improvements. For example, the performance gains could partly stem from the training objective or ensemble effects rather than routing quality."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy and Pass@1 on specific benchmarks and frames results in terms of those metrics. Claims match measurement granularity — e.g., 'accuracy on GSM8K' rather than broad claims about 'reasoning ability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: Qwen2.5-3B, Qwen2.5-7B, Llama3.1-8B, Llama3.1-70B (Section 4.1). These include family, version, and parameter count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Agent roles (Generator, Decomposer, Verifier, etc.) are described by name and function but no actual prompt text is provided. The reader cannot reconstruct the prompts sent to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: L=4 (max turns), λ=200 (cost penalty), θ=0.3 (role selection threshold), temperature=0, learning rate α=0.01. Sensitivity analysis for λ and L in Figure 7."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail: role routing (Eq. 2), model routing (Eq. 3), early stopping mechanism, confidence-aware optimization (Eq. 5), and the overall workflow is illustrated in Figure 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states a 4:1 train/test split (Appendix A) but does not describe how the split was performed (random, stratified by difficulty, etc.) or any preprocessing applied to the benchmark data."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the conclusion, discussing three specific limitations: agent memory representation, scalability guarantees, and agent safety."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'the work does not explicitly investigate how agent memory should be represented, maintained, and governed over time' and 'it is not guaranteed that the same balance will hold uniformly in highly concurrent, large-scale deployments.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses what wasn't investigated but doesn't explicitly bound what the results do not show. For example, it doesn't state that results don't generalize beyond the four models tested or the five benchmark domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, routing decisions, per-example results) is made available for independent verification. Only aggregated results are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses well-documented standard benchmarks (GSM8K, MATH, MedQA, GPQA, MBPP, HumanEval) with full citations. Appendix A describes each dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The routing policy training pipeline is described conceptually (RL with confidence-aware objective) but intermediate steps, data transformations, and filtering during training are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or acknowledgments section mentioning financial support is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology (5 authors), Chinese Academy of Sciences (1 author). No product being evaluated is affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. Absence of disclosure is not absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four backbone models (Qwen2.5-3B/7B, Llama3.1-8B/70B), despite these being pre-trained models evaluated on public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models may have seen benchmark data (GSM8K, MATH, MBPP, etc., all published 2021) during pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K, MATH, MBPP, MedQA, and HumanEval were all published in 2021, well before the training data collection for Qwen2.5 and Llama3.1. No contamination risk is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is a primary metric. Figure 3 shows cost-accuracy trade-offs, Table 2 reports cost in dollars, and Appendix C details the token-based pricing scheme used."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (A100 80G GPUs) but total GPU hours, training time for the routing policy, or total computational budget are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. While LLM decoding uses temperature=0, the RL-based routing policy training involves stochasticity. No results across multiple seeds are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to come from a single training run of the routing policy."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Figure 7 shows sensitivity analysis for λ and L, but does not report the total number of configurations tried, the search method, or compute spent on hyperparameter search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final hyperparameters (λ=200, L=4) are stated but no explanation is given for how they were selected (e.g., selected on validation set, or chosen from the sensitivity analysis)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares OI-MAS against baselines presumably re-implemented by the authors, but never acknowledges self-comparison bias. The statement 'all baselines are evaluated under identical hardware and decoding settings' (Section 4.1) addresses hardware fairness but not implementation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 explicitly plots accuracy vs. inference cost for OI-MAS and all baselines across four benchmarks, enabling direct performance-per-compute comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K, MATH, MedQA, GPQA, or MBPP actually measure the capabilities the paper claims to evaluate. The benchmarks are taken at face value."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different baselines use different scaffolding architectures (debate, swarm optimization, tree search, supernet, cascaded controller) yet performance differences are attributed to OI-MAS's routing mechanism. The confounding effect of scaffold design differences is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Most benchmarks (GSM8K, MATH, MBPP, MedQA, HumanEval) were published in 2021, and models were trained on data collected years later."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities with pre-training data or whether train/test examples within the 4:1 split are truly independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method (canary strings, membership inference, decontamination) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "OI-MAS consistently outperforms all vanilla backbone models with accuracy gains ranging from 9.52% to 21.13%.",
       "evidence": "Table 1 shows OI-MAS achieves 78.23% average vs the best vanilla model (Llama3.1-70B) at 68.71%, across five benchmarks (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "OI-MAS achieves an average improvement of 7.68% over MasRouter and outperforms it on four of five benchmarks.",
       "evidence": "Table 1: OI-MAS avg 78.23% vs MasRouter 70.55%. OI-MAS matches on GSM8K (95.80%) and exceeds on MATH (+7.56%), MedQA (+7.56%), GPQA (+8.98%), MBPP (+14.28%).",
       "supported": "moderate"
     },
     {
       "claim": "OI-MAS reduces inference cost by 17.05%–78.47% compared to baseline multi-agent systems while maintaining or improving accuracy.",
       "evidence": "Figure 3 shows cost-accuracy plots across four benchmarks (Section 4.2). Cost computed via token-based API pricing (Appendix C).",
       "supported": "moderate"
     },
     {
       "claim": "OI-MAS achieves 23.12s wall-clock latency per query on GPQA, outperforming all baselines.",
       "evidence": "Figure 4 shows latency comparison on GPQA only. Nearest baseline is MasRouter at 36.82s, MaAS at 192.62s.",
       "supported": "weak"
     },
     {
       "claim": "The routing policy adapts model selection to problem difficulty, using smaller models for easier problems and larger models for harder ones.",
       "evidence": "Figure 5 shows model selection distribution across five MATH difficulty levels, with clear progression from small to large models as difficulty increases (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "OI-MAS generalizes out-of-distribution from MBPP to HumanEval with 91.46% Pass@1 at the lowest cost among all baselines.",
       "evidence": "Table 2 shows OOD results: OI-MAS achieves 91.46% at $0.097 vs MaAS 89.63% at $0.222. Only one OOD setting tested.",
       "supported": "moderate"
     },
     {
       "claim": "The confidence-aware mechanism is critical for maintaining task accuracy.",
       "evidence": "Table 3 ablation: removing Conf(·) drops MedQA from 78.99% to 76.47% and MBPP from 91.59% to 87.39%. Only two benchmarks tested.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OI-MAS introduces a hierarchical role-model routing framework for multi-agent systems that jointly selects agent roles and model scales at each reasoning step, using confidence-aware optimization to balance accuracy and cost. On five benchmarks (GSM8K, MATH, MedQA, GPQA, MBPP), it achieves up to 12.88% accuracy improvement and up to 79.78% cost reduction over multi-agent baselines. The routing policy learns interpretable patterns: larger models for generative roles and harder problems, smaller models for structural roles and easy subtasks. The approach generalizes from MBPP to HumanEval without retraining.",
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All performance claims are based on comparing point estimates across systems. With RL-based routing policy training, results may vary across training runs, but no variance, confidence intervals, or significance tests are provided."
     },
     {
       "flag": "Proxy cost metric",
       "detail": "All models are run locally on A100 GPUs, but costs are computed using Together AI API token pricing (Appendix C). One model's price (Qwen2.5-3B) is estimated via a scaling law rather than actual pricing. The API pricing proxy may not reflect actual local inference costs."
     },
     {
       "flag": "Complete contamination silence",
       "detail": "Four pre-trained models are evaluated on five benchmarks published in 2021 (GSM8K, MATH, MBPP, MedQA, HumanEval). No training cutoff dates, no contamination analysis, no decontamination. Results may partly reflect memorization."
     },
     {
       "flag": "No failure analysis",
       "detail": "The paper shows only success cases. The case study (Figure 8) illustrates correct routing. No examples of poor routing decisions, failure modes, or systematic errors are discussed."
     },
     {
       "flag": "Ablation on subset of benchmarks",
       "detail": "The ablation study (Table 3) is performed on only MedQA and MBPP. Key claims about component importance are not validated on GSM8K, MATH, or GPQA."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2023,
       "relevance": "Foundational multi-agent framework for software development using structured role-based collaboration."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent system for software development through communicative agent collaboration."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "General multi-agent conversation framework enabling complex LLM-based applications."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2024,
       "relevance": "LLM routing framework that learns to select between models using preference data, directly related to model routing efficiency."
     },
     {
       "title": "MasRouter: Learning to Route LLMs for Multi-Agent Systems",
       "authors": ["Yanwei Yue", "Guibin Zhang", "Boyang Liu"],
       "year": 2025,
       "arxiv_id": "2502.11133",
       "relevance": "Most directly comparable baseline — routes LLMs in multi-agent settings via cascaded controller, but with static pre-inference decisions."
     },
     {
       "title": "AFlow: Automating Agentic Workflow Generation",
       "authors": ["Jiayi Zhang", "Jinyu Xiang", "Zhaoyang Yu"],
       "year": 2025,
       "relevance": "Automates discovery of agentic workflows through Monte Carlo Tree Search, representative of automated agent design approaches."
     },
     {
       "title": "GPTSwarm: Language Agents as Optimizable Graphs",
       "authors": ["Mingchen Zhuge", "Wenyi Wang", "Louis Kirsch"],
       "year": 2024,
       "relevance": "Formulates LLM agents as optimizable computational graphs, enabling joint optimization of prompts and communication topology."
     },
     {
       "title": "CodeAgent: Enhancing Code Generation with Tool-Integrated Agent Systems for Real-World Repo-Level Coding Challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "relevance": "Tool-integrated agent system for code generation, relevant to agentic AI capabilities evaluation."
     },
     {
       "title": "CodeTree: Agent-Guided Tree Search for Code Generation with Large Language Models",
       "authors": ["Jierui Li", "Hung Le", "Yingbo Zhou"],
       "year": 2025,
       "relevance": "Agent-guided tree search approach to code generation, relevant to LLM programming capabilities."
     },
     {
       "title": "AgentVerse: Facilitating Multi-Agent Collaboration and Exploring Emergent Behaviors",
       "authors": ["Weize Chen", "Yusheng Su", "Jingwei Zuo"],
       "year": 2024,
       "relevance": "Multi-agent collaboration platform studying emergent behaviors in LLM agent teams."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Directly relevant cost-efficiency approach for LLM inference through model cascading and routing."
     },
     {
       "title": "X-MAS: Towards Building Multi-Agent Systems with Heterogeneous LLMs",
       "authors": ["Rui Ye", "Xiangrui Liu", "Qimin Wu"],
       "year": 2025,
       "relevance": "Multi-agent systems using heterogeneous LLM pools, closely related to OI-MAS's multi-scale model approach."
     }
   ]
 }