ai-research-survey

scan.json (27893B)

---

 {
   "paper": {
     "title": "Optimal-Agent-Selection: State-Aware Routing Framework for Efficient Multi-Agent Collaboration",
     "authors": [
       "Jingbo Wang",
       "Sendong Zhao",
       "Haochun Wang",
       "Yuzheng Fan",
       "Lizhe Zhang",
       "Yan Liu",
       "Ting Liu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.02200",
     "doi": "10.48550/arXiv.2511.02200"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "STRMAC, a state-aware routing framework for multi-agent collaboration, achieves up to 23.8% accuracy improvement over baselines on collaborative reasoning tasks (PDDP clinical prediction and EBFC fact-checking) while drastically reducing token consumption to 5.9–21.6% of competing methods. A self-evolving data generation strategy reduces training data collection overhead by up to 90.1% compared to exhaustive search. Routers trained on execution data from smaller models (Llama-3.1-8B) transfer effectively to larger agent models (Llama-3.1-70B, Qwen2.5-32B) and even closed-source GPT-4o.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "PDDP is constructed from MIMIC-III (publicly available with data use agreement) and EBFC from AMBIFC (Glockner et al. 2024), both publicly accessible datasets. However, the specific multi-agent task formulations and router training data are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'A100 with 80G memory' and lists model names but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts to replicate experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, 3, and 4 are point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims STRMAC outperforms baselines based solely on comparing accuracy numbers. No p-values, t-tests, or any statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., 'improvements of +4.4%, +8.1%, and +1.7%' (Section 4.2) and '+6.0%, +14.8% and +23.8%' over Random-Chain, with absolute accuracy values provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the dataset sizes used in PDDP or EBFC. The number of test instances is never explicitly stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported. All results appear to be single-run numbers with no indication of result stability across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: Random-Chain, LLM-Debate (Du et al. 2023), MACNET (Qian et al. 2024), MOA (Wang et al. 2024a), MDAgents (Kim et al. 2024), and IG-MAS (Wang et al. 2025)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: IG-MAS (2025), MDAgents (NeurIPS 2024), MACNET (2024), and MOA (2024). These represent current state of the art in multi-agent collaboration."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "While Section 5.1 shows the benefit of iterative data generation (pruning + router-guided exploration), there is no ablation of the core STRMAC architecture: the state encoder, agent embeddings, contrastive learning objective, or cosine similarity routing are never individually removed or modified to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Accuracy (Acc), Token consumption (Token), and Cost-Adjusted Score (CAS), which combines accuracy and cost."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated against ground-truth labels on PDDP (discharge disposition) and EBFC (fact verification) tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'validation performance' for tuning learning rate but never explicitly describes a train/test split, the number of test examples, or confirms that reported results are on data unseen during any selection decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (PDDP vs EBFC) and by model (Llama3.1-70B, Llama3.1-8B, Qwen2.5-32B) in Tables 1 and 4. Individual agent performance is shown in Table 5, and path distribution analysis in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 discusses that collaborative paths 'do not always result in higher accuracy, suggesting that only carefully selected, relevant agents can provide meaningful improvements, while the inclusion of less relevant agents may introduce noise and negatively impact performance.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 shows that multi-agent collaboration paths sometimes have lower accuracy than single-agent paths (e.g., Figure 3 shows AgentBHC alone at 70.9% vs AgentBHC→AgentSH at 60.4% for Llama70B), demonstrating that collaboration can hurt performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 23.8% improvement over baselines' which matches EBFC Qwen32B (48.5%→72.3% in Table 1), and '90.1% reduction in data collection overhead' which matches EBFC Llama70B (9.9% proportion in Table 2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('STRMAC improves accuracy') are supported by controlled comparisons: the same models and datasets are used across all methods, with only the collaboration/routing strategy varying. This controlled single-variable manipulation is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Optimal-Agent-Selection' and 'Efficient Multi-Agent Collaboration' broadly, but results are limited to two specific tasks (clinical discharge prediction and fact-checking) with specific multi-agent setups where agents hold private information subsets. The paper does not bound its claims to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why STRMAC outperforms baselines. For instance, the router may simply learn to pick the single strongest agent (which Section 5.4 partially confirms), but this is presented as a feature rather than explored as an alternative explanation for the gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures classification accuracy on PDDP (4-class) and EBFC (binary) and frames this as 'collaborative reasoning' and 'multi-agent reasoning accuracy' throughout. No discussion of whether classification accuracy captures reasoning quality, or whether these specific benchmarks measure the broader collaborative reasoning capability claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The main experiments specify 'Llama-3.1-70B-Instruct', 'Llama-3.1-8B-Instruct', and 'Qwen2.5-32B-Instruct' — specific model versions available on HuggingFace. GPT-4o supplementary experiments reference OpenAI (2024) but lack a snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are provided. The paper describes agents receiving 'private context' and 'shared context' but never shows the actual prompt text used for any agent."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Learning rate range (1×10⁻⁴ to 5×10⁻⁵) and router model (mDeBERTaV3-base, 86M params) are reported, but no temperature, top-p, max tokens, or other LLM inference settings are stated for the agent models."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent routing framework IS the contribution and is described in detail: Section 3.2 covers the state encoder and agent embeddings, Section 3.3 the contrastive learning objective, and the overall workflow is shown in Figure 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Dataset construction is deferred to 'the protocols of (Wang et al. 2025)' without restating the details. How medical records were fragmented across agents, how AMBIFC claims were selected, and filtering criteria are not documented in this paper."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations, threats-to-validity, or discussion section addressing limitations. The paper moves directly from analysis to conclusion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes no explicit statements about what the results do NOT show, what settings are excluded, or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the underlying MIMIC-III and AMBIFC datasets are publicly available, the specific task formulations, agent-specific data splits, router training execution paths, and processed data are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes PDDP (from MIMIC-III, clinical prediction with fragmented records across agents) and EBFC (from AMBIFC, fact-checking with evidence distribution to agents), including what each agent receives and the task objective."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from existing datasets (MIMIC-III, AMBIFC)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The self-evolving data generation pipeline (Section 3.4) is well-described for router training data, but the end-to-end pipeline from raw MIMIC-III/AMBIFC data to final agent inputs and evaluation setup is not documented. The number of test instances is never stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, acknowledgments, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology (academic) and China Mobile Group Heilongjiang Co., Ltd (industry)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. China Mobile is listed as an affiliation but the paper does not evaluate any China Mobile product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for Llama-3.1, Qwen2.5, or GPT-4o. This is important because the benchmark data (MIMIC-III from 2016, AMBIFC from 2024) could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MIMIC-III clinical data or AMBIFC claims appeared in any model's pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MIMIC-III (published 2016) is widely used and could be in training data of models trained after 2016. No contamination analysis or temporal split is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption is reported for all methods across all experiments in Tables 1 and 4. The CAS metric (Equation 4.1) explicitly penalizes token cost. STRMAC uses 338–972 tokens vs thousands for baselines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('A100 with 80G memory') but total GPU hours, training time for the router, or total API cost for data generation are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single runs. No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper never states how many runs produced the reported results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Learning rate is 'tuned within the range of 1×10⁻⁴ to 5×10⁻⁵ according to validation performance' but no details on number of configurations tried, search method, or compute spent on tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions tuning 'according to validation performance' but does not describe the selection process, how many configurations were evaluated, or whether selection was done on a proper validation set separate from the test data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own versions of all baselines and compare against them without acknowledging the systematic bias of evaluating one's own system against one's own re-implementations of competing methods."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The CAS metric (Section 4.2) explicitly evaluates performance as a function of inference cost (token consumption), and token counts are reported alongside accuracy for all methods, enabling cost-performance comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether PDDP and EBFC actually measure 'collaborative reasoning' as claimed. The paper does not question whether classification accuracy on these tasks captures the multi-agent reasoning capability it claims to improve."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same base LLMs are used across all methods, with only the collaboration/routing framework varying. The scaffold differences are the intentional independent variable being studied, and the paper controls for model choice."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MIMIC-III was published in 2016 and AMBIFC in 2024. No discussion of whether these datasets or their contents appeared in model training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether agent prompts or the task setup inadvertently leak information about the correct answer."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the training instances (used for router training) and test instances are independent or share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "STRMAC achieves up to 23.8% accuracy improvement over baselines on collaborative reasoning tasks.",
       "evidence": "Table 1 shows EBFC with Qwen2.5-32B: Random-Chain 48.5% → STRMAC 72.3%, a +23.8% absolute improvement. Improvements are consistent across models and tasks.",
       "supported": "strong"
     },
     {
       "claim": "Self-evolving data generation reduces training data collection overhead by up to 90.1% compared to exhaustive search.",
       "evidence": "Table 2 shows EBFC with Llama70B requires only 9.9% of exhaustive search paths (126.1 vs 1275.1 theoretical maximum), representing a 90.1% reduction.",
       "supported": "strong"
     },
     {
       "claim": "STRMAC achieves highest Cost-Adjusted Score (CAS) across all models and tasks.",
       "evidence": "Table 1 shows STRMAC CAS values of 48.6–82.4 across all settings, consistently exceeding all baselines. Token consumption is 5.9–21.6% of IG-MAS.",
       "supported": "strong"
     },
     {
       "claim": "Routers trained on smaller model data (Llama-8B) generalize to larger agent models.",
       "evidence": "Table 3 shows Llama70B accuracy with same-model training (64.0%) vs Llama8B-sourced training (63.4%) on PDDP, and 85.2% vs 83.2% on EBFC — marginal differences.",
       "supported": "moderate"
     },
     {
       "claim": "STRMAC transfers effectively to closed-source GPT-4o using open-source model training data and embeddings.",
       "evidence": "Table 4 shows STRMAC achieves 64.4% (PDDP) and 89.1% (EBFC) on GPT-4o, outperforming all baselines while using open-source model execution data and Llama70B agent embeddings.",
       "supported": "moderate"
     },
     {
       "claim": "Agent execution order significantly impacts multi-agent collaboration performance.",
       "evidence": "Figure 1 shows that two different orderings of the same five agents produce substantially different accuracies across four models. No statistical significance test accompanies this claim.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across Tables 1–4 are single point estimates. Without variance across runs or statistical tests, it is impossible to assess whether reported differences are statistically meaningful or within noise range."
     },
     {
       "flag": "No ablation of core architecture",
       "detail": "The paper's main novelty (state encoder, agent embeddings, contrastive learning, cosine similarity routing) is never ablated. We cannot determine which components actually contribute to the reported improvements."
     },
     {
       "flag": "No limitations discussed",
       "detail": "The paper contains no limitations section, no threats to validity, and no discussion of what the results do not show. This is a significant omission."
     },
     {
       "flag": "Only two narrow benchmarks",
       "detail": "Results are reported on only PDDP (clinical discharge prediction) and EBFC (fact-checking), both with specific private-information-sharing setups. The broad claims about 'efficient multi-agent collaboration' are poorly supported by this narrow evaluation."
     },
     {
       "flag": "Router may just learn single-agent selection",
       "detail": "Section 5.4 shows the dominant routing path across all models is to select only AgentBHC (the agent with the most comprehensive information). The router's main 'skill' may be identifying which single agent has the best information, not enabling meaningful multi-agent collaboration."
     },
     {
       "flag": "Test set details missing",
       "detail": "The number of test instances, train/test split ratios, and whether reported results are on truly held-out data are never stated."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam et al."],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM technical report relevant to understanding model capabilities evaluated in multi-agent systems."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian et al."],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent LLM system for code generation with static role-based pipelines, a key baseline paradigm."
     },
     {
       "title": "AgentVerse: Facilitating Multi-Agent Collaboration and Exploring Emergent Behaviors in Agents",
       "authors": ["Weize Chen et al."],
       "year": 2023,
       "arxiv_id": "2308.10848",
       "relevance": "Multi-agent collaboration framework exploring emergent behaviors, relevant to multi-agent system design."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational LLM routing work balancing cost and performance across models."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong et al."],
       "year": 2024,
       "relevance": "LLM routing system learning from preference data, relevant to model selection and cost-quality tradeoffs."
     },
     {
       "title": "RouterDC: Query-based Router by Dual Contrastive Learning for Assembling Large Language Models",
       "authors": ["Shuhao Chen et al."],
       "year": 2024,
       "relevance": "Contrastive learning-based LLM router that STRMAC directly builds upon for its routing architecture."
     },
     {
       "title": "MDAgents: An Adaptive Collaboration of LLMs for Medical Decision-Making",
       "authors": ["Yubin Kim et al."],
       "year": 2024,
       "relevance": "Adaptive multi-agent medical reasoning framework, used as a baseline in this work."
     },
     {
       "title": "GPTSwarm: Language Agents as Optimizable Graphs",
       "authors": ["Mingchen Zhuge et al."],
       "year": 2024,
       "relevance": "Trainable graph-based multi-agent communication framework optimizing collaboration structure."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Yilun Du et al."],
       "year": 2023,
       "relevance": "Foundational multi-agent debate approach for improving LLM reasoning, used as a baseline."
     },
     {
       "title": "Mixture-of-Agents Enhances Large Language Model Capabilities",
       "authors": ["Junlin Wang et al."],
       "year": 2024,
       "arxiv_id": "2406.04692",
       "relevance": "Layered aggregation approach for combining multiple LLM outputs, used as a baseline."
     },
     {
       "title": "Scaling Large-Language-Model-based Multi-Agent Collaboration",
       "authors": ["Chen Qian et al."],
       "year": 2024,
       "arxiv_id": "2406.07155",
       "relevance": "Studies scaling multi-agent collaboration and demonstrates agent ordering impacts performance (MACNET baseline)."
     },
     {
       "title": "AgentCoder: Multi-Agent-Based Code Generation with Iterative Testing and Optimisation",
       "authors": ["Dong Huang et al."],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent system for code generation with iterative refinement, relevant to agentic AI development."
     },
     {
       "title": "MedAgents: Large Language Models as Collaborators for Zero-shot Medical Reasoning",
       "authors": ["Xiangru Tang et al."],
       "year": 2023,
       "arxiv_id": "2311.10537",
       "relevance": "Multi-agent medical reasoning system using static role-based pipelines, predecessor to adaptive approaches."
     }
   ]
 }