ai-research-survey

scan.json (26719B)

---

 {
   "paper": {
     "title": "SC-MAS: Constructing Cost-Efficient Multi-Agent Systems with Edge-Level Heterogeneous Collaboration",
     "authors": ["Di Zhao", "Longhui Ma", "Siwei Wang", "Miao Wang", "Yi Kong"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.09434",
     "doi": "10.48550/arXiv.2601.09434"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SC-MAS models multi-agent systems as directed acyclic graphs with edge-level heterogeneous collaboration strategies, achieving 1.46-3.53% accuracy gains with 11-16% cost reductions over MasRouter across five benchmarks (MMLU, GSM8K, MATH, HumanEval, MBPP). Ablation shows the edge optimizer and LLM router are the most impactful components. The framework can generalize to unseen LLMs (Deepseek-v3) without retraining and implicitly filters unhelpful agents during selection.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five benchmarks used (MMLU, GSM8K, MATH, HumanEval, MBPP) are publicly available standard datasets. The paper references their original publications."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section C.2 mentions '8G GPU for training' but provides no requirements.txt, Dockerfile, library versions, or detailed environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithm description (Algorithm 2) describes the method but is not a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Despite stating 'We run the experiment three times on each dataset and report the average,' Tables 1, 2, and 4 report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. All claims of improvement (e.g., '3.35% improvement on MMLU') are based on comparing point estimates without any test of statistical significance."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '3.35% on MMLU' (84.25 → 87.60), '3.53% on MBPP' (84.00 → 87.53), '12.13% cost reduction on MBPP.' Both absolute and relative differences are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for dataset sizes or the 1:4 validation/test split ratio. The MATH subset of 519 problems is mentioned as following prior work protocol but not independently justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states experiments are 'run three times' and averages are reported, but no standard deviation, variance, or any spread measure appears in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are compared in Table 1: dynamic MAS methods (GPTSwarm, AgentPrune, AFlow), single-agent routing (PromptLLM, RouteLLM, FrugalGPT, RouterDC), and multi-agent routing (MasRouter), plus vanilla single-model results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include MasRouter (2025), AFlow (2025c), RouteLLM (2025), and other 2024-2025 methods, representing the current state of the art in MAS routing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents a thorough ablation replacing each component (Fθv, Fθe, Fθl) with randomized selection, measuring both accuracy and cost impact across all five datasets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy and inference cost (in dollars) across all benchmarks, providing two complementary evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated via benchmark test suites (accuracy on MMLU, GSM8K, MATH, HumanEval, MBPP)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'we partition each dataset into validation and test subsets using a 1:4 ratio,' with validation used for training and test for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across 5 benchmarks (Table 1), per-component ablation (Table 2), role selection distributions (Figure 3), LLM allocation distributions (Figure 4), and case studies per dataset (Appendix D)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure analysis or error cases are presented. Section 5.4 shows filtering of deliberately noisy agents but does not analyze cases where SC-MAS fails on actual tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows SC-MAS outperforming all baselines. The sensitivity analysis (Section 5.6) shows expected cost-performance tradeoffs but no surprising negative findings or approaches that were tried and abandoned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 3.35% accuracy gain on MMLU (84.25→87.60 in Table 1) and 3.53% gain on MBPP (84.00→87.53) are confirmed by Table 1. Cost reduction claims are supported by Figure 2 and Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2) uses controlled single-variable manipulation (replacing each component with random selection), which is an adequate design for claims like 'removing Fθe resulted in a performance drop.' This supports the causal role of each component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Constructing Cost-Efficient Multi-Agent Systems' generally, but results are limited to 5 specific benchmarks (knowledge QA, math, coding). No qualification is provided about task types not covered (e.g., open-ended generation, creative tasks, long-form reasoning)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for SC-MAS's improvements. For example, whether the gains come specifically from heterogeneous collaboration or simply from using more diverse LLM combinations, or from the training procedure itself, is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on standard benchmarks and reports it as accuracy, and measures token cost and reports it as cost. The claims match the granularity of measurements without inflated framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only 'gpt-4o-mini-0718' has a specific version identifier. 'claude-3.5-haiku', 'gemini-1.5-flash', and 'llama-3.1-70b' are named without specific API versions or snapshot dates. Model behavior can change across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The collaboration strategies (Chain, Debate, Criticism, CoT, Reflection) are named and cited but actual prompt text is never provided. The 26 agent role definitions are also not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports learning rate α=0.01, temperature τ=1, cost penalty λ=5 (from {0,5,10,15,20}), and states LLM temperature parameters 'remain at default settings.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The SC-MAS framework is described in detail: Node Selector (Section 4.1), Edge Optimizer (Section 4.2), LLM Router (Section 4.3), graph execution (Algorithm 1), and the full training workflow (Algorithm 2). Collaboration strategies are defined and the DAG constraint is formalized."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 documents the 1:4 validation/test split and the stratified sampling procedure for MATH (519 problems spanning difficulty levels), following the MasRouter protocol."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitation' is a dedicated section discussing two specific limitations: structural DAG constraints and limited interpretability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to SC-MAS: (1) the DAG constraint 'precludes cyclical communication patterns and feedback loops' needed for iterative refinement, (2) the 'rationale behind selecting specific agents, collaboration strategies, and large language models remains non-trivial to interpret.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses design constraints but does not explicitly state what the results do NOT show — e.g., no statement about untested task types, scale limitations, or settings where SC-MAS would not apply."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-run results, agent selections, generated graphs) is released. Only aggregated results in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the benchmark datasets used, their sources (with citations), and the evaluation protocol including validation/test splits and stratified sampling."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation uses standard benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from query input through node selection, edge optimization, LLM assignment, graph execution, and reward computation is fully documented in Algorithms 1 and 2. Dataset splitting is described in Section 5.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: College of Computer Science and Technology, National University of Defense Technology, and Academy of Military Sciences."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed. The authors are at military institutions which could have institutional interest in cost-efficient AI systems, but no funding statement addresses this."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four LLM backbones (GPT-4o-mini, Claude 3.5 Haiku, Gemini 1.5 Flash, Llama 3.1 70B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (MMLU, GSM8K, MATH, HumanEval, MBPP — all published 2021) appeared in the training data of models trained through 2024."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All five benchmarks were published in 2021, well before the training cutoffs of models used (2024 models). No contamination analysis is performed despite this obvious risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference costs in dollars are reported in Table 2, Table 4, and Figure 2, comparing SC-MAS against baselines across all five benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section C.2 states '8G GPU for training' and Table 5 provides detailed training token counts (prompt tokens, completion tokens, total cost) for SC-MAS and baselines on MATH and MMLU."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs experiments three times but does not report per-seed results or any measure of seed sensitivity. Only averaged results appear in the tables."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states 'We run the experiment three times on each dataset and report the average as the final result.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The values α=0.01, τ=1 are stated without explaining how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The λ parameter is varied across {0, 5, 10, 15, 20} with full results shown in Figure 5. The choice of λ=5 for main results is transparent, and the sensitivity analysis reveals the tradeoff curve."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No multiple comparison correction is applied. The paper compares SC-MAS against 8+ baselines across 5 datasets without any family-wise error rate control, and no significance tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their SC-MAS implementation against their own implementations of baselines without acknowledging potential author-evaluation bias (Lucic et al. 2018 issue)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 directly compares inference costs between SC-MAS and MasRouter. Table 4 provides detailed cost-performance data. Table 5 compares training costs across methods. The λ sensitivity analysis (Figure 5) shows performance as a function of cost penalty."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU, GSM8K, MATH, HumanEval, or MBPP actually measure the capabilities claimed. The benchmarks are used without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SC-MAS constructs different MAS configurations (scaffolds) per query. While the LLM pool is held constant across routing methods, different MAS structures represent different scaffolding. The confound between scaffold design and LLM capability is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. All five benchmarks were published in 2021, and the LLMs used were trained on data through 2024, creating a 3-year window of potential temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the evaluation setup leaks information beyond what would be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No verification of independence between training data of the LLMs and test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used (no canary strings, membership inference, decontamination, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "SC-MAS improves accuracy by 3.35% on MMLU while reducing inference cost by 15.38% compared to MasRouter.",
       "evidence": "Table 1 shows SC-MAS 87.60% vs MasRouter 84.25% on MMLU. Cost comparison in Figure 2. Point estimates only, no error bars despite 3 runs.",
       "supported": "moderate"
     },
     {
       "claim": "SC-MAS achieves a 3.53% accuracy gain with a 12.13% cost reduction on MBPP compared to MasRouter.",
       "evidence": "Table 1 shows SC-MAS 87.53% vs MasRouter 84.00% on MBPP. Table 4 shows costs: SC-MAS $0.913 vs MasRouter $1.039.",
       "supported": "moderate"
     },
     {
       "claim": "Edge-level heterogeneous collaboration is more effective than homogeneous graph-level collaboration.",
       "evidence": "Ablation in Table 2: removing Fθe drops accuracy 1.17-3.83% and increases cost 16.67-41.32%. Comparison with MasRouter (homogeneous) in Table 1.",
       "supported": "moderate"
     },
     {
       "claim": "SC-MAS can generalize to unseen LLMs without retraining (inductive ability).",
       "evidence": "Section 5.5 and Figure 4: adding Deepseek-v3 without retraining improves MMLU from 87.60% to 88.14% and MATH from 76.75% to 77.84%. Only one new LLM tested.",
       "supported": "weak"
     },
     {
       "claim": "SC-MAS implicitly filters unhelpful agents by assigning low selection probabilities.",
       "evidence": "Section 5.4 and Figure 3: two deliberately noisy agents received only 7.63% and 4.58% selection rates on HumanEval. Single experiment with artificially constructed noisy agents.",
       "supported": "weak"
     },
     {
       "claim": "SC-MAS reduces token consumption by 11.17% to 16.35% while improving accuracy by 1.46% to 3.34% over state-of-the-art methods.",
       "evidence": "Section 1 contributions and Section 5.2 performance analysis. Table 1 for accuracy, Figure 2 and Table 4 for costs. Range spans multiple benchmarks.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars despite multiple runs",
       "detail": "The paper states experiments are run 3 times and averaged, but no standard deviation, confidence interval, or any spread measure is reported in any table. This makes it impossible to assess whether the claimed improvements (e.g., 3.35% on MMLU) are within noise."
     },
     {
       "flag": "No significance tests for claimed improvements",
       "detail": "All comparative claims are based on comparing point estimates. With only 3 runs and no significance tests, claimed improvements of 1-3% could easily be within random variation."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "All five benchmarks (MMLU, GSM8K, MATH, HumanEval, MBPP) were published in 2021, well before the training cutoffs of the models used (2024). No contamination analysis is performed, despite this being a known issue for these benchmarks."
     },
     {
       "flag": "Uniformly positive results across all benchmarks",
       "detail": "SC-MAS outperforms all baselines on every single benchmark and metric. No failure cases or negative results are reported, which is unusual for a method tested across diverse task types."
     },
     {
       "flag": "No code release for reproducibility",
       "detail": "No source code, repository, or reproducibility artifacts are provided. Combined with missing prompts and incomplete model version specifications, independent reproduction is not possible."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Key multi-agent framework for software development that SC-MAS builds upon and compares against conceptually."
     },
     {
       "title": "MasRouter: Learning to route LLMs for multi-agent systems",
       "authors": ["Yanwei Yue", "Guibin Zhang", "Boyang Liu"],
       "year": 2025,
       "relevance": "Primary baseline and predecessor for multi-agent routing; SC-MAS directly extends its homogeneous collaboration to heterogeneous edge-level strategies."
     },
     {
       "title": "GPTSwarm: language agents as optimizable graphs",
       "authors": ["Mingchen Zhuge", "Wenyi Wang", "Louis Kirsch"],
       "year": 2024,
       "relevance": "Graph-based MAS optimization baseline that models agents as optimizable graphs, compared against in Table 1."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Foundational work on LLM cost reduction through routing strategies, evaluated as a single-agent routing baseline."
     },
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "relevance": "Single-agent LLM routing method using preference data, compared as baseline for routing quality."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": ["Jiayi Zhang", "Jinyu Xiang", "Zhaoyang Yu"],
       "year": 2025,
       "relevance": "Automated agentic workflow generation method compared as dynamic MAS baseline."
     },
     {
       "title": "Cut the crap: An economical communication pipeline for LLM-based multi-agent systems",
       "authors": ["Guibin Zhang", "Yanwei Yue"],
       "year": 2025,
       "relevance": "AgentPrune method for reducing communication overhead in MAS, directly compared as dynamic MAS baseline."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent software development system using chain collaboration patterns referenced in SC-MAS's edge strategy design."
     },
     {
       "title": "GraphRouter: A graph-based router for LLM selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Graph-based LLM routing method used as single-agent routing baseline (PromptLLM)."
     },
     {
       "title": "More agents is all you need",
       "authors": ["junyou li", "Qin Zhang", "Yangbin Yu"],
       "year": 2024,
       "relevance": "Studies scaling properties of multi-agent systems, relevant to understanding when more agents help vs. hurt."
     },
     {
       "title": "Mixture-of-agents enhances large language model capabilities",
       "authors": ["Junlin Wang", "Jue Wang", "Ben Athiwaratkun"],
       "year": 2025,
       "relevance": "Mixture-of-agents approach for combining LLMs, related to the heterogeneous LLM assignment in SC-MAS."
     },
     {
       "title": "Automated design of agentic systems",
       "authors": ["Shengran Hu", "Cong Lu", "Jeff Clune"],
       "year": 2025,
       "relevance": "Automated agent architecture search using MCTS, related to SC-MAS's search over MAS configurations."
     }
   ]
 }