ai-research-survey

scan.json (25075B)

---

 {
   "paper": {
     "title": "Single-Agent Scaling Fails Multi-Agent Intelligence: Towards Foundation Models with Native Multi-Agent Intelligence",
     "authors": ["Shuyue Hu", "Haoyang Yan", "Yiqun Zhang", "Yang Chen", "Dongzhan Zhou", "Lei Bai"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.08743",
     "doi": "10.48550/arXiv.2512.08743"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Across 41 LLMs from the Qwen and LLaMA families (0.5B–235B parameters), single-agent task performance improves substantially across model generations while multi-agent understanding gains are modest and multi-agent planning gains are near-flat. Logarithmic regression shows diminishing returns: even models with 0.6–0.8 SA accuracy show high variance in MA planning (R²≈0.6). The authors argue multi-agent intelligence requires intentional training beyond single-agent scaling.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or mention of code release found anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 7 benchmarks used are publicly available (MATH-500, HumanEval, MMLU-Pro, GPQA, ToMBench, EmoBench, CoordinationQA) and all models are open-weight (Qwen, LLaMA families on HuggingFace)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM with 'officially recommended hyperparameters' but provides no environment specification, dependency list, or hardware details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts provided. The experimental procedure is described at a high level but lacks step-by-step reproducibility guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (single accuracy values) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SA scaling does not yield MA improvement but provides no statistical tests for these comparisons. The logarithmic regressions in Figure 3 report R² and p-values but no tests are applied to the generational comparisons that form the paper's central claim."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: 'accuracy nearly triples from Qwen-1-1.8B to Qwen-3-1.7B' for SA tasks vs. 'only about 30%' increase for MA planning. Specific accuracy values (e.g., 0.23→0.64 for SA, 0.44→0.55 for MA understanding) provide baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 41 models were selected, why only two model families were used, or whether this sample is sufficient to support the generalization claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviation, variance across seeds, or spread measures reported for any benchmark."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper's design inherently uses single-agent benchmarks (MATH-500, HumanEval, MMLU-Pro, GPQA) as baselines for comparison against multi-agent benchmarks, and compares across model generations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models span from 2023 to 2025 including recent releases (Qwen3, LLaMA-3.3). Benchmarks include contemporary choices like MMLU-Pro and GPQA."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing models on existing benchmarks; there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 7 benchmarks with multiple sub-scores (e.g., CoordinationQA has EC, ToM, JP; EmoBench has EU, EA; ToMBench has Tasks and Abilities)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; the paper evaluates LLM performance on existing benchmarks with automated scoring."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used are standard held-out test sets (MATH-500, HumanEval, MMLU-Pro, GPQA, ToMBench, EmoBench, CoordinationQA). The paper does not tune on these benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 in the Appendix provide per-benchmark, per-model breakdowns. CoordinationQA is broken into EC, ToM, and JP sub-scores. EmoBench into EU and EA. ToMBench into Tasks and Abilities."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of failure cases, error analysis, or qualitative examples of where models fail on multi-agent tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding is essentially a negative result: single-agent scaling fails to produce multi-agent intelligence. The paper also notes cases where MA planning accuracy declines across generations (e.g., LLaMA-3 to LLaMA-3.1 for 70B)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'scaling single-agent performance alone does not automatically yield robust multi-agent intelligence' across 41 LLMs and 7 benchmarks, which is supported by the results in Section 3, Figures 2-3, and Tables 1-2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'scaling single-agent abilities does not yield comparable improvements in multi-agent abilities' (Section 3.2). However, the study is observational — it observes correlations across model generations but cannot control for confounds like different training data, RLHF procedures, or instruction tuning approaches that vary across generations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim broadly about 'foundation models' and 'multi-agent intelligence' but only test two open-weight model families (Qwen, LLaMA). Closed-source models (GPT, Claude, Gemini) and other open families are not tested. The paper does not bound its claims to the tested families."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed pattern. For example, it does not consider whether the MA benchmarks are simply harder (ceiling effects), whether instruction tuning varies across generations, or whether the MA benchmarks have lower construct validity."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates performance on ToMBench, EmoBench, and CoordinationQA with 'multi-agent intelligence' without discussing whether these QA-style benchmarks actually capture the multi-agent capabilities described in Section 2 (real-time adaptation, efficient communication, decentralized planning). The paper even acknowledges in Section 4.2 that QA-style evaluation is limited, yet does not apply this caveat to its own results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-2 list specific model versions (e.g., 'Qwen-1.8B-Chat', 'Qwen2.5-72B-Instruct', 'Llama-2-7b-chat-hf', 'Meta-Llama-3-70B-Instruct') and the paper states these are 'official checkpoints' from HuggingFace."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are provided. The paper does not describe how benchmarks were administered to the models beyond mentioning vLLM deployment."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states only 'officially recommended hyperparameters' without specifying temperature, top-p, max tokens, or any other setting."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; models are evaluated directly on QA-style benchmarks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how benchmark data was preprocessed, formatted, or administered to the models. No mention of answer extraction or parsing procedures."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not bound claims to the tested model families or acknowledge that QA-style MA benchmarks may not capture the full scope of multi-agent intelligence."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated accuracy scores are provided in Tables 1-2. No raw model outputs, per-example predictions, or detailed results are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the model selection criteria (two open-weight families with documented histories, 41 models from 0.5B to 235B, instruction-tuned versions) and the 7 benchmarks used with their characteristics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from running benchmarks to final accuracy numbers. Answer extraction, parsing, and scoring procedures are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Shanghai Artificial Intelligence Laboratory, with email addresses at pjlab.org.cn."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Shanghai AI Laboratory is a research institution that could have interests in the outcomes of FM evaluation research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 41 pre-trained models on benchmarks but never states any model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any benchmark data appeared in the models' training data, despite using well-known benchmarks like HumanEval (published 2021) and MATH-500."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks (HumanEval, MMLU-Pro) were available online before many tested models were trained. No discussion of contamination risk, which could differentially affect SA vs MA benchmarks and confound the central claim."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, token counts, or wall-clock time reported despite running 41 models across 7 benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total GPU hours, hardware used, or computational budget for running the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-run evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single accuracy values."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper mentions 'officially recommended hyperparameters' but does not report what these are or whether alternatives were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of configuration selection. The paper uses 'officially recommended hyperparameters' without justification or comparison to alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 41 models across 7 benchmarks but applies no multiple comparison correction. The logarithmic regressions report p-values without correction."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing models on existing benchmarks; there is no 'own system' being compared against baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis despite comparing models ranging from 0.5B to 235B parameters with vastly different compute requirements."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses QA-style benchmarks (ToMBench, EmoBench, CoordinationQA) as proxies for 'multi-agent intelligence' without discussing whether these benchmarks actually measure the capabilities described in Section 2. Section 4.2 acknowledges QA limitations generally but does not apply this critique to its own evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; models are evaluated directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MATH-500 (2023), and other SA benchmarks predate many tested models. If SA benchmarks are more contaminated than MA benchmarks, this confounds the central claim. Not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information through context or prompt formatting."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples share structural similarities with training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied despite the critical confound: if SA benchmarks are more contaminated than MA benchmarks, the observed SA>MA improvement pattern could be an artifact."
       }
     }
   },
   "claims": [
     {
       "claim": "Scaling single-agent performance does not automatically yield robust multi-agent intelligence across 41 LLMs and 7 benchmarks.",
       "evidence": "Section 3.2, Figures 2-3: SA accuracy of ~8B Qwen models rises from 0.23 to 0.64 across generations while MA understanding only increases from 0.44 to 0.55 and MA planning remains 0.2-0.35.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-agent planning performance shows little to no improvement across model generations and in some cases declines.",
       "evidence": "Section 3.2, Figure 2: For ~8B models, MA planning accuracy stabilizes between 0.2-0.35 across Qwen-1 to Qwen-2.5 and LLaMA-2 to LLaMA-3.1. For ~70B LLaMA models, accuracy slightly declines from LLaMA-3 to LLaMA-3.1.",
       "supported": "moderate"
     },
     {
       "claim": "Even models with high single-agent accuracy (0.6-0.8) exhibit substantial variability in multi-agent planning accuracy.",
       "evidence": "Figure 3: Logarithmic regression with R²=0.608 (Qwen) and R²=0.576 (LLaMA) shows substantial unexplained variance. 95% CI bands are wide.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Contamination confound threatens central claim",
       "detail": "SA benchmarks like HumanEval (2021) and MATH-500 (2023) are older and more likely contaminated in newer model training data than MA benchmarks like CoordinationQA. Differential contamination could inflate SA improvements relative to MA, making the SA-MA gap an artifact. This is never discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations, threats to validity, or scope boundary discussion despite making strong generalization claims about 'foundation models' based on only two model families."
     },
     {
       "flag": "Benchmark difficulty confound",
       "detail": "The paper does not consider whether MA benchmarks are simply harder or have different performance ceilings than SA benchmarks, which could explain slower improvement independent of any fundamental SA-MA gap."
     },
     {
       "flag": "No variance or multi-run results",
       "detail": "All 287 data points (41 models × 7 benchmarks) appear to be single-run evaluations with no uncertainty quantification, making it impossible to assess result stability."
     },
     {
       "flag": "Overclaiming from observational data",
       "detail": "The paper treats the observed correlation between model generation and differential SA/MA improvement as evidence of a fundamental inability, but model generations differ in training data, RLHF, and instruction tuning — not just scale. The causal claim is not supported by the observational design."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language model based multi-agents: A survey of progress and challenges",
       "authors": ["Tao Guo", "Xiao Chen"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Comprehensive survey of LLM-based multi-agent systems, relevant to the survey's coverage of agentic AI."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges claims of emergent LLM abilities, directly relevant to methodology quality assessment in AI evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational for AI code generation evaluation."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper, central to the debate about whether capabilities emerge from scale."
     },
     {
       "title": "ToMBench: Benchmarking Theory of Mind in Large Language Models",
       "authors": ["Zhuang Chen", "Jincenzi Wu"],
       "year": 2024,
       "arxiv_id": "2402.15052",
       "relevance": "Multi-agent understanding benchmark used in this study, relevant to LLM capability evaluation."
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
       "authors": ["Yubo Wang", "Xueguang Ma"],
       "year": 2024,
       "relevance": "Major LLM evaluation benchmark, relevant to AI capability assessment methodology."
     },
     {
       "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
       "authors": ["David Rein", "Betty Li Hou"],
       "year": 2024,
       "relevance": "Graduate-level QA benchmark for LLM evaluation, relevant to AI capability assessment."
     },
     {
       "title": "Large Language Models Miss the Multi-Agent Mark",
       "authors": ["Emanuele La Malfa"],
       "year": 2025,
       "relevance": "Directly related work arguing FMs lack multi-agent capabilities, relevant to survey's coverage of LLM limitations."
     },
     {
       "title": "Secret collusion among ai agents: Multi-agent deception via steganography",
       "authors": ["Sumeet Motwani"],
       "year": 2024,
       "relevance": "Demonstrates multi-agent safety risks (covert collusion), relevant to AI safety assessment."
     },
     {
       "title": "Multi-agent risks from advanced ai",
       "authors": ["Lewis Hammond"],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Taxonomy of multi-agent AI risks, directly relevant to AI safety research methodology."
     },
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of multi-agent LLM system failures, relevant to agentic AI reliability research."
     },
     {
       "title": "Towards a Science of Scaling Agent Systems",
       "authors": ["Yoonsang Kim"],
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Framework for scaling agent systems, directly relevant to agentic AI methodology."
     }
   ]
 }