ai-research-survey

scan.json (28206B)

---

 {
   "paper": {
     "title": "MARSHAL: Incentivizing Multi-Agent Reasoning via Self-Play with Strategic LLMs",
     "authors": ["Huining Yuan", "Zelai Xu", "Zheyue Tan", "Xiangmin Yi", "Mo Guang", "Kaiwen Long", "Haojia Hui", "Boxun Li", "Xinlei Chen", "Bo Zhao", "Xiao-Ping Zhang", "Chao Yu", "Yu Wang"],
     "year": 2025,
     "venue": "ICLR 2026",
     "arxiv_id": "2510.15414"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code, model checkpoints, and training scripts required to reproduce the findings of this paper are publicly available at https://github.com/thu-nics/MARSHAL' in the Reproducibility Statement."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The games use OpenSpiel and VS-Bench (public frameworks), and evaluation uses standard public benchmarks (MATH500, GSM8K, AIME24, GPQA-Diamond, etc.). No proprietary datasets were created."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using ROLL, vLLM, Megatron-LM, and OpenSpiel but does not provide a requirements.txt, Dockerfile, or detailed library version specifications. Only framework names are given, not specific versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The Reproducibility Statement says the repository 'includes all necessary configurations to replicate our key experiments.' Combined with detailed hyperparameters in Table 5 and Appendix B-D, this provides step-by-step reproducibility."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 3, 4, and Figures 3-6 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., 'up to 10.0% on AIME') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports improvements with baseline context, e.g., 'up to 28.7% performance improvements in held-out games', '10.0% on AIME' (from 56.67% to 66.67%), and percentage point differences throughout Tables 1, 6-7."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "For game evaluation, the paper states '1000 games' but provides no justification for why 1000 is sufficient. For reasoning benchmarks, sample sizes depend on the benchmark (AIME has only 30 problems, AMC has 40) with no discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Qwen3-4B base model and SPIRAL (a recent self-play method) across all evaluations (Section 4.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SPIRAL (Liu et al., 2025) is a concurrent/recent work on self-play for LLMs. Qwen3-4B is a current model. Both are appropriate contemporary baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 ablates both key components: turn-level advantage estimator and agent-specific advantage normalization. Additional ablations compare self-play vs. fixed-opponent (Table 3) and decoupling algorithm vs. game environments (Appendix F)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates across multiple dimensions: game return (normalized), multiple reasoning benchmarks (MATH500, GSM8K, AQUA-RAT, AIME24, AMC23, MMLU-STEM, GPQA-Diamond), and failure mode analysis."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated — game returns against MCTS/NE opponents and automated benchmark scoring. No human evaluation of reasoning quality or game play is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly partitions games into training (Tic-Tac-Toe, Kuhn Poker, Mini Hanabi) and held-out testing (Connect Four, Leduc Hold'em, Simple Hanabi) sets. Reasoning benchmarks serve as zero-shot out-of-domain evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by individual game (Figure 3), by individual benchmark (Table 1), by first-move/second-move player roles, and by failure category (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides a detailed failure mode analysis on GPQA-Diamond within MAD, categorizing failures into System Design Issues, Inter-Agent Misalignment, and Task Verification (Figure 5), with sub-category breakdown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The fixed-opponent ablation (Table 3) shows severe performance degradation (strategic mode collapse). The ablation removing components shows performance drops. The Hanabi specialist underperforms in MAD (competitive setting), showing skill mismatch."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'up to 28.7% performance improvements in held-out games' matches Leduc Hold'em generalist results. Claims of '10.0% on AIME, 7.6% on GPQA-Diamond, and 3.5% on average' are supported by Table 1 data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('MARSHAL incentivizes multi-agent reasoning') and supports them with controlled ablation studies (Section 4.5) that isolate individual components. The self-play vs. fixed-opponent comparison and algorithm-vs-games decoupling (Appendix F) provide reasonable causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-Agent Reasoning' generally but experiments are limited to Qwen3-4B and Qwen3-8B, two-player games only, and specific MAS frameworks (MAD, AutoGen). The Discussion acknowledges the two-player limitation but the abstract and title do not bound the claims to these settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the generalization results. For example, the reasoning improvements could stem from additional training compute or exposure to structured turn-based reasoning rather than specifically from game-theoretic self-play. No robustness checks against these confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims game self-play develops 'multi-agent reasoning capabilities' but measures this via benchmark accuracy (MATH, AIME, GPQA). The gap between benchmark scores and 'reasoning capability' is not acknowledged. Benchmark accuracy is a proxy for reasoning ability but the paper treats them as equivalent."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Qwen3-4B' and 'Qwen3-8B' without specifying exact model versions, snapshot dates, or checkpoint identifiers. The reference is to Yang et al. (2025) but no specific version string is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full game prompts are provided in Appendix I (Listings 1-5) for all five games, including system prompts and user prompts with complete text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides comprehensive hyperparameters including learning rate (1e-6), batch size (128), PPO clip (0.2), KL coefficient (0.2), sampling temperature (0.5/0.6), top-P (0.99), top-K (100), and optimizer settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The self-play framework is described in detail: Figure 2 shows the architecture, Section 3 describes the GRPO-based training loop, game environments serve as the interaction scaffold, and the player trajectory generation is fully documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The game environments are standardized via OpenSpiel/VS-Bench. The reward normalization across games is documented (Section 3.4). Evaluation protocols including opponent types and evaluation scripts are specified in Appendix D."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) includes substantive discussion of limitations: the study uses only two-player games, and scaling to N-player environments introduces 'non-stationarity, population diversity, and credit assignment' challenges."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion identifies specific threats: two-player games may not capture N-player dynamics, classic games differ from complex social sandboxes, and the challenges of 'non-stationarity, population diversity, and credit assignment' in scaling."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Discussion mentions two-player limitation, the paper does not explicitly state what the results do NOT show. No explicit statement about model-specificity (only tested on Qwen3), framework-specificity (only MAD and AutoGen), or language-specificity of the results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (game trajectories, per-example benchmark results) is made available. Only aggregated results in tables and figures are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: self-play trajectories generated via the MARSHAL framework, 1000 games for strategic evaluation, standard benchmark datasets with evaluation scripts specified (Appendix D)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from model self-play and standard benchmark evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: self-play generates trajectories → turn-level advantage estimation → GRPO training. Evaluation pipeline: model generates responses → evaluated against MCTS/NE opponents or benchmark scoring scripts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding: National Natural Science Foundation of China, Ant Group, BNRist, Beijing Innovation Center for Future Chips, Shenzhen Key Laboratory, Tsinghua Shenzhen International Graduate School."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Tsinghua University (multiple departments), Aalto University, Li Auto Inc., and Infinigence AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Ant Group (a major tech company) is listed as a sponsor. Li Auto Inc. and Infinigence AI are corporate affiliations of some authors. These entities have potential commercial interest in LLM agent capabilities but this conflict is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Li Auto Inc. and Infinigence AI may have financial interests related to LLM agent capabilities, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for Qwen3-4B or Qwen3-8B. These models are evaluated on benchmarks like MATH500, GSM8K, and AIME24 which may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems (MATH500, GSM8K, AIME24, etc.) appeared in Qwen3's training data. These are well-known public benchmarks likely included in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), MATH (2021), MMLU (2020), and AIME24 are all publicly available benchmarks that predate Qwen3's training. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed is reported for any evaluation setting."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states 'Models were trained on a single server with 8 NVIDIA H100 GPUs' and training was for 200 optimization steps. However, total GPU hours or wall-clock time is not explicitly stated — the hardware configuration is provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of training runs is not stated. For game evaluation, 1000 games are stated but it is unclear if this is a single evaluation pass or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Table 5 presents a single configuration with no discussion of how it was selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents one hyperparameter configuration (Table 5) without justifying how it was selected. Only the length penalty α is ablated (Appendix H) — other hyperparameters appear chosen without documented justification."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is moot. However, the paper makes many simultaneous comparisons across 7 benchmarks, 6 games, multiple model variants without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their MARSHAL system against their own re-implementation of SPIRAL and MT-GRPO without acknowledging potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MARSHAL requires additional RL training beyond the base model, but performance is not analyzed as a function of compute. SPIRAL presumably uses similar compute but this is not verified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses AIME, GPQA-Diamond, MATH500 etc. as proxies for 'multi-agent reasoning capability' without discussing whether these benchmarks actually measure what is claimed. Benchmark accuracy in a multi-agent framework may reflect many factors beyond multi-agent reasoning."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the same MAS frameworks (MAD, AutoGen via MASLab) across all model comparisons, controlling for the scaffold confound. The comparison is models within the same framework, not across different frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qwen3's training data includes solutions to MATH500, GSM8K, AIME24, or other benchmarks used for evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the game observation format or benchmark evaluation setup leaks information not available in realistic settings."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The same model architecture and training procedure are used across all evaluations."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied for any of the evaluation benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "MARSHAL agents trained from Qwen3-4B achieve up to 28.7% performance improvements in held-out games.",
       "evidence": "Figure 3 and Tables 14-15 show the generalist model achieves 28.7% improvement on Leduc Hold'em (first move) compared to the Qwen3-4B baseline.",
       "supported": "strong"
     },
     {
       "claim": "MARSHAL achieves zero-shot performance gains of up to 10.0% on AIME, 7.6% on GPQA-Diamond, and 3.5% on average when integrated into MAS frameworks.",
       "evidence": "Table 1: AutoGen generalist achieves 66.67% on AIME vs 56.67% baseline (+10.0%), MAD generalist achieves 45.45% on GPQA vs 37.88% baseline (+7.57%), and 3.51% average gain in MAD.",
       "supported": "moderate"
     },
     {
       "claim": "Self-play is essential for generalization; training against fixed opponents leads to overfitting and strategic mode collapse.",
       "evidence": "Table 3 shows the Kuhn Poker fixed-opponent model collapses to 0.0% on non-poker games, while the self-play model generalizes across all games.",
       "supported": "strong"
     },
     {
       "claim": "Both turn-level advantage estimator and agent-specific normalization are critical components.",
       "evidence": "Table 4 shows removing either component degrades performance. Turn-level removal causes largest drop in long-horizon games; agent-specific removal causes largest drop in competitive games with asymmetric returns.",
       "supported": "strong"
     },
     {
       "claim": "MARSHAL cultivates generalizable multi-agent reasoning skills including role understanding and intent recognition.",
       "evidence": "Table 2 provides qualitative examples of reasoning traces. Figure 5 shows quantitative reduction in Inter-Agent Misalignment failures (11.5% reduction) on GPQA-Diamond.",
       "supported": "moderate"
     },
     {
       "claim": "MARSHAL scales to larger models (Qwen3-8B) with consistent improvements.",
       "evidence": "Tables 6-7 (Appendix E) show improvements on 8B model: 2.60% average gain in MAD, 3.90% average in AutoGen.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MARSHAL, an end-to-end RL framework using self-play in cooperative and competitive games, significantly improves multi-agent reasoning in LLMs. Training Qwen3-4B via self-play yields up to 28.7% improvement in held-out games and consistent zero-shot gains (up to 10% on AIME) when integrated into multi-agent systems like MAD and AutoGen. The turn-level advantage estimator and agent-specific normalization are both critical for learning, and self-play is essential over fixed-opponent training to avoid strategic mode collapse. Skills acquired in games (role understanding, intent recognition) transfer to downstream multi-agent reasoning tasks.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are reported as single point estimates across all experiments (games, benchmarks, ablations). With small benchmark sizes (AIME has 30 problems, AMC has 40), a few-problem difference can appear as a large percentage gain. Without variance across seeds or runs, the statistical reliability of improvements is unknown."
     },
     {
       "flag": "Small benchmark sample sizes inflate apparent gains",
       "detail": "AIME24 has only 30 problems. The claimed '10.0% improvement' on AIME corresponds to 3 additional correct answers (from 17 to 20 out of 30). AMC23 has ~40 problems. These tiny absolute differences may not be statistically significant."
     },
     {
       "flag": "No contamination analysis for standard benchmarks",
       "detail": "GSM8K, MATH, MMLU, and AIME problems are widely available online. Qwen3 likely saw them during pretraining. The MARSHAL training adds RL on games but the base model's contamination is never assessed. Improvements in MAS settings could partially reflect the RL training helping the model better retrieve memorized solutions."
     },
     {
       "flag": "Qualitative evidence presented as primary for generalization mechanism",
       "detail": "The core claim that game skills generalize to reasoning is supported primarily by cherry-picked reasoning trace examples (Table 2) and a failure mode analysis on a single benchmark (GPQA in MAD). Alternative explanations (e.g., additional training compute, regularization effects of multi-task learning) are not considered."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Key work on RL for LLM reasoning that MARSHAL builds upon, demonstrating reward design for reasoning models."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu"],
       "year": 2024,
       "relevance": "Cooperative multi-agent framework used as evaluation testbed for MARSHAL's generalization to MAS."
     },
     {
       "title": "Encouraging divergent thinking in large language models through multi-agent debate",
       "authors": ["Tian Liang"],
       "year": 2023,
       "arxiv_id": "2305.19118",
       "relevance": "MAD framework used as competitive evaluation testbed; foundational work on LLM multi-agent debate."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent system for software development, representative of static role-based MAS workflows."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Multi-agent framework with specialized roles for software development tasks."
     },
     {
       "title": "SPIRAL: Self-play on zero-sum games incentivizes reasoning via multi-agent multi-turn reinforcement learning",
       "authors": ["Bo Liu"],
       "year": 2025,
       "arxiv_id": "2506.24119",
       "relevance": "Primary baseline; concurrent work on self-play for LLM reasoning in competitive games only."
     },
     {
       "title": "Self-playing adversarial language game enhances LLM reasoning",
       "authors": ["Pengyu Cheng"],
       "year": 2024,
       "relevance": "SPAG: prior work showing self-play in adversarial language games can enhance LLM reasoning."
     },
     {
       "title": "CAMEL: Communicative agents for 'mind' exploration of large language model society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "relevance": "Cooperative multi-agent framework for general reasoning and question answering."
     },
     {
       "title": "SWE-RL: Advancing LLM reasoning via reinforcement learning on open software evolution",
       "authors": ["Yuxiang Wei"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "Applies RL to software engineering tasks, extending reasoning RL to coding domain."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Failure taxonomy for multi-agent LLM systems adopted for MARSHAL's failure mode analysis."
     },
     {
       "title": "Reinforcing multi-turn reasoning in LLM agents via turn-level credit assignment",
       "authors": ["Siliang Zeng"],
       "year": 2025,
       "arxiv_id": "2505.11821",
       "relevance": "MT-GRPO: concurrent work on turn-level credit assignment that MARSHAL compares against."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO which forms the algorithmic foundation for MARSHAL's training objective."
     }
   ]
 }