ai-research-survey

scan.json (26868B)

---

 {
   "paper": {
     "title": "Stronger-MAS: Multi-Agent Reinforcement Learning for Collaborative LLMs",
     "authors": ["Yujie Zhao", "Lanxiang Hu", "Yang Wang", "Minmin Hou", "Hao Zhang", "Ke Ding", "Jishen Zhao"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.11062"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "AT-GRPO, an agent- and turn-wise grouped RL algorithm for multi-agent LLM systems, dramatically improves long-horizon planning tasks from 14-47% to 96-99.5% accuracy. On coding and math benchmarks, it yields average gains of 3.87-7.62% and 9.0-17.93% respectively over single-agent baselines. The paper shows that whether to use role-sharing or role-specialized policies depends on task characteristics, and that training agents jointly in a MAS environment is critical for emergent collaboration.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/pettingllms-ai/PettingLLMs in the abstract. The reproducibility statement also says 'source code used for our experiments is included in the supplementary material.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All datasets used in this study are publicly available' (Sec. 8) and uses standard public benchmarks: APPS, LiveCodeBench, CodeContests, AIME24/25, OlympiadBench, and programmatically generated Sudoku/Sokoban/Plan-Path instances."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency specifications are mentioned. The paper mentions 8×H100 GPUs and Qwen3 models but does not specify library versions or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises 'Upon acceptance, we will release the complete, documented source code' but does not include step-by-step reproduction instructions in the paper itself. Hyperparameters are listed in Appendix C.1 but no runnable commands or scripts are documented."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., '99.00', '24.00') with no confidence intervals or error bars. Figure 6(a) shows shaded bands but the main results tables have no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('outperforms', 'gains of 3.87-7.62%') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report absolute gains and relative improvements with baseline context (e.g., '+84% (+560.0%)' in Fig. 1, gains from baseline shown in parentheses throughout). The reader can assess magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of evaluation instances used for each benchmark. The paper does not discuss whether the sample sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6(a) shows shaded variability bands for training curves, but the main results in Tables 1-4 report single numbers with no standard deviation, variance, or spread measures across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares five variants (Sec. 5.1): Single Agent, Single Agent + GRPO, MAS (prompt-only), MAS + GRPO, and MAS + AT-GRPO with shared/per-role policies. Also compares against MAPORL, MARFT, and CURE in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary works: MAPORL (2025), MARFT (2025), CURE (2025), and GRPO. These are recent and relevant to the multi-agent RL for LLMs space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 4 present ablation studies: training agents in SA vs. MAS, swapped policies (96% → 6%), outcome-only rewards (Table 6), and multi-turn single-agent variants (Tables 7-8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on accuracy across multiple domains: game (Sudoku, Sokoban), planning (Plan-Path), code (LiveCodeBench, APPS, CodeContests as Pass@1), and math (AIME24, AIME25, OlympiadBench)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here — all tasks have automated verifiers (unit tests, symbolic checkers, math verifiers)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5.1 states 'we generate distinct training and validation sets using different random seeds and verify there is no overlap' for Sudoku/Sokoban/Plan-Path. Code uses separate training (APPS/CodeContests train) and evaluation (LiveCodeBench, APPS test, CodeContests test) sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide per-task breakdowns across all 9 benchmarks for each method variant, rather than just aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses 'Limitations of MAS-GRPO' where directly applying GRPO to MAS causes performance degradation (e.g., CodeContests 17.60→10.30). Case studies in Appendix H show before/after RL failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that MAS+GRPO sometimes degrades performance (Tab. 2: CodeContests 17.60→10.30, OlympiadBench 56.50→53.20). Tables 7-8 show multi-turn SA hurts performance. Role-specialized policies sometimes underperform shared policies (e.g., OlympiadBench 39.60 shared vs 35.20 per-role)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 96.0-99.5% on planning (supported by Tables 1-2), 3.87-7.62% coding gains and 9.0-17.93% math gains are supported by the experimental results, though some individual benchmark results are more modest."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (Sec. 5.4): removing MAS training drops Plan-Path from 96% to 16%; swapping policies drops to 6%. These controlled single-variable manipulations adequately support the causal claims about AT-GRPO's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-Agent Reinforcement Learning for Collaborative LLMs' broadly but experiments are limited to Qwen3 models at 1.7B and 8B scales. No other model families are tested. The paper does not bound claims to Qwen3 specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the gains. For example, the MAS framework provides more compute (multiple agents, multiple turns) which could explain gains independently of the AT-GRPO algorithm. The outcome-only ablation partially addresses reward engineering but not compute confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match its measurements: accuracy on specific benchmarks (Sudoku, Sokoban, LiveCodeBench, AIME, etc.). It does not make broader claims about general 'intelligence' or 'reasoning ability' beyond the tested tasks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Qwen3 models at 1.7B and 8B in the no-thinking mode' (Sec. 5.1) and cites the Qwen3 technical report. For comparisons, specific models are named: Phi-3-mini-128k (3.4B), Qwen2.5-Coder-3B-Instruct, Qwen-2.5-7B-Instruct."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 provides full prompt text for all agents across all workflows: Code MAS (coder and tester, generation and refinement phases), Math MAS, Sudoku MAS, Plan-Path (Appendix D), and Sokoban."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 provides detailed hyperparameters: max response length 4096, batch size 128, mini-batch 64, learning rate 1e-6, weight decay 0.01, γ=1.0, λ=1.0, temperature 1.0, top-p=1.0, K=4 samples, T=4 turns, α=1, 150 training steps."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MAS workflows are described in detail: Fig. 2 shows domain-specific workflows, Sec. 4.2 describes the training system architecture (Fig. 4), Algorithm 1 provides the full AT-GRPO procedure, and Appendix C.2 details all agent interaction patterns."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5.1 describes data sources and preprocessing: APPS introductory subset for 1.7B, CodeContests for 8B, Polaris-Dataset-53K for math. Train/test splits are described with random seed separation and overlap verification."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations section. There is an Ethics Statement (Sec. 7) and Reproducibility Statement (Sec. 8) but neither serves as a limitations discussion. The brief mentions of limitations in Sec. 5.2 ('Limitations of MAS-GRPO') refer to a baseline method, not limitations of the proposed approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The ethics statement mentions generic dual-use risks and societal biases but does not discuss specific threats to the validity of the experimental findings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to Qwen3 models, does not discuss whether results transfer to other model families, and does not state limitations of the tested domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual run outputs, per-example results, training logs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5.1 describes data sources: APPS training split (introductory), CodeContests, Polaris-Dataset-53K, and programmatic generation of Sudoku/Sokoban/Plan-Path instances with distinct seeds for train/test."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks and programmatically generated instances."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: Sec. 5.1 describes dataset selection, Appendix C.1 covers training details, and reward designs for each task are fully specified in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are from UCSD and Intel Corporation but no funding disclosures are made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of California, San Diego and Intel Corporation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Intel Corporation is listed as an affiliation for three authors (Yang Wang, Minmin Hou, Ke Ding). Intel has commercial interest in LLM training infrastructure. No discussion of whether this creates a conflict, and no funding independence statement."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for the Qwen3 models used. This is relevant since benchmarks like APPS, AIME, and CodeContests may overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "For the programmatically generated tasks (Sudoku, Sokoban, Plan-Path), the paper verifies no train/test overlap via distinct seeds. However, for public benchmarks (APPS, LiveCodeBench, AIME, CodeContests), no discussion of whether Qwen3's pretraining data included these benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of contamination risk for APPS (2021), AIME (2024/2025), CodeContests, or OlympiadBench. These public benchmarks could plausibly be in Qwen3's training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G.3 provides empirical latency: 'one on-policy iteration for the single-agent baseline requires approximately 4 minutes for rollout and 1 minute for AT-GRPO training' on 4×H100 GPUs. MAS setting takes ~8 min rollout + 2 min training."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5.1 states 'All runs use a single node with 8× H100 GPUs' and Appendix C.1 specifies 150 training steps. Appendix G.3 provides per-iteration timing."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Tables 1-2 show single numbers per configuration with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the reported results. It appears to be single-run results given the absence of variance reporting."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper states α=1 'without further tuning' (Sec. 5.1) but does not disclose whether other hyperparameters were searched."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why K=4, T=4, learning rate 1e-6, etc. were chosen. Some appear fixed ('without further tuning') but the selection process is not documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across 9 benchmarks × 6 methods × 2 model scales with no statistical tests at all, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their AT-GRPO system against their own implementations of baselines (GRPO, MAS) without acknowledging potential bias. Results from MAPORL, MARFT, and CURE are cited from original papers but not independently replicated."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MAS+AT-GRPO uses multiple agents with tree sampling (K=4 branches per agent per turn), consuming substantially more compute than single-agent GRPO. Appendix G discusses complexity scaling but does not compare performance at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure what is claimed. For example, Sudoku and Sokoban are used as proxies for 'game' and 'planning' ability without discussing construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The MAS framework itself acts as a scaffold. When comparing SA+GRPO vs MAS+AT-GRPO, the difference includes both the algorithm (AT-GRPO vs GRPO) and the scaffold (MAS vs SA). The ablation in Table 4 partially addresses this but does not fully disentangle scaffold from algorithm effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. APPS (2021), AIME24/25, CodeContests, and OlympiadBench could all be in Qwen3's pretraining data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For the RL-trained models, the training and test benchmarks are separated, but pretraining leakage is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "For programmatic tasks, independence is ensured by distinct random seeds. For public benchmarks, no discussion of whether train/test examples share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection methods (canary strings, n-gram overlap, membership inference) are applied to any benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "AT-GRPO boosts long-horizon planning accuracy from 14-47% (single-agent RL) to 96.0-99.5%",
       "evidence": "Tables 1-2: Plan-Path goes from 11%/47% (SA+GRPO) to 96-97%/93-96% (MAS+AT-GRPO) for 1.7B/8B; Sokoban from 3%/14% to 10-11.5%/96-98%",
       "supported": "strong"
     },
     {
       "claim": "Average coding gains of 3.87-7.62% and math gains of 9.0-17.93% over single-agent baseline",
       "evidence": "Tables 1-2 show per-benchmark gains. Coding gains range from +1.20 to +16.30 across benchmarks; math from +1.60 to +38.70. Average figures appear computed across benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "RL training on MAS reinforces role-specific specialization",
       "evidence": "Table 4 ablation: swapping trained role-specific policies drops Plan-Path from 96% to 6%. Fig. 6(a) shows diverging reward curves for tool and plan agents.",
       "supported": "strong"
     },
     {
       "claim": "MAS+AT-GRPO scales effectively with increasing number of agents while MAS+GRPO saturates",
       "evidence": "Fig. 5(b): AT-GRPO scales from 18.2% to 47.7% on AIME24 as agent count increases, while GRPO saturates at 34.1%.",
       "supported": "moderate"
     },
     {
       "claim": "On-policy RL training within MAS is critical; training agents in SA and combining in MAS gives limited benefit",
       "evidence": "Table 4: Training in SA, eval in MAS achieves only 16% on Plan-Path vs 96% for MAS RL training.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or significance reporting",
       "detail": "All main results (Tables 1-4) are single point estimates with no error bars, standard deviations, or significance tests. Given that RL training is known to have high variance across seeds (Henderson et al. 2018), single-run results are insufficient to support the comparative claims."
     },
     {
       "flag": "Compute confound not addressed",
       "detail": "MAS+AT-GRPO uses multiple agents with tree-structured sampling (K=4 branches per agent per turn), consuming far more compute than single-agent GRPO. Performance is never compared at matched compute budgets, making it unclear whether gains come from the algorithm or simply from more compute."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations discussion despite making broad claims about a general-purpose MAS RL training method."
     },
     {
       "flag": "Industry affiliation without conflict disclosure",
       "detail": "Three authors are from Intel Corporation, which has commercial interest in LLM training infrastructure. No conflicts of interest or funding statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "relevance": "Foundational multi-agent LLM framework with role-sharing architecture."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Multi-agent collaborative framework for software development tasks."
     },
     {
       "title": "MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution",
       "authors": ["Wei Tao"],
       "year": 2024,
       "relevance": "Multi-agent LLM system applied to software engineering (GitHub issue resolution)."
     },
     {
       "title": "ToolRL: Reward is All Tool Learning Needs",
       "authors": ["Cheng Qian"],
       "year": 2025,
       "arxiv_id": "2504.13958",
       "relevance": "RL-based tool use training for LLMs, relevant to agentic AI capabilities."
     },
     {
       "title": "CURE: Co-evolving LLM Coder and Unit Tester via Reinforcement Learning",
       "authors": ["Yinjie Wang"],
       "year": 2025,
       "arxiv_id": "2506.03136",
       "relevance": "RL co-training of coder and tester agents, directly compared baseline."
     },
     {
       "title": "MAPORL: Multi-Agent Post-Co-Training for Collaborative Large Language Models with Reinforcement Learning",
       "authors": ["Chanwoo Park"],
       "year": 2025,
       "relevance": "Multi-agent RL post-training for LLMs via debate, directly compared baseline."
     },
     {
       "title": "MARFT: Multi-Agent Reinforcement Fine-Tuning",
       "authors": ["Junwei Liao"],
       "year": 2025,
       "arxiv_id": "2504.16129",
       "relevance": "Multi-agent reinforcement fine-tuning framework, directly compared baseline."
     },
     {
       "title": "SPIRAL: Self-Play on Zero-Sum Games Incentivizes Reasoning via Multi-Agent Multi-Turn Reinforcement Learning",
       "authors": ["Bo Liu"],
       "year": 2025,
       "arxiv_id": "2506.24119",
       "relevance": "Self-play RL for LLMs in multi-agent settings."
     },
     {
       "title": "RAGen: Understanding Self-Evolution in LLM Agents via Multi-Turn Reinforcement Learning",
       "authors": ["Zihan Wang"],
       "year": 2025,
       "arxiv_id": "2504.20073",
       "relevance": "Multi-turn RL for LLM agent self-evolution."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Failure analysis of multi-agent LLM systems, relevant to understanding MAS limitations."
     },
     {
       "title": "GiGPO: Group-in-Group Policy Optimization for LLM Agent Training",
       "authors": ["Lang Feng"],
       "year": 2025,
       "arxiv_id": "2505.10978",
       "relevance": "Group-relative policy optimization for LLM agents, foundational to AT-GRPO's design."
     },
     {
       "title": "CodeSteer: Symbolic-Augmented Language Models via Code/Text Guidance",
       "authors": ["Yongchao Chen"],
       "year": 2025,
       "arxiv_id": "2502.04350",
       "relevance": "Symbolic task benchmark (SymBench) used for Sudoku/Sokoban/Plan-Path evaluation."
     }
   ]
 }