ai-research-survey

scan.json (24404B)

---

 {
   "paper": {
     "title": "BenchMARL: Benchmarking Multi-Agent Reinforcement Learning",
     "authors": [
       "Matteo Bettini",
       "Amanda Prorok",
       "Vincent Moens"
     ],
     "year": 2023,
     "venue": "arXiv preprint (under review)",
     "arxiv_id": "2312.01472"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly provides the GitHub URL: https://github.com/facebookresearch/BenchMARL. The library is described as open-sourced."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The experiments use publicly available environments (VMAS, SMACv2, MPE, SISL, MeltingPot). Interactive benchmark results are published on Wandb, and fine-tuned configurations are in the repository's conf folder. No proprietary data is involved."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide a requirements.txt, Dockerfile, or detailed dependency/version listing. It mentions TorchRL, Hydra, and PyTorch as dependencies but does not specify library versions or provide an environment specification file."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes how to run benchmarks from one-line command inputs using Hydra. The paper states fine-tuned hyperparameters are in the fine_tuned/vmas folder in the repository, and references examples and notebooks. The execution diagram (Fig. 2) shows the workflow."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 4 and Fig. 5 explicitly report '95% stratified bootstrap confidence intervals over 3 random seeds for each experiment.' Shaded regions are visible in the sample efficiency curves."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "While the paper uses stratified bootstrap confidence intervals, it does not perform formal significance tests (e.g., p-values, t-tests) when comparing algorithms. Claims like 'MA versions obtain the best overall performance' are made based on visual inspection of plots and aggregate scores without statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports aggregate scores (median, IQM, mean, optimality gap) with confidence intervals in Fig. 4c, but does not report effect sizes (e.g., Cohen's d) for pairwise algorithm comparisons. Differences between algorithms are discussed qualitatively rather than quantified."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Experiments use 3 random seeds per algorithm-task combination. No justification is given for why 3 seeds is sufficient, and no power analysis is discussed. Three seeds is a small number for reliable statistical inference in RL."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% stratified bootstrap confidence intervals over 3 random seeds, which provides a measure of spread. The IQM with confidence intervals across seeds is shown in all experimental plots (Figs. 4 and 5)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 9 different MARL algorithms against each other (MAPPO, IPPO, MADDPG, IDDPG, MASAC, ISAC, QMIX, VDN, IQL), serving as mutual baselines. This is appropriate for a benchmarking paper."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The algorithms include contemporary methods (MAPPO 2021, MASAC/ISAC as novel implementations) alongside established baselines (MADDPG 2017, QMIX 2018, VDN 2017, IQL 1993). The mix of recent and classic algorithms is appropriate for establishing a benchmark."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is provided. The paper does not ablate library components (e.g., parameter sharing, vectorized training, different model architectures). The experiments only compare algorithms, not the design choices of BenchMARL itself."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 4 reports multiple aggregate metrics: median, IQM (inter-quartile mean), mean, and optimality gap. Performance profiles and sample efficiency curves are also provided."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for a MARL benchmarking library. The outputs are algorithmic performance metrics in simulation environments, which are fully automated."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a reinforcement learning benchmarking paper, not a supervised learning paper. There is no train/test split concept in the traditional sense -- agents are trained and evaluated in simulation environments."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 5 provides individual task results (Sampling, Navigation, Balance) alongside the aggregate results in Fig. 4. This allows readers to see how algorithms perform on each task separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses why Q-Learning algorithms (IQL, VDN, QMIX) 'perform suboptimally compared to the actor-critic ones,' attributing this to discrete action discretization in continuous control tasks. This is a substantive discussion of where certain approaches fail."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Q-Learning algorithms underperform actor-critic methods on the VMAS tasks, and discusses the likely reason (discretized actions in continuous control). This is a negative result for those algorithm classes on these tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims BenchMARL is 'the first MARL training library created to enable standardized benchmarking across different algorithms, models, and environments.' The paper supports this by describing the library architecture and demonstrating it experimentally. The abstract does not make performance claims beyond the library's capabilities."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that MA versions (centralized critics) outperform independent versions due to conditioning on global state. This is supported by the controlled comparison where the only difference is centralization (e.g., MAPPO vs IPPO, MADDPG vs IDDPG, MASAC vs ISAC), which constitutes a reasonable ablation-like design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The experiments only cover 3 VMAS tasks (Navigation, Sampling, Balance) out of 27 available in VMAS and many more across other environments. The paper does not explicitly bound its conclusions to these 3 tasks. Statements like 'MA versions of all algorithms obtain the best overall performance' could be read as general, though they are presented in the context of the VMAS benchmark."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one explanation for Q-Learning underperformance (discrete actions in continuous domains) but does not consider alternative explanations such as hyperparameter tuning differences, training budget differences, or other algorithmic factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use LLMs or pre-trained language models. It trains RL agents from scratch using neural networks (MLP, GNN, CNN, LSTM, GRU). Model architecture details are in Table 3."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It is a reinforcement learning paper where agents learn through environment interaction, not language model prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The experiment hyperparameters are available in the fine_tuned/vmas folder' of the repository. Default configurations are available in the conf folder. While not printed in the paper itself, they are accessible in the open-source repository."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding (in the LLM sense) is used. The paper describes a MARL training pipeline, not an LLM agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 (Tasks) describes how environments are processed: action masking, custom transforms for visual observations (storing images as integers, casting to float for neural networks), and the agent grouping mechanism. Section 4 specifies that default BenchMARL configurations were used."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper has Introduction, Related Work, BenchMARL (system description), Experiments, and Conclusion. No limitations or threats to validity are discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as hyperparameter sensitivity, limited task diversity (only 3 tasks tested), small number of seeds (3), or potential performance differences across environments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the benchmark results do NOT show. For instance, it does not acknowledge that results on 3 VMAS tasks may not generalize to other environments (SMACv2, MeltingPot, etc.) or to tasks with different characteristics."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'An interactive version of these results is available here' (referring to Wandb), and fine-tuned configurations and results are in the public repository. This allows independent verification of the experimental results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the experimental setup: 9 algorithms on 3 VMAS tasks, using default BenchMARL configurations with fine-tuned hyperparameters from the conf folder. The results collection process (training runs with 3 seeds, evaluation metrics) is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from automated RL experiments in simulation environments."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The execution diagram (Fig. 2) shows the full pipeline from configuration to results. Section 3.2 (Reporting) describes how experiment data is output in JSON format for the marl-eval reporting library, with integration to Wandb and other loggers."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'The majority of the work was done during an internship at PyTorch, Meta. This work was supported in part by European Research Council (ERC) Project 949940 (gAIa) and ARL DCIST CRA W911NF-17-2-0181.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Matteo Bettini (University of Cambridge, with footnote 'Work done during an internship at PyTorch, Meta'), Amanda Prorok (University of Cambridge), Vincent Moens (PyTorch, Meta)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Meta/PyTorch is the developer of TorchRL, which BenchMARL is built on, so there is an affiliation interest. However, the paper primarily benchmarks MARL algorithms (not Meta products vs. competitors), and the ERC and ARL grants are independent academic funders. The funder does not have a direct financial interest in which MARL algorithm performs best."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper. While affiliations are disclosed, there is no explicit declaration of financial interests or conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains RL agents from scratch in simulation environments. There is no pre-trained model with a training data cutoff to consider."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "RL agents interact with simulation environments in real-time; there is no pre-existing training corpus that could overlap with test data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination (in the LLM sense of training data containing test examples) is not applicable to RL agents trained from scratch in procedurally generated simulation environments."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report training time, wall-clock time, or inference latency for the benchmark experiments. Given that the paper promotes the library for practical benchmarking, reporting cost per experiment would be relevant."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget are reported for the experiments. The paper mentions SLURM and HPC cluster support but does not state what hardware was used for the reported results."
       }
     }
   },
   "claims": [
     {
       "claim": "BenchMARL is the first MARL training library created to enable standardized benchmarking across different algorithms, models, and environments.",
       "evidence": "The paper describes the library architecture (Sec. 3) with standardized components, configuration via Hydra, and integrated reporting tools. Comparison with related work (Sec. 2) argues other libraries lack this standardization.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-Agent (MA) versions of all algorithms (MASAC, MADDPG, MAPPO) obtain the best overall performance on VMAS tasks.",
       "evidence": "Fig. 4 shows aggregate scores with 95% confidence intervals. MA versions consistently rank higher in IQM, median, and mean metrics across 3 VMAS tasks with 3 seeds per experiment (Sec. 4).",
       "supported": "moderate"
     },
     {
       "claim": "Q-Learning algorithms (IQL, VDN, QMIX) perform suboptimally compared to actor-critic algorithms, potentially due to action discretization in continuous control tasks.",
       "evidence": "Figs. 4 and 5 show Q-Learning algorithms ranking lower. The explanation that discrete action spaces impede performance on continuous control tasks is plausible but not directly tested (Sec. 4).",
       "supported": "weak"
     },
     {
       "claim": "BenchMARL results match those reported by Bou et al. (2024) and Bettini et al. (2022).",
       "evidence": "Stated in Sec. 4: 'The obtained results match the ones reported by Bou et al. [2024] and Bettini et al. [2022].' No detailed comparison is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BenchMARL provides a standardized MARL benchmarking framework built on TorchRL, supporting 9 algorithms, 5 environments (101 tasks), and 6 model architectures. Initial benchmarks on 3 VMAS tasks with 9 algorithms show that multi-agent versions with centralized critics (MAPPO, MADDPG, MASAC) consistently outperform their independent counterparts, while Q-Learning methods underperform on continuous control tasks. The library integrates statistically rigorous reporting tools from Gorsane et al. (2022) and outputs 95% stratified bootstrap confidence intervals.",
   "red_flags": [
     {
       "flag": "Very limited experimental scope",
       "detail": "Only 3 out of 101 available tasks across 5 environments are benchmarked. The paper supports environments like SMACv2, MPE, SISL, and MeltingPot but provides no experimental results for them, limiting the generalizability of the benchmark findings."
     },
     {
       "flag": "Only 3 random seeds",
       "detail": "All experiments use only 3 random seeds, which is small for reliable statistical inference in RL, where variance across seeds is known to be high. No justification is provided for this choice."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or discussion of what the results do not show. This is a notable omission for a paper about benchmarking methodology and standardization."
     },
     {
       "flag": "No compute cost reported",
       "detail": "For a benchmarking library, the computational cost of running benchmarks is directly relevant to practitioners. No training time, hardware, or GPU hours are reported."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "BenchMARL is built on TorchRL (Meta/PyTorch), and one author is from Meta. The paper advocates for TorchRL as the backend, which could be seen as promoting Meta's ecosystem. However, the algorithmic comparisons themselves appear fair."
     }
   ],
   "cited_papers": [
     {
       "title": "Torchrl: A data-driven decision-making library for pytorch",
       "authors": ["Albert Bou", "Matteo Bettini", "Sebastian Dittert", "Vikash Kumar", "Shagun Sodhani", "Xiaomeng Yang", "Gianni De Fabritiis", "Vincent Moens"],
       "year": 2024,
       "relevance": "Core backend library for BenchMARL; relevant as an example of standardized RL tooling and benchmarking infrastructure."
     },
     {
       "title": "Towards a standardised performance evaluation protocol for cooperative marl",
       "authors": ["Rihab Gorsane", "Omayma Mahjoub", "Ruan John de Kock", "Roland Dubb", "Siddarth Singh", "Arnu Pretorius"],
       "year": 2022,
       "relevance": "Proposes the reporting tools and statistical evaluation protocol that BenchMARL integrates, addressing the MARL reproducibility crisis."
     },
     {
       "title": "Deep reinforcement learning at the edge of the statistical precipice",
       "authors": ["Rishabh Agarwal", "Max Schwarzer", "Pablo Samuel Castro", "Aaron Courville", "Marc G Bellemare"],
       "year": 2021,
       "relevance": "NeurIPS Outstanding Paper proposing statistically rigorous evaluation methods for RL, directly used in BenchMARL's reporting."
     },
     {
       "title": "Benchmarking multi-agent deep reinforcement learning algorithms in cooperative tasks",
       "authors": ["Georgios Papoudakis", "Filippos Christianos", "Lukas Schäfer", "Stefano V. Albrecht"],
       "year": 2021,
       "relevance": "Prior MARL benchmarking effort (EPyMARL) that BenchMARL aims to improve upon and supersede."
     },
     {
       "title": "Marllib: A scalable and efficient multi-agent reinforcement learning library",
       "authors": ["Siyi Hu", "Yifan Zhong", "Minquan Gao", "Weixun Wang", "Hao Dong", "Xiaodan Liang", "Zhihui Li", "Xiaojun Chang", "Yaodong Yang"],
       "year": 2023,
       "relevance": "Competing MARL library built on RLlib; relevant for comparing approaches to standardized MARL benchmarking."
     },
     {
       "title": "Jaxmarl: Multi-agent rl environments and algorithms in jax",
       "authors": ["Alexander Rutherford", "Benjamin Ellis", "Matteo Gallici"],
       "year": 2024,
       "relevance": "Concurrent Jax-based MARL library, representing a complementary ecosystem for MARL benchmarking."
     },
     {
       "title": "The surprising effectiveness of ppo in cooperative, multi-agent games",
       "authors": ["Chao Yu", "Akash Velu", "Eugene Vinitsky", "Yu Wang", "Alexandre Bayen", "Yi Wu"],
       "year": 2021,
       "arxiv_id": "2103.01955",
       "relevance": "Introduces MAPPO, one of the key algorithms benchmarked, and provides evidence for PPO's effectiveness in MARL."
     },
     {
       "title": "Multi-agent actor-critic for mixed cooperative-competitive environments",
       "authors": ["Ryan Lowe", "Yi I Wu", "Aviv Tamar", "Jean Harb", "Pieter Abbeel", "Igor Mordatch"],
       "year": 2017,
       "relevance": "Introduces MADDPG, a foundational multi-agent algorithm benchmarked in BenchMARL."
     },
     {
       "title": "Smacv2: An improved benchmark for cooperative multi-agent reinforcement learning",
       "authors": ["Benjamin Ellis", "Skander Moalla", "Mikayel Samvelyan", "Mingfei Sun", "Anuj Mahajan", "Jakob N. Foerster", "Shimon Whiteson"],
       "year": 2022,
       "arxiv_id": "2212.07489",
       "relevance": "Key MARL benchmark environment supported by BenchMARL, representing discrete-action cooperative scenarios."
     },
     {
       "title": "Scalable evaluation of multi-agent reinforcement learning with melting pot",
       "authors": ["Joel Z. Leibo", "Edgar Duénez Guzmán", "Alexander Sasha Vezhnevets"],
       "year": 2021,
       "relevance": "Large-scale MARL evaluation environment (49 tasks) integrated into BenchMARL, testing cooperative and competitive scenarios."
     }
   ]
 }