ai-research-survey

scan.json (22676B)

---

 {
   "paper": {
     "title": "Scaling Laws for a Multi-Agent Reinforcement Learning Model",
     "authors": ["Oren Neumann", "Claudius Gros"],
     "year": 2022,
     "venue": "International Conference on Learning Representations",
     "arxiv_id": "2210.00849",
     "doi": "10.48550/arXiv.2210.00849"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Player strength in AlphaZero scales as a power law with neural network parameter count (exponent ~0.88) and with compute (exponent ~0.55) on Connect Four and Pentago. The scaling exponents are nearly identical across both games, suggesting possible universality classes. The optimal model size scales as a power law with compute, with an exponent derivable from the individual scaling laws. Larger models are more sample-efficient, outperforming smaller models given the same training data.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and data used in our experiments are available online' with a GitHub link (https://github.com/OrenNeumann/AlphaZero-scaling-laws) in Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all code and data are available online at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is described in the paper. They mention using OpenSpiel but do not specify library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Hyperparameters are listed in Appendix A but there are no explicit instructions for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The plots in Figures 2-5 show data spread across multiple training seeds. Elo scores are shown with visible variance across the 6 seeds per configuration."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests are reported. Claims of scaling laws are based on visual log-linear fits without p-values or goodness-of-fit statistics."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Scaling exponents are reported precisely (Table 1: αN=0.88/0.87, αC=0.55/0.55, αopt_C=0.62/0.63) providing quantitative effect sizes for the scaling relationships."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 6 seeds per configuration were chosen, nor why specific numbers of model sizes were tested. The total pools of 1326 and 714 agents are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each training configuration is repeated 6 times with different random seeds (Section 4), and variance across seeds is visible in the figures. Appendix D shows averaged compute scaling."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Agents are benchmarked against a perfect game solver for Connect Four (Pons, 2015) and a near-optimal benchmark player for Pentago using a 4TB solver dataset (Irving, 2014)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Perfect/near-perfect game solvers are the strongest possible baselines. The paper also compares against prior scaling law results (Jones 2021, Kaplan et al. 2020)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper varies model size and compute but does not ablate components of the AlphaZero algorithm."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Elo scores, playing strength (Bradley-Terry γ), expected game scores, and also examines test-set loss for both value and policy heads (Appendix F)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper; it studies scaling laws of automated game-playing agents measured by Elo ratings."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Elo scores are computed from match outcomes between agents, which is inherently a held-out evaluation (agents play against each other at test time, not on training data). Appendix F uses a separate solver-generated test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by game (Connect Four, Pentago, Oware in appendix), by scaling type (size, compute, optimal), and by training duration (Fig. 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E discusses Oware where large models fail to converge, with analysis of possible causes (integer vs boolean input format). Section E.4 discusses training issues."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Oware's large model failures are reported (Appendix E). Appendix F shows that test-set loss is NOT a good predictor of performance scaling, a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (power-law scaling with parameters, compute, optimal size, sample efficiency) are supported by corresponding results in Sections 5.1-5.5 with figures and numerical exponents."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about scaling relationships. These are supported by controlled experiments varying individual factors (model size with fixed compute, compute with optimal sizing), with 6 seeds per configuration."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to bound claims to Connect Four and Pentago, noting Oware shows different behavior (Appendix E) and that extrapolation to Go/Chess/Shogi is speculative (Section 5.4.1). The title says 'a Multi-Agent RL Model' not 'all MARL'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses why Hex exponents differ (game types, architecture choices). Section 5.4.1 discusses why AlphaGo/AlphaZero fall below the curve (different architectures, game complexity). Section 6 notes hyperparameter tuning could affect exponents."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly relates Elo rating to playing strength via the Bradley-Terry model (Section 3.2), discusses when Elo fails (non-transitive dynamics, Appendix F last paragraph), and notes test loss is a poor proxy for performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The architecture is fully specified: MLP with 2 fully-connected hidden layers as torso, policy and value heads each with 1 hidden layer, width varied 4-256. Uses OpenSpiel's AlphaZero implementation (Lanctot et al., 2019)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use language model prompting; it trains RL agents."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 in Appendix A provides a comprehensive list of all hyperparameters with values and descriptions, including MCTS parameters, learning rate, batch size, replay buffer, temperature, etc."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard AlphaZero training pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is described: self-play with MCTS generates training data, replay buffer management is specified (size 2^16, reuse factor 10). Compute is defined as C = S·T·F·D (Section 5.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion (Section 6) mentions hyperparameter tuning as future work and notes limitations are spread across the text, but no formal limitations section exists."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: hyperparameter tuning could change exponents (Section 6), Oware's training issues with non-boolean inputs (Section E.4), Elo's failure with non-transitive dynamics (Appendix F), and the architectural difference between MLPs and ResNets (Section 5.4.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4.1 explicitly states the scaling law 'might not accurately describe AlphaGo Zero and AlphaZero trained on Go, chess and shogi due to the differences between the games.' Section 6 notes significance is 'evidence of power-law scaling' not the exact exponent values."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all code and data are available at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is self-play: agents generate their own training data through MCTS-guided games. The process is described in Section 4 with all relevant parameters."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is generated through self-play simulations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: self-play generates games → states stored in replay buffer → optimization steps → Elo calculation via BayesElo. Compute definition is explicit (Section 5.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned. The acknowledgments section thanks individuals for discussions but does not mention any funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are from Institute for Theoretical Physics, Goethe University Frankfurt. Affiliations are clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed; appears to be unfunded academic research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains RL agents from scratch via self-play; it does not evaluate a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — agents generate their own training data through self-play and are evaluated via matches against each other and solvers."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model is evaluated on an existing benchmark. Agents are trained from scratch."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B discusses inference-time compute scaling in detail, examining performance under inference compute constraints with varying MCTS steps."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training compute is defined explicitly as C = S·T·F·D (Section 5.3) and measured in FLOPs. Results are plotted as a function of compute budget in Fig. 4."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each configuration is trained 6 times with different random seeds (Section 4), and the variance across seeds is visible in all figures."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'repeating each training six times with different random seeds to ensure reproducibility (Henderson et al., 2018).'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly avoids hyperparameter tuning: 'we specifically avoid hyperparameter tuning whenever possible, fixing most hyperparameters to the ones suggested in OpenSpiel's AlphaZero example code' (Section 4). Only temperature drop is varied."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses default OpenSpiel hyperparameters rather than selecting best configurations. For compute scaling, the Pareto front approach is well-defined. Appendix D confirms scaling holds for averaged (not cherry-picked) results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors train and evaluate their own AlphaZero agents without discussing self-evaluation bias. However, the use of perfect game solvers as external benchmarks partially mitigates this."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is a central result of the paper. Figure 4 shows performance as a function of compute, and the optimal model size vs compute relationship is the main contribution (Fig. 1)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses Elo's construct validity: Section 3.2 derives the relationship between Elo and playing strength, Appendix C tests for measurement bias, and Appendix F's last paragraph acknowledges Elo fails for non-transitive dynamics."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All agents use the same AlphaZero algorithm with MCTS."
       }
     }
   },
   "claims": [
     {
       "claim": "Player strength scales as a power law with neural network parameter count (αN ≈ 0.88 for Connect Four, 0.87 for Pentago)",
       "evidence": "Figure 2 shows log-linear Elo scaling with model size for both games. Table 1 reports exponents. 1326 Connect Four agents and 714 Pentago agents trained with 6 seeds each.",
       "supported": "strong"
     },
     {
       "claim": "Player strength scales as a power law with compute when training optimally sized agents (αC ≈ 0.55 for both games)",
       "evidence": "Figure 4 shows log-linear Pareto front scaling with compute for both games. Table 1 reports identical exponents. Appendix D confirms with averaged (not best-of) results.",
       "supported": "strong"
     },
     {
       "claim": "Optimal model size scales as a power law with compute, with an exponent derivable from the other two scaling laws (αopt_C = αC/αN ≈ 0.62-0.63)",
       "evidence": "Figure 1 shows the predicted and observed scaling. The analytically derived exponent from Eq. 8 matches empirical data for both games.",
       "supported": "strong"
     },
     {
       "claim": "AlphaGo Zero and AlphaZero were substantially undersized relative to their training compute",
       "evidence": "Figure 1 right shows these models fall 'well below the optimal trend.' However, this extrapolates from Connect Four/Pentago MLPs to Go/Chess ResNets, which the authors acknowledge may not apply.",
       "supported": "weak"
     },
     {
       "claim": "Larger AlphaZero models are more sample efficient",
       "evidence": "Figure 5 shows larger models outperform smaller ones given the same training data for both games.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No formal goodness-of-fit statistics",
       "detail": "Scaling law fits are presented visually with log-linear trends but no R² values, residual analysis, or formal statistical tests for the power-law hypothesis vs alternatives."
     },
     {
       "flag": "Extrapolation to state-of-the-art models is weakly supported",
       "detail": "The claim that AlphaGo Zero and AlphaZero were undersized extrapolates from small games with MLPs to large games with ResNets. The authors acknowledge this but still present it prominently in Figure 1."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B Brown", "Benjamin Chess", "Rewon Child", "Scott Gray", "Alec Radford", "Jeffrey Wu", "Dario Amodei"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper for language models that this work extends to MARL."
     },
     {
       "title": "Training Compute-Optimal Large Language Models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla paper recalculating optimal compute allocation; directly compared scaling exponents."
     },
     {
       "title": "Deep Reinforcement Learning that Matters",
       "authors": ["Peter Henderson", "Riashat Islam", "Philip Bachman", "Joelle Pineau", "Doina Precup", "David Meger"],
       "year": 2018,
       "relevance": "Seminal paper on reproducibility in deep RL; motivated the 6-seed experimental design used here."
     },
     {
       "title": "Scaling Scaling Laws with Board Games",
       "authors": ["Andy L Jones"],
       "year": 2021,
       "arxiv_id": "2104.03113",
       "relevance": "Closest prior work showing compute scaling in AlphaZero on Hex; directly compared and extended."
     },
     {
       "title": "A Generalist Agent",
       "authors": ["Scott Reed", "Konrad Zolna", "Emilio Parisotto"],
       "year": 2022,
       "arxiv_id": "2205.06175",
       "relevance": "Gato multi-task agent showing performance scaling with model size across diverse tasks."
     },
     {
       "title": "Multi-Game Decision Transformers",
       "authors": ["Kuang-Huei Lee", "Ofir Nachum", "Mengjiao Yang"],
       "year": 2022,
       "arxiv_id": "2205.15241",
       "relevance": "Applies Transformer architecture to Atari games with model-size scaling analysis."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper demonstrating the practical impact of scaling laws in creating large models."
     },
     {
       "title": "Deep Learning Scaling is Predictable, Empirically",
       "authors": ["Joel Hestness", "Sharan Narang", "Newsha Ardalani"],
       "year": 2017,
       "arxiv_id": "1712.00409",
       "relevance": "Early observation of power-law scaling across ML architectures."
     },
     {
       "title": "Energy and Policy Considerations for Deep Learning in NLP",
       "authors": ["Emma Strubell", "Ananya Ganesh", "Andrew McCallum"],
       "year": 2019,
       "arxiv_id": "1906.02243",
       "relevance": "Discusses environmental and access implications of scaling trends; cited re: small-lab concerns."
     }
   ]
 }