ai-research-survey

scan.json (19986B)

---

 {
   "paper": {
     "title": "Deep Reinforcement Learning that Matters",
     "authors": ["Peter Henderson", "Riashat Islam", "Philip Bachman", "Joelle Pineau", "Doina Precup", "David Meger"],
     "year": 2018,
     "venue": "AAAI 2018",
     "arxiv_id": "1709.06560"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code link provided: https://git.io/vFHnf (footnote 1). They also reference using publicly available OpenAI Baselines implementations."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All experiments use publicly available OpenAI Gym MuJoCo environments (Hopper-v1, HalfCheetah-v1, Walker2d-v1, Swimmer-v1). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters for all algorithms (DDPG, PPO, TRPO, ACKTR) are provided in the supplemental material, including network architectures, learning rates, batch sizes, discount factors, and all relevant settings. Specific codebases and their versions are referenced."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Code is released with a link, default hyperparameters are fully enumerated in the supplemental material, and experimental setup is described in detail including which codebases to use and which hyperparameters to vary."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap 95% confidence intervals are reported in Table 3. All learning curves show mean and standard error across random seeds. Tables 1 and 2 report mean ± standard error."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "2-sample t-tests, Kolmogorov-Smirnov tests, and bootstrap percent differences with 95% confidence intervals are reported in Tables 9-12 in the supplemental. The paper explicitly advocates for significance testing in RL."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap percent differences with confidence bounds are reported (e.g., '44.47% (-80.62%, 111.72%)' for ACKTR vs DDPG on Walker2d). These provide effect size context beyond just p-values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses bootstrap power analysis to determine if sample sizes are sufficient (Section on Power Analysis). They acknowledge that 5 trials may be insufficient in some settings and demonstrate this empirically."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard error is reported across 5 random seeds for all experiments. Tables 1 and 2 show mean ± standard error. All graphs show mean and standard error bands."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares four policy gradient algorithms (TRPO, DDPG, PPO, ACKTR) against each other across multiple environments, codebases, and hyperparameter settings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All four algorithms (TRPO 2015, DDPG 2015, PPO 2017, ACKTR 2017) were contemporary state-of-the-art policy gradient methods at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual hyperparameters (network architecture, activation functions, reward scaling, batch size) one at a time while holding others constant, which functions as ablation of these design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation metrics are used and discussed: average return, maximum return, bootstrap confidence intervals, t-test statistics, KS-test statistics, and bootstrap percent differences."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not applicable to a study comparing RL algorithm reproducibility on simulated continuous control tasks."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a supervised learning study. RL evaluation is done via policy rollouts in the environment, which is the standard evaluation paradigm."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per algorithm (DDPG, TRPO, PPO, ACKTR), per environment (HalfCheetah, Hopper, Walker2d, Swimmer), per codebase, and per hyperparameter configuration."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are extensively discussed: DDPG failing in unstable environments like Hopper (getting stuck at local optima), Swimmer local optima where the agent curls up rather than swimming, PPO with large networks failing completely."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is fundamentally about negative results: showing that reported improvements may not be meaningful, that random seeds can create statistically different distributions, and that codebase differences lead to dramatically different performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about variability in reported metrics, difficulty of reproduction, and the need for significance testing are all supported by the extensive experimental results across hyperparameters, seeds, environments, and codebases."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made about hyperparameters, seeds, and codebases affecting performance. These are justified through controlled single-variable experiments where one factor is varied while others are held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits scope to policy gradient methods in continuous control MuJoCo domains: 'For clarity, we focus our investigation on policy gradient (PG) methods in continuous control.' The discussion acknowledges that findings may not extend to other RL settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple explanations for observed phenomena, e.g., explaining why reward scaling affects DDPG through gradient saturation theory, why large networks may need different hyperparameters (KL divergence 33.52x higher), and why DDPG fails in unstable environments."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use LLM APIs or pre-trained language models. The RL algorithms are trained from scratch with specified architectures."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a reinforcement learning study, not an LLM study."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Exhaustive hyperparameters are reported in the supplemental material for all four algorithms: network architectures, activation functions, learning rates, batch sizes, discount factors, noise parameters, optimizer settings, etc."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study evaluates standard RL algorithms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Preprocessing is documented: normalized observations with running mean filter for all algorithms. Modifications to baseline implementations are described (e.g., changing DDPG evaluation to use 10 full trajectories)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion and Conclusion section explicitly discusses limitations and open problems, including that the significance tests used 'may not be the best fit for comparing RL algorithms' and that 'further work is needed in significance testing and statistical analysis.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the assumption that corrected significance metrics from supervised learning apply to RL may not hold; 5 trials may be insufficient; the focus on continuous control PG methods may not generalize to discrete settings or value-based methods."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'we focus our investigation on policy gradient (PG) methods in continuous control' using MuJoCo environments. The paper notes that DQN in Atari domains behaves differently and results may not transfer."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://git.io/vFHnf which would allow regeneration of raw experimental data. All experiments use publicly available environments."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is fully described: 5 trials per experiment with different preset random seeds, 2M training timesteps, evaluation via average returns across last 100 trajectories. Bootstrap uses 10k iterations with pivotal method."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study using simulated environments."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from training to evaluation is documented: train for 2M timesteps, evaluate using final average returns across last 100 trajectories, compute bootstrap statistics with 10k iterations, run significance tests using scipy and Facebook Bootstrapped."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section: 'We thank NSERC, CIFAR, the Open Philanthropy Project, and the AWS Cloud Credits for Research Program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: McGill University and Microsoft Maluuba. The paper does not evaluate Microsoft products, so no conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSERC, CIFAR, Open Philanthropy Project, AWS) have no financial interest in the outcome of RL reproducibility research. The paper evaluates open-source algorithms, not funder products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained models are evaluated on benchmarks. All RL agents are trained from scratch."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. RL agents are trained and evaluated in the same environment by design; there is no benchmark contamination concern for RL policy learning."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study trains RL agents from scratch in simulated environments, not evaluating pre-trained models on static benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, wall-clock time, or computational cost per experiment is reported despite running hundreds of experiments across multiple algorithms, environments, and hyperparameter configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or hardware specifications are reported. The acknowledgements mention AWS Cloud Credits but no quantification of compute used."
       }
     }
   },
   "claims": [
     {
       "claim": "Hyperparameter choices (network architecture, activation functions, reward scaling) can significantly affect baseline algorithm performance in deep RL.",
       "evidence": "Tables 1 and 2 show final evaluation performance varying dramatically across architectural choices. For PPO on HalfCheetah, (400,300) network achieves -1180 while (64,64) achieves 2201. DDPG reward scaling experiments (Figure 3) show failure to learn below σ=0.01.",
       "supported": "strong"
     },
     {
       "claim": "Random seeds alone can produce statistically different performance distributions with identical hyperparameters.",
       "evidence": "Figure 5 shows two groups of 5 seeds producing significantly different TRPO learning curves on HalfCheetah (t=-9.09, p=0.0016). Supplemental Figures 24-25 show similar results for DDPG.",
       "supported": "strong"
     },
     {
       "claim": "Different codebases implementing the same algorithm can produce dramatically different results.",
       "evidence": "Figures 6, 34, 35 and Tables 1-2 show TRPO and DDPG performance varying substantially across three codebases (OpenAI Baselines, rllab, original) using identical hyperparameters.",
       "supported": "strong"
     },
     {
       "claim": "No single algorithm consistently outperforms others across all benchmark environments.",
       "evidence": "Figure 4 and Tables 3, 9-12 show DDPG excels on HalfCheetah but fails on Hopper, while TRPO dominates Swimmer but is weak on HalfCheetah. Significance tests confirm many pairwise comparisons are not statistically significant.",
       "supported": "strong"
     },
     {
       "claim": "Bootstrap confidence intervals and significance testing can help assess whether reported improvements are meaningful.",
       "evidence": "Tables 3, 9-12 demonstrate bootstrap CIs and significance tests. For example, ACKTR vs DDPG on Walker2d shows overlapping CIs (t=1.03, p=0.334) despite apparent performance difference, while TRPO vs DDPG on HalfCheetah is significant (t=-4.59, p=0.002).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Reproducibility in deep RL is severely affected by hyperparameter choices, random seed selection, choice of evaluation environment, and implementation codebase. The paper demonstrates that random seeds alone can create statistically different distributions, that different codebases of the same algorithm yield dramatically different results, and that no single algorithm dominates across environments. The authors advocate for bootstrap confidence intervals, significance testing, and power analysis as standard practice in RL evaluation.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Benchmarking deep reinforcement learning for continuous control",
       "authors": ["Yan Duan", "Xi Chen", "Rein Houthooft", "John Schulman", "Pieter Abbeel"],
       "year": 2016,
       "relevance": "Foundational RL benchmarking paper providing baseline implementations used in this study; directly relevant to evaluation methodology."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "Key RL algorithm evaluated in the study; demonstrates how implementation details affect reproducibility."
     },
     {
       "title": "Trust region policy optimization",
       "authors": ["John Schulman", "Sergey Levine", "Pieter Abbeel", "Michael Jordan", "Philipp Moritz"],
       "year": 2015,
       "relevance": "Core algorithm evaluated across multiple codebases showing dramatic performance differences."
     },
     {
       "title": "Evaluating the replicability of significance tests for comparing learning algorithms",
       "authors": ["Remco R. Bouckaert", "Eibe Frank"],
       "year": 2004,
       "relevance": "Methodological foundation for significance testing in ML that the paper adapts for RL evaluation."
     },
     {
       "title": "Machine learning that matters",
       "authors": ["Kiri Wagstaff"],
       "year": 2012,
       "arxiv_id": "1206.4656",
       "relevance": "Influential position paper on ML evaluation practices and real-world applicability that inspired the RL reproducibility investigation."
     },
     {
       "title": "Revisiting the arcade learning environment: Evaluation protocols and open problems for general agents",
       "authors": ["Marlos C. Machado", "Marc G. Bellemare", "Erik Talvitie", "Joel Veness", "Matthew Hausknecht", "Michael Bowling"],
       "year": 2017,
       "arxiv_id": "1709.06009",
       "relevance": "Proposes better evaluation methods for RL benchmarks, complementary to this paper's reproducibility focus."
     },
     {
       "title": "Inference for the generalization error",
       "authors": ["Claude Nadeau", "Yoshua Bengio"],
       "year": 2000,
       "relevance": "Statistical methodology for comparing ML algorithms that informs the significance testing approach proposed here."
     }
   ]
 }