ai-research-survey

scan.json (20261B)

---

 {
   "paper": {
     "title": "Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)",
     "authors": ["Joelle Pineau", "Philippe Vincent-Lamarre", "Koustuv Sinha", "Vincent Larivière", "Alina Beygelzimer", "Florence d'Alché-Buc", "Emily Fox", "Hugo Larochelle"],
     "year": 2021,
     "venue": "Journal of Machine Learning Research",
     "arxiv_id": "2003.12206"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided for the analysis scripts used to produce the figures and statistics in the paper. Ironic given the subject matter."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The underlying data (checklist responses, reviewer scores, code submission status) from NeurIPS 2019 is not released. Only aggregated figures and statistics are shown."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided for the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The reader cannot recreate the analysis from the paper alone."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as percentages and counts without confidence intervals or error bars. For example, '74.4% of papers provided code' with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "A significance test is reported for the association between code availability and reviewer scores: 'the availability of code at submission was positively associated with the reviewer score (p < 1e-08)' (Section 3)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The p-value for code-score association is reported without effect size. No Cohen's d, odds ratio, or magnitude of the score difference is given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (6743 submissions, 173 papers claimed for reproduction) but no justification or power analysis is discussed. The samples are convenience samples from the conference."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported for any of the statistics presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares code submission rates across NeurIPS 2018, ICML 2019, and NeurIPS 2019, providing temporal baselines. Table 2 compares reproducibility challenge participation across ICLR 2018, ICLR 2019, and NeurIPS 2019."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparisons use recent conferences (2018-2019) which are the most relevant contemporary venues."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive report on a conference program, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: code submission rates, acceptance rates, checklist response distributions, reviewer usefulness ratings, reproducibility challenge participation counts."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This paper reports on a conference program; human evaluation of system outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No ML model is being trained or evaluated; this is a descriptive observational study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Breakdowns are provided by affiliation (academia vs. industry) in Figure 2, by checklist question in Figure 4, and by reviewer usefulness assessment in Figure 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses limitations including that only 34% of reviewers found the checklist useful, and acknowledges they 'do not have concluding evidence that these processes indeed have an impact on the quality of the work.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 36% of authors judged error bars not applicable to their results (a concerning finding), and that only 34% of reviewers found the checklist useful. The Discussion acknowledges many open questions about actual effectiveness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are descriptive ('we describe each of these components, how it was deployed, as well as what we were able to learn') and the paper delivers on this description."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal relationships (e.g., code submission policy leading to increased code availability, checklist usefulness leading to higher scores) but the study design is observational with many confounds. Section 5 notes 'it is too early to rule out potential covariates' but still presents suggestive findings without adequate causal identification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that findings are from NeurIPS 2019 specifically, and the Discussion carefully notes that extending to other venues would require further investigation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly notes potential covariates for the checklist-acceptance association ('paper's topic, reviewer expectations, etc.') and discusses the confound of the NA-acceptance correlation disappearing when filtering for reviewers who found the checklist useful."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No ML models are used in this study; it is a report on conference processes."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting of language models is involved."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No ML models are trained or evaluated in this study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the raw CMT/OpenReview data was processed into the statistics and figures presented. No filtering criteria, data cleaning steps, or exclusion rules are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) serves as a substantive limitations section, listing open questions and explicitly stating 'we do not have concluding evidence that these processes indeed have an impact on the quality of the work.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion raises specific threats: potential covariates in the checklist-acceptance association, that increased submissions may not indicate improved quality, and that cultural/organizational change is needed beyond checklists."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion lists specific open questions the study does NOT answer: long-term value of code submitted, effect of incentives on challenge participation, accuracy of self-reported checklist answers, measurable effect on paper quality."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (checklist responses, reviewer scores, submission metadata) is not released. Only aggregated figures and summary statistics are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: checklist responses were collected via the CMT platform at submission and camera-ready stages, reviewer survey responses were collected at the end of the review period, and reproducibility challenge data came from OpenReview."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how reproducibility challenge participants were recruited: through graduate ML courses at universities, with high participation from McGill, KTH, Brown, and IIT Roorkee. For authors and reviewers, these are naturally the NeurIPS 2019 submitters and committee."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how raw CMT/OpenReview data was transformed into the figures and statistics. Intermediary steps between data collection and final analysis are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of grants is present. The Acknowledgments section thanks people and organizations for support but does not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed, including dual affiliations with industry (Facebook AI Research, Google, Yahoo! Research, Apple)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Several authors are affiliated with companies (Facebook, Google, Apple, Yahoo) that have stakes in ML research practices, but this potential conflict is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark. This is an observational study of conference processes."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The reproducibility program involved surveys of authors and reviewers (human participants), but no pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The study analyzed responses from thousands of authors and reviewers but no IRB or ethics approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Participant demographics are partially reported: affiliation type (academia vs. industry) in Figure 2, geographic distribution of reproducibility challenge participants in Figure 8, and institutional breakdown in Section 4."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria are stated for which submissions or reviewer responses were included in the analysis."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of a conference process, not a randomized experiment. Assignment to conditions is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports that 173 papers were claimed for reproduction and 84 reports were reviewed, showing attrition. Figure 3(b) shows the proportion of self-reported code submissions confirmed by reviewers."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a report on conference processes, not a method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a report on conference processes, not a computationally intensive method."
       }
     }
   },
   "claims": [
     {
       "claim": "Code submission rates increased from less than 50% (NeurIPS 2018) to 74.4% at camera-ready (NeurIPS 2019) with a voluntary code submission policy.",
       "evidence": "Table 1 shows NeurIPS 2018 had <50% code at submission, while NeurIPS 2019 had 40% at submission and 74.4% at camera-ready. Section 3.",
       "supported": "moderate"
     },
     {
       "claim": "Code availability at submission was positively associated with higher reviewer scores (p < 1e-08).",
       "evidence": "Section 3 states this p-value directly, though no effect size is provided.",
       "supported": "moderate"
     },
     {
       "claim": "One-third of reviewers found the ML reproducibility checklist useful for evaluating submissions.",
       "evidence": "Section 5 reports that 34% of reviewers responded 'Yes' when asked if checklist answers were useful.",
       "supported": "strong"
     },
     {
       "claim": "Participation in the reproducibility challenge increased 92% from ICLR 2019 to NeurIPS 2019.",
       "evidence": "Table 2 shows 90 papers claimed at ICLR 2019 vs. 173 at NeurIPS 2019. Section 4.",
       "supported": "strong"
     },
     {
       "claim": "Reviewers who found the checklist useful gave higher scores to papers.",
       "evidence": "Figure 6(a) shows this relationship. However, the direction of causality is unclear — it could be that better papers had better checklist responses.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["observational", "case-study"],
   "key_findings": "The NeurIPS 2019 reproducibility program deployed three mechanisms: a voluntary code submission policy (achieving 74.4% compliance at camera-ready), a reproducibility challenge (173 papers claimed, 84 reports reviewed), and an ML reproducibility checklist. Code availability was associated with higher reviewer scores (p < 1e-08), and 34% of reviewers found the checklist useful. The paper honestly acknowledges it cannot establish that these mechanisms improved actual research quality, and identifies several open questions for future investigation.",
   "red_flags": [
     {
       "flag": "No raw data released",
       "detail": "A paper about reproducibility does not release its own underlying data (checklist responses, reviewer scores, submission metadata), making its own results not independently verifiable."
     },
     {
       "flag": "No analysis code released",
       "detail": "The analysis scripts used to generate figures and statistics are not provided, despite the paper advocating for code release."
     },
     {
       "flag": "Causal language with observational data",
       "detail": "Associations between code submission/checklist use and acceptance rates are presented suggestively, though the paper does note covariates cannot be ruled out."
     },
     {
       "flag": "Self-reported data without validation",
       "detail": "Checklist responses are self-reported by authors. Figure 3(b) shows reviewer confirmation rates but notes discrepancies, and the paper itself asks 'What is the accuracy of the ML checklist answers?' as an open question."
     }
   ],
   "cited_papers": [
     {
       "title": "Deep reinforcement learning that matters",
       "authors": ["Peter Henderson", "Riashat Islam", "Philip Bachman", "Joelle Pineau", "Doina Precup", "David Meger"],
       "year": 2018,
       "relevance": "Foundational paper on reproducibility failures in deep RL, motivating the need for reproducibility standards."
     },
     {
       "title": "A step toward quantifying independently reproducible machine learning research",
       "authors": ["Edward Raff"],
       "year": 2019,
       "relevance": "Empirical study quantifying ML reproducibility rates, directly relevant to reproducibility methodology assessment."
     },
     {
       "title": "Unreproducible research is reproducible",
       "authors": ["Xavier Bouthillier", "César Laurent", "Pascal Vincent"],
       "year": 2019,
       "relevance": "Studies variance in ML results due to implementation choices, relevant to understanding reproducibility challenges."
     },
     {
       "title": "Are GANs created equal? A large-scale study",
       "authors": ["Mario Lucic", "Karol Kurach", "Marcin Michalski", "Sylvain Gelly", "Olivier Bousquet"],
       "year": 2018,
       "relevance": "Large-scale benchmark evaluation study examining whether reported GAN improvements are real, relevant to evaluation methodology."
     },
     {
       "title": "State of the art: Reproducibility in artificial intelligence",
       "authors": ["Odd Erik Gundersen", "Sigbjørn Kjensmo"],
       "year": 2018,
       "relevance": "Survey of reproducibility practices in AI research, directly relevant to methodology quality assessment."
     },
     {
       "title": "On reproducible AI: Towards reproducible research, open science and digital scholarship in AI publications",
       "authors": ["Odd Erik Gundersen", "Yolanda Gil", "David W Aha"],
       "year": 2018,
       "relevance": "Develops a checklist for AI publications covering data, code, methods, and experiments — directly comparable to this survey's scope."
     },
     {
       "title": "A manifesto for reproducible science",
       "authors": ["Marcus R Munafò", "Brian A Nosek"],
       "year": 2017,
       "relevance": "Influential position paper on reproducibility across sciences, provides framework for understanding methodological quality."
     },
     {
       "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding",
       "authors": ["Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova"],
       "year": 2018,
       "arxiv_id": "1810.04805",
       "relevance": "Referenced as an example of large-scale ML work that was successfully reproduced despite high compute requirements."
     }
   ]
 }