ai-research-survey

scan.json (20091B)

---

 {
   "paper": {
     "title": "Leakage and the Reproducibility Crisis in ML-based Science",
     "authors": ["Sayash Kapoor", "Arvind Narayanan"],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2207.07048"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code and data for the civil war prediction case study are uploaded to a CodeOcean capsule (https://doi.org/10.24433/CO.4899453.v1), stated in the Materials and Methods section. The model info sheet template is available at https://reproducible.cs.princeton.edu."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The data required to reproduce the civil war prediction case study is included in the CodeOcean capsule. The survey data (Table 1) is presented in the paper itself."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The CodeOcean capsule replicates the exact computational environment used, which is the purpose of the platform (Clyburne-Sherin et al., 2019)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The CodeOcean capsule provides a reproducible computational environment. Appendix B provides detailed description of methods for the civil war prediction case study."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence intervals for the civil war prediction results, e.g., '[0.66-0.95]' for the smoothed AUC of Blair & Sambanis (2020). Bootstrapped test set resampling is used."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper performs significance tests comparing model performance, reporting Z-statistics and p-values (e.g., 'Z = 0.64, 1.09, 0.42, 0.67; p = 0.26, 0.14, 0.34, 0.25' in footnote 3). Uses smoothed ROC curve comparison test from Robin et al."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports AUC differences between reported and corrected results (e.g., 'Difference in AUC between Adaboost and Logistic Regression drops from 0.14 to 0.01' for Wang 2019). Figure 1 provides visual comparison of effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not justify the sample size of the survey (20 papers across 17 fields) or the case study (12 papers in civil war prediction). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While confidence intervals are reported for some results (Blair & Sambanis), variance or standard deviation across experimental runs is not systematically reported for the corrected civil war prediction results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The civil war prediction case study compares corrected results of complex ML models against Logistic Regression baselines. The paper also compares against the original (incorrect) reported results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are the same models used in the original papers (Random Forests, Adaboost, GBT, Logistic Regression), which is appropriate since the goal is to evaluate the original claims."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey and reproducibility study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The civil war prediction case study uses both AUC and accuracy metrics. The paper also discusses metric choice issues across fields (Section 2.5)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims, which are about data leakage detection and reproducibility verification."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The civil war prediction case study uses the same held-out test sets as the original papers, and the paper emphasizes proper train-test separation as a core concern."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-field breakdown of leakage types across 17 fields. Figure 1 provides per-paper breakdown of reported vs corrected results for 4 civil war prediction papers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses limitations of model info sheets. The paper also discusses cases where errors could not be detected by reading papers alone."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 8 of 12 civil war prediction papers had no errors detected. Section 3.4 explicitly discusses limitations of the proposed solution."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 17 fields, 329 affected papers, taxonomy of 8 leakage types, and civil war prediction findings are all supported by Table 1, Section 2.4, and Section 4/Figure 1 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that data leakage caused irreproducible results. This is justified by correcting specific errors and showing results change (Section 4, Figure 1). The causal mechanism is demonstrated by fixing the leakage and observing performance drops."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the survey results are 'a lower bound of reproducibility issues' (Section 2.1) and that the case study is limited to civil war prediction. It does not overclaim beyond the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses five alternative diagnoses for reproducibility failures beyond leakage (lack of understanding limits to prediction, hype, inadequate expertise, lack of standardization, lack of computational reproducibility). Appendix B discusses methodological choices in the corrected results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use LLMs or API-based models. It uses classical ML models (Random Forests, Logistic Regression, Adaboost) via R packages, which are specified."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report hyperparameters for the ML models used in the civil war prediction case study (e.g., number of trees in Random Forests, learning rate for Adaboost). It states they follow the original papers' implementations but does not detail the settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides detailed documentation of data preprocessing, including the imputation correction methodology (mice package vs rfImpute), the precise mechanism of leakage, and step-by-step corrections for each paper."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 is titled 'Limitations of model info sheets' and discusses three specific limitations. The paper also acknowledges survey limitations in Section 2.1."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses specific threats: claims cannot be verified without computational reproducibility, incorrect claims may provide false assurances, and ML expertise is required. Section 2.1 notes the survey is a lower bound due to inconsistent terminology across fields."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly defines scope: 'We focus on reproducibility issues in ML-based science' and distinguishes from ML methods research, ethics research, engineering applications, and modeling contests. The survey is explicitly presented as 'a lower bound' not comprehensive."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The CodeOcean capsule (https://doi.org/10.24433/CO.4899453.v1) provides code and data for the civil war prediction case study, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes the paper selection process for the civil war prediction review: search terms used, database searched (Dimensions), time period (Jan 2016 - May 2021), yielding 124 papers narrowed to 12. Section 2.1 describes the cross-disciplinary survey methodology."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study analyzes published papers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 documents the pipeline: 124 papers from search → 15 focused on civil war prediction with train-test split → 12 with complete code and data → 4 with identified errors. Each filtering step has counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed in the paper. There is an acknowledgments section thanking individuals for feedback but no mention of grants or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as from 'Department of Computer Science and Center for Information Technology Policy, Princeton University.' No product being evaluated, so no vendor conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed. The absence of funding disclosure is noted."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It uses classical ML models trained from scratch on specific datasets."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — no pre-trained model benchmark evaluation. Train-test overlap in the civil war prediction context is actually the core topic of the paper, but not in the contamination sense."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey and reproducibility study, not a proposed method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey and reproducibility study. Compute requirements are minimal (classical ML models on small datasets)."
       }
     }
   },
   "claims": [
     {
       "claim": "Data leakage is a widespread problem across 17 scientific fields adopting ML methods, collectively affecting 329 papers.",
       "evidence": "Table 1 surveys 20 papers across 17 fields documenting leakage errors. Each field has at least one type of leakage identified (Section 2.1).",
       "supported": "strong"
     },
     {
       "claim": "All 4 civil war prediction papers claiming superior ML model performance over Logistic Regression fail to reproduce due to data leakage.",
       "evidence": "Figure 1 and Section 4 show that correcting leakage in Muchlinski et al. (2016), Colaresi & Mahmood (2017), Wang (2019), and Kaufman et al. (2019) eliminates the claimed advantage. Detailed corrections in Appendix B.",
       "supported": "strong"
     },
     {
       "claim": "Complex ML models don't perform substantively better than decades-old Logistic Regression models for civil war prediction once leakage is corrected.",
       "evidence": "Figure 1 shows corrected AUC/accuracy values are comparable across model types. Wang (2019) AUC difference drops from 0.14 to 0.01 (Section 4).",
       "supported": "strong"
     },
     {
       "claim": "Model info sheets would enable detection of leakage in each civil war prediction paper with errors.",
       "evidence": "Appendix C demonstrates how model info sheets address each type of leakage found. However, this is argued by construction rather than tested empirically.",
       "supported": "moderate"
     },
     {
       "claim": "9 of 12 civil war prediction papers with complete code and data included no significance tests or uncertainty quantification.",
       "evidence": "Table A6 and Section 4, Finding 2. The paper provides specific p-values and confidence intervals to illustrate the problem with Blair & Sambanis (2020).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["meta-analysis", "case-study"],
   "key_findings": "Data leakage is pervasive across 17 scientific fields adopting ML, affecting at least 329 papers. A taxonomy of 8 leakage types is presented, ranging from textbook errors (no test set) to open problems (sampling bias). In civil war prediction, all 4 papers claiming ML superiority over Logistic Regression fail to reproduce once leakage is corrected. The authors propose 'model info sheets' as a reporting standard to detect and prevent leakage.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Dos and Don'ts of Machine Learning in Computer Security",
       "authors": ["D. Arp", "E. Quiring", "F. Pendlebury", "A. Warnecke", "F. Pierazzi", "C. Wressnegger", "L. Cavallaro", "K. Rieck"],
       "year": 2022,
       "relevance": "Comprehensive study of ML pitfalls in computer security, documenting leakage and reproducibility failures across 30 papers."
     },
     {
       "title": "Common pitfalls and recommendations for using machine learning to detect and prognosticate for COVID-19 using chest radiographs and CT scans",
       "authors": ["M. Roberts", "D. Driggs", "M. Thorpe"],
       "year": 2021,
       "relevance": "Found all 62 reviewed COVID-19 ML papers had methodological pitfalls, directly relevant to ML evaluation quality."
     },
     {
       "title": "Reproducibility in machine learning for health research: Still a ways to go",
       "authors": ["M. B. A. McDermott", "S. Wang", "N. Marinsek", "R. Ranganath", "L. Foschini", "M. Ghassemi"],
       "year": 2021,
       "relevance": "Documents reproducibility failures in ML-based health research, parallel to our survey's scope on ML methodology quality."
     },
     {
       "title": "Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)",
       "authors": ["J. Pineau", "P. Vincent-Lamarre", "K. Sinha"],
       "year": 2020,
       "relevance": "Reports on NeurIPS reproducibility program including checklists for ML research, directly relevant to methodology quality assessment."
     },
     {
       "title": "Model Cards for Model Reporting",
       "authors": ["M. Mitchell", "S. Wu", "A. Zaldivar"],
       "year": 2019,
       "relevance": "Introduced model cards for transparent ML model reporting, foundational work for the model info sheets proposed in this paper."
     },
     {
       "title": "The worst of both worlds: A comparative analysis of errors in learning from data in psychology and machine learning",
       "authors": ["J. Hullman", "S. Kapoor", "P. Nanayakkara", "A. Gelman", "A. Narayanan"],
       "year": 2022,
       "arxiv_id": "2203.06498",
       "relevance": "Cross-disciplinary analysis of statistical and ML methodology errors, directly relevant to methodology quality in ML-based research."
     },
     {
       "title": "Be careful of when: an empirical study on time-related misuse of issue tracking data",
       "authors": ["F. Tu", "J. Zhu", "Q. Zheng", "M. Zhou"],
       "year": 2018,
       "relevance": "Documents temporal leakage issues in software engineering ML applications, relevant to SE methodology quality."
     },
     {
       "title": "Shortcut learning in deep neural networks",
       "authors": ["R. Geirhos", "J.-H. Jacobsen", "C. Michaelis", "R. Zemel", "W. Brendel", "M. Bethge", "F. A. Wichmann"],
       "year": 2020,
       "relevance": "Identifies shortcut learning as a fundamental problem in ML evaluation, closely related to data leakage and benchmark validity."
     },
     {
       "title": "Measuring the predictability of life outcomes with a scientific mass collaboration",
       "authors": ["M. J. Salganik"],
       "year": 2020,
       "relevance": "Demonstrates limits of ML prediction for social outcomes, supporting the paper's argument about overoptimism in ML-based science."
     }
   ]
 }