ai-research-survey

scan.json (25126B)

---

 {
   "paper": {
     "title": "Diagnostic Codes in AI prediction models and Label Leakage of Same-admission Clinical Outcomes",
     "authors": [
       "Bashar Ramadan",
       "Ming-Chieh Liu",
       "Michael C. Burkhart",
       "William F. Parker",
       "Brett K. Beaulieu-Jones"
     ],
     "year": 2025,
     "venue": "medRxiv",
     "doi": "10.1101/2025.08.09.25333360"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["observational", "meta-analysis"],
   "key_findings": "ICD diagnostic codes, finalized only after hospital discharge, can predict in-hospital mortality with AUROC 0.97-0.98 when used as the sole features in ML models trained on MIMIC-IV, with top predictors being 'brain death,' 'cardiac arrest,' and 'encounter for palliative care.' A systematic literature review found that 40.2% (37/92) of published MIMIC-based prediction studies use same-admission ICD codes as input features despite both MIMIC publications explicitly warning against this practice. This widespread label leakage renders the resulting models clinically useless for real-time prediction.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Full source code is available on Github (https://github.com/bbj-lab/data-leakage).' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses MIMIC-IV v2.2, 'a publicly available deidentified electronic healthcare record database.' This is a standard public dataset accessible through PhysioNet."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions scikit-learn and XGBoost libraries but provides no requirements.txt, Dockerfile, library versions, or detailed environment specification sufficient to recreate the computational environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released on GitHub, the paper itself contains no step-by-step reproduction instructions, README description, or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "AUROCs are reported as point estimates (0.97-0.98) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper applies the Benjamini-Hochberg procedure to control for false discovery rate with a threshold of p < 0.05 for ICD codes in the logistic regression model."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Odds ratios are reported for ICD codes in the logistic regression model, and AUROCs provide effect size context. The 40.2% prevalence rate in the literature review also provides magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided for either the MIMIC-IV cohort (180,640 patients) or the literature review sample (100 papers, with 50 per MIMIC version chosen arbitrarily)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single experimental runs. No variance, standard deviation, or spread measures across multiple runs are reported for any model."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares ICD-only model performance against published models: 'These results are even better than published models trained on the same data that also included many additional predictive features' (refs 1, 2). Three model types (LR, RF, XGBoost) are also compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Renc et al. (2024) and Rajkomar et al. (2018). The former is contemporary; the latter is a foundational reference in healthcare AI. The comparison purpose is to show ICD-only models match or exceed full-feature models."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system is intentionally minimal by design (ICD codes + age + sex) to demonstrate label leakage, not to optimize a multi-component system. There is essentially one component being tested."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both AUROC and balanced accuracy as evaluation metrics: 'performance assessed using AUROC and balanced accuracy.'"
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to demonstrating label leakage in ML models. The claim is about data leakage mechanics, not output quality requiring human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a temporal train/validation/test split: 'We partitioned the dataset by the date of admission into train (70%), validation (10%), test (20%) sets per TRIPOD-AI+ guidelines,' with explicit exclusion of patients appearing in multiple splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Feature importance breakdowns are provided for all three model types: odds ratios and p-values for logistic regression (Figure 1B), Gini importance for random forest, and gain for XGBoost (Figure 1C). Results are reported per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the anomalous appearance of 'external hemorrhoids without complications' as a predictor: 'This anomaly may reflect the model's ability to detect a clinician's focus on documenting less severe conditions, signaling relative patient stability.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All three models show uniformly high AUROC (0.97-0.98). No configurations that failed, approaches that were tried and abandoned, or experiments showing lower performance are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of AUROC 0.97-0.98, the specific ICD codes as top predictors ('brain death,' 'cardiac arrest,' 'encounter for palliative care,' 'do not resuscitate'), and 40.2% prevalence in the literature are all directly supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that ICD codes 'inflate' performance via label leakage is supported by a transparent mechanism: codes like 'brain death' and 'cardiac arrest' are only assigned after the clinical event occurs, making their use to predict that event circular. The feature importance analysis directly demonstrates this mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Diagnostic Codes in AI prediction models' frames the problem generally, and the Discussion states 'It is very unlikely that this problem is isolated to MIMIC database work but reflects a broader challenge.' However, both analyses are limited to MIMIC and Google Scholar-indexed papers. The framing extends beyond what was empirically tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that some diagnoses may be known early in admission (e.g., broken limbs, chronic conditions carried over from prior visits), that problem list codes might be available during a stay, and offers an alternative explanation for the 'external hemorrhoids' anomaly (clinician documentation patterns signaling patient stability)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It measures AUROC on MIMIC-IV and claims this demonstrates label leakage on MIMIC-IV. The distinction between what ICD codes represent (billing/clinical thinking) versus patient state is explicitly discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names scikit-learn (refs 11) and XGBoost (ref 12) but provides no library version numbers or specific model configurations beyond the model type names."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses traditional ML models (logistic regression, random forest, XGBoost), not prompted language models. No prompting is involved."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'tuning hyperparameters in validation set' but does not report any actual hyperparameter values, search ranges, or final selected configurations for any of the three models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains standard ML classifiers."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Preprocessing steps are documented: ICD-10 to ICD-9 conversion, removal of low-variance (<0.0001) and high-covariance (>0.8) ICD codes, temporal train/validation/test split with patient-level exclusion. The literature review filtering pipeline is also documented in Figure 2 with counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Limitations are discussed within the Discussion section but are not set apart as a distinct subsection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the study is limited to MIMIC only; other datasets with timestamped ICD codes may not have this problem; ICD codes represent billing/clinical thinking rather than patient state; the literature review used only Google Scholar and top-cited papers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'Both analyses in our study are limited because they only the benchmark MIMIC dataset' and 'While it is not possible to estimate how frequently this occurs on private, institutional datasets.' It also notes MIMIC lacks timestamps and problem list codes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "MIMIC-IV v2.2 is publicly available through PhysioNet for credentialed users, enabling independent verification of the prediction model results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: 'MIMIC-IV v2.2, a publicly available deidentified electronic healthcare record database of patients admitted to an ICU or emergency department at Beth Israel Deaconess Medical Center between 2008 and 2019.' The literature review search strategy is also described with specific queries and date."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The prediction study uses a standard benchmark database (MIMIC-IV), and the literature review screens published papers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The prediction pipeline documents inclusion criteria (<1% excluded), preprocessing steps (ICD-10→ICD-9, variance/covariance filtering), and split methodology. The literature review pipeline is documented in Figure 2 with counts at each stage (140→128→122→100→92→37)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or sponsorship are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Center for Computational Medicine and Clinical Artificial Intelligence, Department of Medicine, University of Chicago; MacLean Center for Clinical Medical Ethics, University of Chicago."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains traditional ML models (logistic regression, random forest, XGBoost) from scratch on MIMIC-IV data. No pre-trained model's capability is being evaluated on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. The models are trained from scratch with a temporal split, making pre-training contamination inapplicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. The paper trains standard ML classifiers from scratch."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes an existing database (MIMIC-IV) and reviews published literature."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. MIMIC-IV is a deidentified public dataset that does not require per-study IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Patient demographics are reported for the MIMIC-IV cohort (mean age 58.7, 53% female) as data characteristics, not as study participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. No experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or computational time is reported for any of the three trained models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specification, or training time is reported despite training three ML models on 180,640 patients."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The literature review uses reproducible search queries on Google Scholar (two specific queries stated), a defined sorting criterion (citations per year), a stopping rule (100 papers), and a PRISMA-style flow diagram (Figure 2) with counts at each screening stage."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The review categorizes papers by whether they use ICD codes as features but does not assess the methodological quality of included studies. No quality scoring rubric or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The review sorted by citations per year, which introduces selection bias toward highly cited papers, but this is not discussed as a limitation."
       }
     }
   },
   "claims": [
     {
       "claim": "ML models using only ICD codes, age, and sex can predict in-hospital mortality with AUROC 0.97-0.98 on MIMIC-IV",
       "evidence": "Three models (logistic regression, random forest, XGBoost) evaluated on held-out test set (20% temporal split) from 180,640 patients yielded AUROCs of 0.98, 0.97, and 0.97 respectively (Figure 1A, eTable 1).",
       "supported": "strong"
     },
     {
       "claim": "The most important ICD codes for mortality prediction are post-discharge diagnoses that represent label leakage",
       "evidence": "Feature importance analysis shows 'brain death,' 'cardiac arrest,' 'encounter for palliative care,' and 'do not resuscitate status' as top predictors across all three models (Figure 1B, 1C). These codes document events that occur during or after the clinical outcome.",
       "supported": "strong"
     },
     {
       "claim": "40.2% of published MIMIC-based prediction studies use same-admission ICD codes as input features",
       "evidence": "Systematic review of 92 papers building prediction models for same-admission outcomes found 37 (40.2%) used ICD codes as features (Figure 2, eTable 2). Papers were selected from 139 Google Scholar results sorted by citations per year.",
       "supported": "moderate"
     },
     {
       "claim": "ICD-code based models outperform published models that include additional clinical features",
       "evidence": "The paper states ICD-only models achieve 'even better' results than published models (refs 1, 2) 'trained on the same data that also included many additional predictive features from the rest of the electronic medical record.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification for main results",
       "detail": "AUROC values of 0.97-0.98 are reported as point estimates with no confidence intervals, bootstrap intervals, or cross-validation variance. For a paper about methodological rigor in healthcare AI, this is a notable omission."
     },
     {
       "flag": "Literature review limited to single search engine",
       "detail": "The systematic review used only Google Scholar. No PubMed, Scopus, or Web of Science searches were conducted. Papers were sorted by citations per year and screening stopped at 100 papers, introducing selection bias toward highly visible work and potentially missing papers with ICD leakage issues."
     },
     {
       "flag": "No hyperparameters reported despite claiming tuning",
       "detail": "The paper states hyperparameters were tuned on the validation set but reports none of the actual values, search ranges, or selection criteria for any of the three models."
     }
   ],
   "cited_papers": [
     {
       "title": "Scalable and accurate deep learning for electronic health records",
       "authors": ["Alvin Rajkomar", "Eyal Oren", "Kai Chen"],
       "year": 2018,
       "arxiv_id": "1801.07860",
       "relevance": "Foundational work on deep learning for EHR-based clinical prediction, directly relevant as a baseline that the ICD-only models surpass."
     },
     {
       "title": "Zero shot health trajectory prediction using transformer",
       "authors": ["Pawel Renc", "Yugang Jia", "Anthony E. Samir"],
       "year": 2024,
       "relevance": "Recent transformer-based clinical prediction model using MIMIC, serving as a contemporary baseline for comparison."
     },
     {
       "title": "Machine learning for patient risk stratification: standing on, or looking over, the shoulders of clinicians?",
       "authors": ["Brett K. Beaulieu-Jones", "William Yuan", "Gabriel A. Brat"],
       "year": 2021,
       "relevance": "Discusses whether ML models replicate clinical judgment vs. provide independent predictive value, directly relevant to the label leakage concern."
     },
     {
       "title": "A framework for understanding label leakage in machine learning for health care",
       "authors": ["Sharon E. Davis", "Michael E. Matheny", "Suresh Balu", "Mark P. Sendak"],
       "year": 2023,
       "doi": "10.1093/jamia/ocad178",
       "relevance": "Provides the conceptual framework for label leakage in healthcare ML that this paper extends with empirical quantification."
     },
     {
       "title": "Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: A cross-sectional study",
       "authors": ["John R. Zech", "Marcus A. Badgeley", "Manway Liu"],
       "year": 2018,
       "relevance": "Demonstrates shortcut learning in medical imaging AI, part of the broader shortcut learning literature this paper contributes to."
     },
     {
       "title": "Shortcuts causing bias in radiology artificial intelligence: Causes, evaluation, and mitigation",
       "authors": ["Imon Banerjee", "Kamanasish Bhattacharjee", "John L. Burns"],
       "year": 2023,
       "relevance": "Reviews shortcut learning and bias in medical AI systems, directly relevant to the data leakage problem this paper identifies."
     },
     {
       "title": "Shortcut learning in medical AI hinders generalization: method for estimating AI model generalization without external data",
       "authors": ["Cong Ly Ong", "Balagopal Unnikrishnan", "Tony Tadic"],
       "year": 2024,
       "relevance": "Addresses shortcut learning as a generalization barrier in medical AI, complementary to this paper's focus on ICD code leakage."
     },
     {
       "title": "TRIPOD+AI statement: updated guidance for reporting clinical prediction models that use regression or machine learning methods",
       "authors": ["Gary S. Collins", "Karel Moons", "Paula Dhiman"],
       "year": 2024,
       "doi": "10.1136/bmj-2023-078378",
       "relevance": "Reporting guidelines for AI prediction models that the paper follows for train/validation/test split methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly actionable for anyone building clinical prediction models on EHR data: do not use same-admission ICD codes as features."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that 40% of published MIMIC studies contain this basic methodological flaw challenges confidence in healthcare AI literature quality."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises patient safety concerns: clinically deployed models built on leaked features would produce useless predictions for real-time clinical decision-making."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Effectively calls out a large fraction of the healthcare AI literature as methodologically flawed, with AUROC inflation rendering models clinically useless."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub for replication, but the paper is a methodology critique rather than a usable tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University of Chicago affiliation is respected but not a household name; MIMIC is well-known within healthcare AI but not to general audiences."
     }
   }
 }