ai-research-survey

scan.json (25908B)

---

 {
   "paper": {
     "title": "SurvivEHR: a competing risks, time-to-event foundation model for multiple long-term conditions from primary care electronic health records",
     "authors": [
       "Charles Gadd",
       "Krishna Gokhale",
       "Aditya Acharya",
       "Jennifer Cooper",
       "Francesca Crowe",
       "Leah Fitzsimmons",
       "Thomas Jackson",
       "Krishnarajah Nirantharakumar",
       "Christopher Yau"
     ],
     "year": 2025,
     "venue": "medRxiv",
     "doi": "10.1101/2025.08.04.25332916"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SurvivEHR, a GPT-based foundation model trained on 7.6 billion coded events from 23 million UK primary care patients, achieves strong next-event risk stratification (IEC 0.994) and outperforms baseline survival models (RSF, DeepHit, DeSurv) on clinical prediction tasks when fine-tuned. The model captures known clinical associations (e.g., T2DM→metformin, depression→anxiety) and shows particular benefit in low-resource fine-tuning settings (<100K patients). Zero-shot performance is limited, and multi-step forecasting degrades after 4 steps.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code released at http://github.com/cwlgadd/FastEHR and http://github.com/cwlgadd/SurvivEHR under open-source licence, as stated in the Code release policy section."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Raw CPRD data is not publicly available. 'Raw data from the study are not publicly available. Data for the study were obtained under licence from CPRD.' Codelists are released on GitHub but the actual patient data is restricted."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or specific library versions in the paper text. The FastEHR toolkit is described but environment setup details are not provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We provide detailed instructions to allow others to retrain the model from scratch on appropriately licensed data' but these instructions are in the code repositories, not in the paper itself. The paper lacks step-by-step reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports 95% confidence intervals over 5 random seeds for all fine-tuning experiments (e.g., 'SFT 0.816 ± 0.003')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite claiming SurvivEHR-FFT 'achieves superior predictive performance,' no statistical significance tests are reported. Comparisons rely on point estimates with confidence intervals but no formal tests (e.g., paired t-test, Wilcoxon)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results in Table 2 provide absolute metric values with baselines for comparison, giving sufficient context to assess effect magnitude (e.g., Ctd of 0.824 vs 0.772 for DeSurv on hypertension task)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the cohort sizes used. The 572,096 T2DM cohort and 20,000 multimorbidity cohort sizes are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports results as 'average and 95% confidence interval over 5 random seeds' for all fine-tuning and baseline models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: Random Survival Forests (RSF), DeepHit, DeSurv, a prevalence-based baseline, and a cross-entropy pre-training alternative. Also scratch fine-tuning (SFT) vs full fine-tuning (FFT) comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "DeSurv (2022), DeepHit (2018), and RSF are established survival analysis baselines. The paper also explains in Table 1 why alternative longitudinal models couldn't be compared (lack of competing risk capability)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates key design choices: competing risk vs single risk, pre-trained (FFT) vs scratch (SFT) to isolate pre-training benefit, and cohort size ablation (Figure 6C) showing performance vs data size."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics reported: time-dependent concordance (Ctd), Integrated Brier Score (IBS), and Integrated Negative Binomial Log-Likelihood (INBLL)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No clinical expert evaluation of model predictions is included. Evaluation is entirely automated via survival metrics. Given the clinical application domain, expert evaluation of predicted trajectories would be valuable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "90-5-5 site-level split: 23.6M training, 1.4M validation, 1.5M test patients, 'dividing patients by general practices in England to avoid data leakage across practice.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "IEC scores broken down per event type (diagnoses, investigations, medications) in Figure 3A and Supplementary Figures. Fine-tuning evaluated across three distinct clinical tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multi-step forecasting degradation discussed (Figure 3B): 'after four steps it is no longer able to outperform the prevalence-based baseline.' Zero-shot failure noted. Non-causal predictions discussed (NSAID after osteoporosis, SSRIs after visual impairment)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot performance failure is explicitly reported: 'zero-shot predictive performance is not attained by SurvivEHR.' Multi-step degradation and learnt positional embeddings showing 'no benefit for the increased computational cost' also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'strong risk stratification performance' (IEC 0.994), 'outperforms benchmark survival models' (Table 2), 'transfers effectively to fine-tuned prognostic tasks, particularly in low-resource settings' (Figure 6C) are all supported by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about causal claims, explicitly stating 'this experiment should not be considered causal prediction since SurvivEHR only learns associative and not causal relationships.' Ablation claims (SFT vs FFT) are justified by controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds generalization to UK primary care data: 'our specific instance of SurvivEHR is trained only on a UK population and its most prevalent MLTCs.' The Discussion acknowledges data is from one health system."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that improved performance could be from architecture alone (addressed via SFT vs FFT comparison), that predictions are associative not causal, and that the model is subject to biases in training data."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between what is measured (IEC, Ctd, IBS, INBLL on specific clinical tasks) and the broader goal (clinical risk prediction for MLTCs). Claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "This is a custom model architecture fully described in the Methods. The 384-dimensional latent space, transformer architecture details, and DeSurv survival head are specified. Hyperparameters in Supplementary Table S7."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. SurvivEHR is trained from structured EHR data, not via language model prompting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "'Model hyperparameters used for training are given in Supplementary Table S7.' Context window of 512 for fine-tuning is also stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard model training and evaluation pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Extensive preprocessing documentation: DExtER extraction, FastEHR pipeline, code lists from DM+D, inclusion criteria (English patients with 12 months acceptable data before Jan 2005), 90-5-5 site-level split. Post-processing counts provided (51M diagnoses, 4B medications, etc.)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section contains substantial limitations discussion covering: zero-shot failure, multi-step degradation, associative-only learning, training data biases, lack of free text data, privacy limitations on model sharing."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: model cannot produce causal predictions, subject to biases in training data, free text omission from CPRD creates known biases (ref [40]), generative capability could recapitulate real records limiting model sharing, limited to UK population."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states: trained only on UK population, 74 specific long-term conditions, no free text data, no causal predictions. States zero-shot is not achieved. Explicitly notes the model is 'trained only on a UK population and its most prevalent MLTCs.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw CPRD data is not publicly available: 'Raw data from the study are not publicly available.' Access requires Research Data Governance approval from CPRD."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of CPRD data source, DExtER extraction, code lists for 74 conditions, 81 medication classes, 108 test types. Inclusion criteria, time period, and data quality measures described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. The data comes from a routine primary care database (CPRD) covering participating GP practices. This is an observational study using existing administrative data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: CPRD → DExtER extraction → FastEHR preprocessing (tokenisation, cleaning, outlier removal, deduplication) → 90-5-5 site-level split. Final counts provided at each stage: 26.5M patients → 23.6M/1.4M/1.5M split."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed: NIHR Artificial Intelligence for Multiple Long-Term Conditions (Award ID: NIHR202632), UK EPSRC (EP/Y018192/1), UKRI Turing AI Acceleration Fellowship (EP/V023233/1)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations listed: University of Oxford, University of Birmingham, King's College London, Health Data Research UK."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding from NIHR, EPSRC, and UKRI — public research funders with no financial stake in the model's performance. 'The views expressed are those of the author(s) and not necessarily those of the NIHR or the Department of Health and Social Care.'"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "'The authors declare no competing interests' — explicit competing interests statement provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains its own model on CPRD data with controlled site-level train/test splits. It does not evaluate a pre-trained LLM on public benchmarks. Contamination in the LLM-benchmark sense is not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable in the LLM benchmark contamination sense. The paper controls its own train/test split at the practice site level specifically to prevent leakage."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate against pre-existing public benchmarks that could be in an LLM's training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a model development study using existing administrative health records from CPRD."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. CPRD data access was approved under study reference ID 21_000683, which involves data governance rather than human subjects ethics review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Patient demographics of the CPRD cohort are reported (Figure 2C) but this is descriptive statistics of the training data, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. Data inclusion criteria for CPRD records are documented but this is data filtering, not human subject recruitment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a model development study, not a human subjects experiment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable to a model training study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or computational cost per prediction is reported despite the model being proposed for clinical use."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training compute budget stated — no GPU hours, training time, or hardware specifications mentioned for the pre-training or fine-tuning experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports results 'over 5 random seeds' with confidence intervals for all fine-tuning experiments."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'We report the average and 95% confidence interval over 5 random seeds' is explicitly stated for Table 2."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Hyperparameters are listed in Supplementary Table S7 but how they were selected is not described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how hyperparameters or the best configuration were selected. A validation set exists but the selection procedure is not documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied. The paper compares across multiple tasks and models without formal testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their own system and all baselines without acknowledging the bias of evaluating their own system. RSF, DeepHit, and DeSurv implementations are not stated to be official or independently validated."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SurvivEHR uses full patient histories (longitudinal) while baselines use cross-sectional input, and is pre-trained on 23M patients. The compute advantage is not discussed or controlled for in comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses construct validity of its evaluation: proposes the new IEC metric with justification, explains why standard concordance metrics are impractical for sparse EHR events, and discusses the gap between pre-training and evaluation objectives."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. This is a traditional ML model comparison."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The site-level split prevents temporal leakage. Fine-tuning experiments use index dates with 'subsequent events excluded until either an outcome is observed or the last observation within the study period.' FastEHR includes leakage control."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses feature leakage through careful fine-tuning design: 'we increase the context window to 512, prepend any previous diagnoses which may otherwise be lost, and remove repeated events.' For risk factor experiments, weight values are masked to ensure self-consistency."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Site-level splitting explicitly addresses non-independence: '90-5-5 site-level training, validation and test random split, dividing patients by general practices in England to avoid data leakage across practice.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Site-level splitting is a concrete leakage prevention method. FastEHR is described as enabling 'controlling for general practice- and dataset-level data leakage.' This is a structural prevention approach rather than post-hoc detection."
       }
     }
   },
   "claims": [
     {
       "claim": "SurvivEHR achieves a marginal IEC of 0.994, meaning the true next event was among the top ~1.6 predicted events on average.",
       "evidence": "Figure 3A and Results section: 'We obtain a marginal IEC of 0.994' compared to 0.864 for prevalence baseline and 0.699 for cross-entropy pre-training.",
       "supported": "strong"
     },
     {
       "claim": "SurvivEHR-FFT achieves superior predictive performance to all benchmarks on clinical risk prediction tasks.",
       "evidence": "Table 2: FFT achieves best Ctd (0.824 vs 0.772 DeSurv), IBS (0.0765 vs 0.0826), and INBLL (0.242 vs 0.262) on hypertension, and best across all metrics on CVD and multimorbidity tasks. Results over 5 seeds with CIs.",
       "supported": "moderate"
     },
     {
       "claim": "Pre-training provides substantial benefits in low-resource fine-tuning settings with fewer than 100,000 patients.",
       "evidence": "Figure 6C: ablation showing FFT maintains performance with reduced cohort sizes while SFT and other baselines degrade more sharply.",
       "supported": "moderate"
     },
     {
       "claim": "SurvivEHR captures clinically meaningful associations such as T2DM→metformin, depression→anxiety, and T2DM→osteoarthritis.",
       "evidence": "Figures 4A-C: transition probability matrices showing known clinical relationships recovered from model predictions, with clinical literature citations supporting each association.",
       "supported": "strong"
     },
     {
       "claim": "Multi-step forecasting degrades after four steps, falling below the prevalence baseline.",
       "evidence": "Figure 3B: marginal IEC evaluated at increasing prediction horizons shows degradation beyond 4 steps.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No compute budget reported",
       "detail": "Training a transformer on 7.6 billion events from 23 million patients requires substantial compute, but no GPU hours, hardware specs, or training time is reported. This limits reproducibility and practical assessment."
     },
     {
       "flag": "Unfair compute comparison with baselines",
       "detail": "SurvivEHR uses full longitudinal histories and is pre-trained on 23M patients, while baselines (RSF, DeepHit, DeSurv) use only cross-sectional input. The information advantage is not discussed as a confound."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'superior performance' rely on comparing point estimates and CIs but no formal statistical tests are reported. For some tasks (e.g., CVD), differences between FFT and DeSurv are within overlapping confidence intervals."
     },
     {
       "flag": "Restricted data limits independent verification",
       "detail": "CPRD data requires governance approval, and model weights cannot be shared due to privacy concerns. While understandable for medical data, this means results cannot be independently verified."
     }
   ],
   "cited_papers": [
     {
       "title": "The shaky foundations of large language models and foundation models for electronic health records",
       "authors": ["Michael Wornow"],
       "year": 2023,
       "relevance": "Critiques foundation models for EHR, directly relevant to evaluating methodology quality of foundation model claims."
     },
     {
       "title": "MOTOR: a time-to-event foundation model for structured medical records",
       "authors": ["Ethan Steinberg"],
       "year": 2023,
       "relevance": "Most directly comparable prior work — a time-to-event foundation model for EHR, key baseline for methodological comparison."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2024,
       "relevance": "Questions emergent capabilities claims in large models, relevant to evaluating overclaiming in foundation model papers."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["Rishi Bommasani"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Foundational paper on foundation model risks and opportunities, directly relevant to survey scope."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Scaling laws paper referenced for model design decisions, relevant to AI methodology."
     },
     {
       "title": "Attention is all you need",
       "authors": ["Ashish Vaswani"],
       "year": 2017,
       "relevance": "Foundational transformer architecture paper, core to the model design."
     }
   ]
 }