ai-research-survey

scan.json (24755B)

---

 {
   "paper": {
     "title": "ClinNoteAgents: An LLM Multi-Agent System for Predicting and Interpreting Heart Failure 30-Day Readmission from Clinical Notes",
     "authors": ["Rongjia Zhou", "Chengzhuo Li", "Carl Yang", "Jiaying Lu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.07081",
     "doi": "10.48550/arXiv.2512.07081"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the publicly available MIMIC-III database (ref 29), which is a standard public dataset. The data source is clearly identified and accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper states LLM agents used Qwen3-14B and Qwen3-8B but does not specify library versions, hardware, or dependency details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described at a high level but lacks the specificity needed for exact reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports 95% confidence intervals for odds ratios (e.g., Age OR 95% CI: 1.002–1.015; Weight OR 95% CI: 0.643–0.942). However, the classification results in Figure 2 report only point estimates for AUROC without confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Logistic regression p-values are reported in Table 5 and chi-square test p-values in Table 6 for association analyses. Statistical tests are appropriately applied to categorical vs. continuous variables."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports odds ratios with 95% CIs for logistic regression results (e.g., Weight OR=0.778, Age OR=1.008). Chi-square statistics are reported in Table 6. These provide effect size context for the association analyses."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the sample size (2,065 patients, 3,544 notes) is provided. The cohort is defined by the MIMIC-III filtering criteria but there is no discussion of whether this sample size is adequate for the analyses performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Classification results (AUROC values in Figure 2) are reported as single-run point estimates with no standard deviation, variance, or spread across multiple runs. No mention of multiple experimental runs or cross-validation folds with variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares three classifiers (TF-IDF+LR, ClinicalBERT, LoRA-finetuned Qwen3-8B) across four input conditions (raw notes, overall summary, no-number summary, structural extraction). Raw discharge notes serve as the baseline for summarization comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "ClinicalBERT (2020) is now 5 years old. No comparison against more recent clinical NLP models or state-of-the-art HF readmission prediction methods is included. The paper does not justify why ClinicalBERT is still an appropriate baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates the summarization component by comparing four input representations (raw, overall summary, no-number summary, structural extraction) across three classifiers, showing the contribution of different summarization strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "For classification, only AUROC is reported (Figure 2). For extraction, multiple metrics are used (coverage, conditional accuracy, MAE, MAPE), but the core readmission prediction task uses a single metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human expert evaluation of the system's outputs (extracted risk factors, summaries, predictions) is included. The extraction is compared against structured EHR fields as surrogate ground truth, and diagnosis extraction uses LLM-as-a-judge, but no clinician review of outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe any train/test/validation split for the classification experiments. There is no mention of held-out test sets, cross-validation, or how data was partitioned for the readmission prediction evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-variable breakdowns for extraction accuracy across vitals and charted SDOH. Tables 5 and 6 provide per-variable association analyses. Figure 2 breaks down classification results by summarization method and classifier type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section discusses cases where extraction failed or performed poorly: Height (68.50%) and Weight (57.90%) had lower accuracy due to heterogeneous reporting formats and unit-conversion issues. The diagnosis extraction gap is also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly reports that 'summarization did not improve classification performance as we initially expected' and that most SDOH variables commonly reported as influential were not statistically significant in their cohort."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'strong performance in extracting risk factors,' 'identifying key contributing factors,' and 'predicting readmission risk,' which are supported by Tables 2, 5, 6, and Figure 2. The claims are reasonably hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses language like 'Age and BP are positively associated with HF readmission risk' which is correlational. However, the abstract claims the system 'reduces reliance on structured fields' and 'provides a scalable approach' — implying causal improvement. The association analyses (logistic regression, chi-square) are observational and cross-sectional, not causal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on MIMIC-III (a single US ICU dataset) but the title and abstract make broad claims about 'data-limited healthcare systems' and 'developing countries.' The Discussion notes that 'evaluating patients across multiple sites may be needed' but the title and framing significantly overreach the single-dataset evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Discussion offers some interpretation of why certain variables were not significant (under-documentation of SDOH, LLM extraction limitations) but does not systematically consider alternative explanations for the main results, such as whether the summarization performance preservation is due to redundancy in clinical notes rather than quality of summarization."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Qwen3-14B' and 'Qwen3-8B' and 'ClinicalBERT' but does not provide specific version identifiers, snapshot dates, or checkpoint hashes. 'Qwen3-14B' is a model family name, not a precise version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides actual prompt text for the risk factor extractor (with output schema example), the overall summary prompt, and the no-number summary prompt. These are concrete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM inference parameters are reported. For ClinicalBERT and LoRA fine-tuning, no learning rate, batch size, or training hyperparameters are specified. The k-medoids clustering used k=200 but LLM hyperparameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent pipeline is described with three distinct agents (Extractor, Normalizer, Summarizer) and their workflow is outlined in Figure 1. The two-stage normalization process (normalizer generates categories, labeler assigns them) is described in detail."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data selection pipeline is described: ICD-9 codes used for HF patient identification, readmission pair construction from sequential admissions, and the labeling logic (yi=1 if interval ≤30 days). Table 1 provides cohort summary statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' subsection is present in the Discussion section, discussing three specific limitations: indirect evaluation of extracted risk factors, lack of performance gains from summarization, and single-site evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: reliance on structured EHR as surrogate ground truth for extraction evaluation, LLM-as-judge limitations for diagnosis validation, and the need for multi-site evaluation. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section acknowledges multi-site evaluation is needed, the paper does not explicitly state what the results do NOT show. The broad framing about 'developing countries' and 'data-limited healthcare systems' is not bounded by explicit statements about what settings the results do not apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The study uses MIMIC-III, which is publicly available (with credentialed access). The underlying patient data can be independently verified by other researchers with MIMIC-III access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: MIMIC-III database, HF patients identified via specific ICD-9 codes (listed explicitly), readmission pairs constructed by linking sequential admissions, discharge summaries used as input notes."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study uses a standard public clinical database (MIMIC-III) as its data source."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from MIMIC-III to final analysis is documented: ICD-9 filtering → readmission pair construction → discharge note extraction → LLM processing (extraction, normalization, summarization) → statistical analysis/classification. Table 1 provides cohort statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Emory University. The affiliations are clearly stated at the top of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is itself a transparency gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen3-14B and Qwen3-8B (LoRA fine-tuned) and ClinicalBERT for readmission prediction on MIMIC-III data. No training data cutoff dates are stated for any of these models. MIMIC-III data could be in their training corpora."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MIMIC-III data (a widely-used public dataset) may have appeared in the training data of Qwen3 or ClinicalBERT. This is a significant concern given MIMIC-III's ubiquity in ML research."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MIMIC-III has been publicly available since 2016 and is widely used in ML research. The paper does not address the possibility that Qwen3 models may have been exposed to MIMIC-III-derived content during pre-training, which could inflate extraction and classification performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study performs secondary analysis of de-identified clinical records from MIMIC-III."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. MIMIC-III access requires credentialing but not IRB approval for secondary analysis of de-identified data. The paper discusses ethical considerations regarding data handling."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study design. Patient demographics (age, gender) are reported as cohort characteristics in Table 1, but these are properties of the dataset, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. Cohort selection criteria (ICD-9 codes) are described for the clinical dataset but this is data filtering, not participant recruitment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, token counts, API costs, or wall-clock time are reported despite the system making multiple LLM calls per note (extraction, normalization, labeling, summarization) across 3,544 notes. The paper claims the approach is 'scalable' without quantifying cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, hardware specifications, or total compute requirements are reported. The paper mentions models were 'deployed locally within secure computing environments' but provides no details."
       }
     }
   },
   "claims": [
     {
       "claim": "ClinNoteAgents achieves high extraction fidelity for clinical variables, with conditional accuracy above 84% for most vital signs.",
       "evidence": "Table 2 shows conditional accuracy ranging from 57.90% (Weight) to 94.16% (SpO2), with 6 of 8 vitals above 84%. MAE and MAPE metrics are also reported.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-generated summaries preserve most predictive signal despite 60-90% text reduction.",
       "evidence": "Figure 2 shows AUROC drops from 0.6535 (raw) to 0.6434 (no-number summary, 61% reduction) for LR, and to 0.5866 (overall summary, 83% reduction). Performance is retained but some drops are non-trivial.",
       "supported": "moderate"
     },
     {
       "claim": "Age, weight, and blood pressure are significantly associated with HF 30-day readmission.",
       "evidence": "Table 5 reports p=0.008 for Age, p=0.010 for Weight, p<0.001 for BP SYS, p=0.037 for BP DIA, with odds ratios and 95% CIs.",
       "supported": "strong"
     },
     {
       "claim": "Housing is the only statistically significant SDOH variable associated with HF readmission.",
       "evidence": "Table 6 reports chi-square=21.13, p=0.012 for Housing, with all other SDOH variables having p>0.05.",
       "supported": "strong"
     },
     {
       "claim": "ClinNoteAgents provides a scalable and interpretable approach to note-based HF readmission risk modeling in data-limited healthcare systems.",
       "evidence": "No scalability metrics, cost data, or evaluation in data-limited settings is provided. The claim in the abstract is not supported by evidence in the paper.",
       "supported": "weak"
     },
     {
       "claim": "LLM-based diagnosis extraction achieves conditional accuracy of 62.67% with mean similarity score of 3.04.",
       "evidence": "Table 3 reports these metrics from LLM-as-a-judge evaluation comparing LLM-extracted diagnoses to ICD-9 codes.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "ClinNoteAgents is a multi-agent LLM framework using Qwen3-14B to extract structured risk factors from clinical discharge notes for heart failure 30-day readmission prediction. On MIMIC-III (2,065 patients), the extraction agents achieve 84-94% conditional accuracy for most vital signs against structured EHR ground truth. Logistic regression identifies age, weight, and blood pressure as significant predictors, while housing is the only significant SDOH factor. LLM summarization preserves most predictive signal despite 60-90% text reduction, though it does not improve over raw notes.",
   "red_flags": [
     {
       "flag": "No train/test split described for classification",
       "detail": "The readmission prediction experiments (Figure 2) report AUROC values without describing how the data was split into training and test sets, whether cross-validation was used, or how model selection was performed. This makes the classification results unverifiable."
     },
     {
       "flag": "Overclaimed generalizability",
       "detail": "The paper tests only on MIMIC-III (a US ICU dataset) but frames the contribution around 'data-limited healthcare systems' and 'developing countries in Asia and Africa.' No evidence from non-US or resource-limited settings is presented."
     },
     {
       "flag": "No uncertainty quantification for classification",
       "detail": "AUROC values are reported as single numbers without confidence intervals, standard deviations, or multiple-run results. With a 35% readmission rate and moderate AUROC values (0.55-0.65), the differences between methods may not be statistically significant."
     },
     {
       "flag": "MIMIC-III contamination risk",
       "detail": "MIMIC-III has been publicly available since 2016 and is one of the most widely-used clinical datasets in ML. Qwen3 models may have been exposed to MIMIC-III-derived content during pre-training, which could inflate extraction performance. This is not discussed."
     },
     {
       "flag": "LLM-as-a-judge without validation",
       "detail": "Diagnosis extraction is evaluated using an LLM-as-a-judge approach (ref 32), but the judge itself is not validated against human expert ratings. This creates a circular dependency where LLMs evaluate LLM outputs."
     },
     {
       "flag": "No inference cost despite scalability claims",
       "detail": "The paper claims the approach is 'scalable' but reports no inference costs, token counts, or processing time despite running multiple LLM agents per note across 3,544 notes."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models to identify social determinants of health in electronic health records",
       "authors": ["M. Guevara", "S. Chen", "S. Thomas"],
       "year": 2024,
       "relevance": "Directly relevant as prior work on using LLMs for structured information extraction from clinical notes."
     },
     {
       "title": "ClinicalBERT: Modeling Clinical Notes and Predicting Hospital Readmission",
       "authors": ["K. Huang", "J. Altosaar", "R. Ranganath"],
       "year": 2020,
       "arxiv_id": "1904.05342",
       "relevance": "Key baseline model used in the evaluation, represents transformer-based clinical NLP approach."
     },
     {
       "title": "DistillNote: LLM-Based Clinical Note Summaries Improve Heart Failure Diagnosis",
       "authors": ["H. O. Boll", "A. O. Boll", "L. P. Boll"],
       "year": 2025,
       "arxiv_id": "2506.16777",
       "relevance": "Directly related work on LLM summarization of clinical notes for downstream clinical tasks."
     },
     {
       "title": "SDoH-GPT: Using Large Language Models to Extract Social Determinants of Health",
       "authors": ["B. Consoli", "H. Wang", "X. Wu"],
       "year": 2025,
       "relevance": "Prior work on using LLMs specifically for SDOH extraction from clinical text."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["Qwen Team"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for the primary LLM used in the ClinNoteAgents system."
     },
     {
       "title": "LLM Judge: Building Automatic Evaluators with Large Language Models",
       "authors": ["Hugging Face"],
       "year": 2024,
       "relevance": "Framework used for LLM-as-a-judge evaluation of diagnosis extraction quality."
     },
     {
       "title": "Hallucinations and Key Information Extraction in Medical Texts: A Comprehensive Assessment of Open-Source Large Language Models",
       "authors": ["A. B. Das", "S. Ahmed", "S. K. Sakib"],
       "year": 2025,
       "relevance": "Relevant to LLM reliability concerns in clinical information extraction, cited for hallucination evidence."
     },
     {
       "title": "Mining Social Determinants of Health for Heart Failure Patient 30-Day Readmission via Large Language Model",
       "authors": ["M. Shao", "Y. Kang", "X. Hu"],
       "year": 2025,
       "relevance": "Direct predecessor work from same group on LLM-based SDOH mining for HF readmission."
     },
     {
       "title": "MIMIC-III, a freely accessible critical care database",
       "authors": ["A. E. Johnson", "T. J. Pollard", "L. Shen"],
       "year": 2016,
       "relevance": "The primary dataset used in this study; widely used benchmark for clinical NLP research."
     }
   ]
 }