ai-research-survey

scan.json (26304B)

---

 {
   "paper": {
     "title": "Retrieval-Augmented Generation for Electrocardiogram-Language Models",
     "authors": [
       "Xiaoyu Song",
       "William Han",
       "Tony Chen",
       "Chaojing Duan",
       "Michael A. Rosenberg",
       "Emerson Liu",
       "Ding Zhao"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.00261",
     "doi": "10.48550/arXiv.2510.00261"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Adding a RAG pipeline to Electrocardiogram-Language Models (ELMs) consistently improves NLG performance across three public ECG datasets and multiple ELM architectures. Ablation studies show that using RAG during both training and inference yields the best results, top-1 retrieval marginally outperforms larger k values, RAG content placement (system prompt vs. user query) has negligible effect, and accurate retrieval content is necessary for gains (noise injection nullifies the benefit).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL: 'Our code is available at: https://github.com/willxxy/ECG-Bench' in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: MIMIC-IV-ECG, PTB-XL, ECG-QA, and ECG-Chat Instruct. The paper cites their public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA RTX A6000 48 GB GPUs' and 'HuggingFace Transformers API' but provides no requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released, the paper itself does not include commands or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "All tables report ± notation across 3 random seeds (e.g., '22.85 ± 0.18' in Table 1)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'RAG consistently improves performance') but reports no statistical significance tests (no p-values, t-tests, or bootstrap tests). Differences are assessed only by comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables report both baseline and RAG-augmented numbers with sufficient context to assess magnitude (e.g., BLEU-4 from 22.85% to 38.10% on ECG-Chat Instruct, accuracy from 27.21% to 57.77% on MIMIC-IV ECG in Table 1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 400,000 training and 20,000 test instances but provides no justification for these sizes and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across 3 random seeds are reported in all results tables (e.g., '38.10 ± 0.05' in Table 1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares ECG-Byte with and without RAG. Figure 2 extends to three additional ELM architectures (Merl, SigLIP Signal, SigLIP Image) with and without RAG."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ECG-Byte [12] (2024), ECG-Chat [4] (2024), and the SigLIP/Merl variants from [14] (2025) are all recent and represent the state of the art for ELMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive ablation studies in Tables 2-5: RAG during training/inference combinations, varying top-k, RAG placement in prompt, and noise injection into retrieved content."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are reported: BLEU-4, ROUGE-L, METEOR, BERTScore F1, and Accuracy (Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated text quality is included. All evaluation relies on automated NLG metrics. For a system generating clinical ECG interpretations, human expert evaluation would be valuable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states: 'We inference on a separate test set of size 20,000 instances.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset breakdowns across three datasets (ECG-Chat Instruct, ECG-QA PTB-XL, ECG-QA MIMIC-IV ECG) rather than a single aggregate."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis, qualitative failure examples, or discussion of where the approach breaks down. The paper only reports aggregate metrics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows training-only RAG hurts performance vs. baseline (BLEU-4: 20.08 vs. 22.85). Table 3 shows increasing k doesn't help. Table 5 shows noisy RAG is worse than clean RAG on accuracy (8.19% vs. 18.27%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'ELMs with RAG consistently improves performance over non-RAG baselines,' which is supported by Table 1 and Figure 2. Claims about ablation study insights are supported by Tables 2-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about RAG improving performance are supported by controlled ablation studies (Tables 2-5) with single-variable manipulation: RAG on/off, varying k, placement, and noise injection."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to ELMs and tested settings. The title specifies 'Electrocardiogram-Language Models,' and the paper tests on three specific datasets and four architectures. Discussion states 'showing consistent improvements in NLG across multiple datasets and architectures' without overclaiming beyond ECG tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why RAG helps. The paper does not consider confounds such as whether the gains come from additional context length, memorization of training examples, or other factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures NLG metrics (BLEU-4, ROUGE-L, METEOR, BERTScore, Accuracy) and frames them as evidence that RAG improves ELM performance. However, it does not discuss the gap between these automated metrics and actual clinical utility or diagnostic accuracy, which is what would matter in practice for ECG interpretation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model checkpoints are identified: 'Llama-3.2-1B-Instruct checkpoint' (Section 2.3), 'siglip-base-patch16-224 checkpoint from HuggingFace' (Section 3.1), 'Res-Net101 backbone' (Section 3.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The prompt structure is described (system prompt + retrieved diagnostic reports + ECG tokens + query) in Figure 1 and Section 2.2, but the actual prompt text (system prompt content, formatting) is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports: Adam optimizer, learning rate 1e-4, weight decay 1e-2, batch size 2, 1 epoch, 400k samples, LoRA rank 16, input length 1024, BPE 3500 merges, FAISS IndexIVFFlat with L2 distance."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard RAG-augmented fine-tuned LLM pipeline without agent loops, tool use, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents preprocessing in detail: standardization to PTB-XL lead configuration, powerline noise removal (notch filter Q=30 at 50/60 Hz), Butterworth bandpass filter (0.5-100 Hz), baseline drift correction (0.05 Hz high-pass), wavelet denoising (Daubechies-6, level 4, MAD soft thresholding), resampling to 250 Hz, 5-second windowing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 5 ('Discussion and Conclusions') summarizes findings and 'insights for future work' but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show or what settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All three datasets are publicly available: MIMIC-IV-ECG [9], PTB-XL [10], and ECG-QA [11]. The raw ECG signals and associated data can be independently accessed."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data sources: MIMIC-IV-ECG and PTB-XL datasets adapted for NLG, ECG-Chat Instruct dataset with conversational data, and ECG-QA dataset with ChatGPT-generated Q&A pairs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public ECG datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes preprocessing steps but does not document the full pipeline with counts at each stage. It states '400,000 randomly sampled ECGs' for training and '20,000 instances' for test without explaining total available data, sampling criteria, or filtering losses."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section only mentions a collaboration with the Mario Lemieux Center for Heart Rhythm Care at Allegheny General Hospital."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University, Columbia University, Allegheny Health Network, University of Colorado."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder from outcome cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama-3.2-1B-Instruct as the base model but does not state its training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained Llama model's training data could include any of the ECG datasets or their textual descriptions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The ECG-QA dataset includes ChatGPT-generated Q&A pairs from publicly available ECG datasets. No discussion of whether these or similar content could appear in Llama's training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example timing is reported despite the method adding a FAISS retrieval step to each inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA RTX A6000 48 GB GPUs' but does not state total GPU hours, training time, or overall computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "All results are averaged over 3 random seeds with standard deviations reported (e.g., Tables 1-5)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Table 1 header state: 'Mean baseline comparisons over 3 random seeds' and 'All results are averaged over 3 random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper states 'using default hyperparameters' without documenting whether any search was performed or why defaults were appropriate."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses default hyperparameters without justification. LoRA rank=16, learning rate 1e-4, and other choices are not justified or compared against alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across Tables 1-5 but no statistical tests are performed, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own RAG pipeline against baselines they implemented. No acknowledgment of potential bias from self-evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RAG adds retrieval overhead (FAISS search, additional context in prompts) but no analysis of performance vs. compute cost is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses NLG metrics (BLEU-4, ROUGE-L, METEOR, BERTScore, Accuracy) without discussing whether these metrics actually measure meaningful ECG interpretation quality or clinical utility."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. RAG is the intervention being tested, and all comparisons use the same base architecture with controlled RAG variations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ECG datasets or their textual descriptions existed before Llama-3.2's training data cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the RAG retrieval mechanism could leak test-time information (e.g., if the RAG database contains training data that overlaps with test queries)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test ECG instances share structural similarities (e.g., from the same patients, same recording sessions, or same hospital systems)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is described."
       }
     }
   },
   "claims": [
     {
       "claim": "ELMs with RAG consistently improve NLG performance over non-RAG baselines across three datasets.",
       "evidence": "Table 1 shows improvements on ECG-Chat Instruct (BLEU-4: 22.85→38.10), ECG-QA PTB-XL (21.07→21.46), and ECG-QA MIMIC-IV (15.52→49.07). Figure 2 shows gains across four ELM architectures.",
       "supported": "moderate"
     },
     {
       "claim": "Using RAG during both training and inference yields the best performance.",
       "evidence": "Table 2 shows RAG at both stages (BLEU-4: 38.10, Acc: 18.27) outperforms training-only (20.08, 7.17), inference-only (32.33, 14.96), and no RAG (22.85, 9.65). Only tested on ECG-Chat Instruct.",
       "supported": "moderate"
     },
     {
       "claim": "Retrieving fewer items (top-1) marginally outperforms larger k values.",
       "evidence": "Table 3: k=1 (38.10 BLEU-4, 18.27 Acc) vs k=5 (37.99, 18.07) vs k=10 (36.91, 17.20). Differences are small. Only tested on ECG-Chat Instruct.",
       "supported": "moderate"
     },
     {
       "claim": "RAG content placement (system prompt vs. user query) produces similar outcomes.",
       "evidence": "Table 4: system prompt (38.08 BLEU-4, 18.11 Acc) vs user query (38.03, 18.17). Differences within noise. Only tested on ECG-Chat Instruct.",
       "supported": "moderate"
     },
     {
       "claim": "Accurate retrieval content is necessary for performance gains; noise injection nullifies benefits.",
       "evidence": "Table 5: RAG with noise (23.48 BLEU-4, 8.19 Acc) performs similarly to no RAG (22.85, 9.65), while proper RAG yields 38.10 BLEU-4 and 18.27 Acc.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No significance tests",
       "detail": "All comparative claims ('RAG consistently improves') are based on comparing point estimates with standard deviations but no formal statistical tests. Some improvements on ECG-QA PTB-XL are marginal (BLEU-4: 21.07→21.46) and could be within noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations section or any discussion of where the approach might fail, what its scope boundaries are, or threats to validity."
     },
     {
       "flag": "Ablations on single dataset only",
       "detail": "All ablation studies (Tables 2-5) use only the ECG-Chat Instruct dataset. The generalizability of the ablation findings to the other two datasets is unknown, especially given the highly variable gains across datasets in Table 1."
     },
     {
       "flag": "No clinical validation or human evaluation",
       "detail": "For a system generating clinical ECG interpretations, the evaluation relies entirely on automated NLG metrics. No clinician reviewed the generated reports for clinical accuracy or safety."
     },
     {
       "flag": "Construct validity of metrics unexamined",
       "detail": "BLEU-4 and ROUGE-L measure n-gram overlap with reference text, not clinical correctness. A generated report could score high on these metrics while being clinically misleading, or score low while being clinically valid but worded differently."
     }
   ],
   "cited_papers": [
     {
       "title": "ECG-Chat: A Large ECG-Language Model for Cardiac Disease Diagnosis",
       "authors": ["Yubao Zhao", "Tian Zhang", "Xu Wang", "Puyu Han", "Tong Chen", "Linlin Huang", "Youzhu Jin", "Jiaju Kang"],
       "year": 2024,
       "relevance": "First work to extend RAG to free-form NLG for ELMs; directly compared baseline in this paper."
     },
     {
       "title": "RankRAG: Unifying Context Ranking with Retrieval-Augmented Generation in LLMs",
       "authors": ["Yue Yu", "Wei Ping", "Zihan Liu", "Boxin Wang", "Jiaxuan You", "Chao Zhang", "Mohammad Shoeybi", "Bryan Catanzaro"],
       "year": 2024,
       "relevance": "Key reference on RAG design for LLMs, cited for insights on how increasing k does not necessarily improve performance."
     },
     {
       "title": "ECG-Byte: A Tokenizer for End-to-End Generative Electrocardiogram Language Modeling",
       "authors": ["William Han", "Chaojing Duan", "Michael A. Rosenberg", "Emerson Liu", "Ding Zhao"],
       "year": 2024,
       "relevance": "Primary ELM architecture used in all experiments; BPE-based ECG tokenization approach."
     },
     {
       "title": "Q-Heart: ECG Question Answering via Knowledge-Informed Multimodal LLMs",
       "authors": ["Hung Manh Pham", "Jialu Tang", "Aaqib Saeed", "Dong Ma"],
       "year": 2025,
       "relevance": "Related RAG-augmented ECG QA system using RAG during instruction tuning."
     },
     {
       "title": "Zero-Shot ECG Diagnosis with Large Language Models and Retrieval-Augmented Generation",
       "authors": ["Han Yu", "Peikun Guo", "Akane Sano"],
       "year": 2023,
       "relevance": "Prior work applying RAG with LLMs for ECG diagnosis using handcrafted features."
     },
     {
       "title": "ECG-QA: A Comprehensive Question Answering Dataset Combined with Electrocardiogram",
       "authors": ["Jungwoo Oh", "Gyubok Lee", "Seongsu Bae", "Joon myoung Kwon", "Edward Choi"],
       "year": 2023,
       "relevance": "Public ECG question-answering benchmark dataset used in evaluation."
     },
     {
       "title": "Signal, Image, or Symbolic: Exploring the Best Input Representation for Electrocardiogram-Language Models through a Unified Framework",
       "authors": ["William Han", "Chaojing Duan", "Zhepeng Cen", "Yihang Yao", "Xiaoyu Song"],
       "year": 2025,
       "relevance": "Unified ELM framework providing the baseline architectures (Merl, SigLIP variants) compared in Figure 2."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J. Hu", "Yelong Shen", "Phillip Wallis", "Zeyuan Allen-Zhu", "Yuanzhi Li", "Shean Wang", "Lu Wang", "Weizhu Chen"],
       "year": 2021,
       "relevance": "Parameter-efficient fine-tuning method used for all ELM training in this paper."
     },
     {
       "title": "Visual Instruction Tuning",
       "authors": ["Haotian Liu", "Chunyuan Li", "Qingyang Wu", "Yong Jae Lee"],
       "year": 2023,
       "relevance": "LLaVA approach adopted for encoder-based ELM architectures in the paper."
     },
     {
       "title": "Retrieval-Augmented Generation for Large Language Models: A Survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao", "Kangxiang Jia", "Jinliu Pan", "Yuxi Bi", "Yi Dai", "Jiawei Sun", "Meng Wang", "Haofen Wang"],
       "year": 2024,
       "relevance": "Comprehensive RAG survey cited for RAG design during pretraining/finetuning practices."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Open-source RAG pipeline for ECG interpretation with code released; applicable to medical AI practitioners but limited to the ECG domain."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "RAG improving NLG performance is expected and well-established in the broader LLM literature."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns are raised; the paper focuses on performance improvement."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward method contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub (ECG-Bench), though requires ECG datasets and GPU hardware to run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Carnegie Mellon University is well-known but the specific lab and authors are not widely recognized in the broader AI community."
     }
   }
 }