ai-research-survey

scan.json (24298B)

---

 {
   "paper": {
     "title": "Exploring AI-Augmented Sensemaking of Patient-Generated Health Data: A Mixed-Method Study with Healthcare Professionals in Cardiac Risk Reduction",
     "authors": [
       "Pavithren V. S. Pakianathan",
       "Rania Islambouli",
       "Diogo Branco",
       "Albrecht Schmidt",
       "Tiago Guerreiro",
       "Jan David Smeddinck"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.05687"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["qualitative", "observational"],
   "key_findings": "In a mixed-methods study with 16 HCPs, LLM-generated summaries of patient-generated health data were perceived as reducing information overload and anchoring data exploration, though workload differences (NASA-TLX) were not statistically significant. Conversational interfaces were valued for bridging data literacy gaps and enabling personalized analysis without data science expertise. HCPs raised concerns about overreliance, deskilling, privacy, and transparency, and broadly viewed AI as an assistant rather than replacement. Trust in AI summaries correlated with confidence in final plans (Spearman r=.46, p=.001).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The prototype was built using Plotly Dash but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic PGHD, personas, and collected study data are not released. No data download links are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python and the Plotly Dash framework but provides no requirements.txt, dependency versions, or environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the study apparatus but does not include steps to replicate the system or study."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results report means and SDs (e.g., SUS AI mean=90.63, SD=8.44) but no confidence intervals or error bars. The Spearman correlation is reported with a p-value but no CI."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Wilcoxon signed-rank tests are used for paired comparisons of NASA-TLX and SUS scores. Spearman correlations are used for trust-confidence associations. Linear mixed-effects models are also mentioned for robustness verification."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Spearman r=.46 for trust-confidence correlation, and provides raw means enabling comparison (e.g., NASA-TLX AI=24.53 vs No-AI=27.40, a ~3.9-point reduction). SUS scores and confidence ratings are reported with means providing baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 16 HCPs is not justified with a power analysis. The limitations section acknowledges the sample was 'underpowered for detecting small or medium effects' but no a priori justification is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported alongside means for all quantitative measures (e.g., SUS AI: SD=8.44, NASA-TLX AI: SD=11.99, confidence AI: SD=0.67, trust: SD=0.98)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a within-subjects design comparing AI Summary vs No-AI Summary conditions, providing a direct baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline (no AI summary, charts only) is appropriate for this formative study comparing AI-augmented vs non-augmented sensemaking of the same data."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is not a system with separable components to ablate. It is an HCI study comparing two conditions (with/without AI summaries) plus a conversational interface exploration."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple measures are used: NASA-TLX (workload, 6 subscales), SUS (usability), self-assessed confidence, trust in AI summaries, MiniVLAT (visualization literacy), plus qualitative thematic analysis."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study IS a human evaluation — 16 HCPs reviewed the system, rated it on multiple scales, and provided qualitative feedback through semi-structured interviews."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study, not a benchmark evaluation. There is no train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "NASA-TLX subscale breakdowns are provided (Figure 6 spider chart). Qualitative themes are broken down by category (sensemaking value, workflow augmentation, risks/tensions). Provenance analysis is broken down by modality (Tables 4, 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "P12 identified a blood pressure classification error in one summary. The paper discusses concerns about accuracy, overreliance, and deskilling. The provenance analysis reveals MAPD variability across modalities."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that workload differences were not statistically significant, SUS differences were not significant, and that confidence was comparable across conditions (AI: M=4.38 vs No-AI: M=4.27)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are hedged appropriately ('Findings show that AI summaries provided quick overviews', 'HCPs raised concerns about transparency, privacy, and overreliance'). These match the qualitative and quantitative findings in the results section."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is carefully formative and exploratory. It uses language like 'perceived', 'valued', 'suggested' rather than causal claims. The within-subjects comparison is appropriate for the claims made. The paper explicitly states it does not 'evaluate clinical effectiveness.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states findings should be 'interpreted as insights into perceptions, practices, and sensemaking dynamics rather than as evaluations of clinical performance.' The Limitations section bounds scope to controlled conditions, single-user sessions, synthetic data, and the specific clinical domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses novelty effects (mitigated via demo mode), notes that lack of time pressure may have muted quantitative differences, discusses that synthetic data may not capture real-world variability, and acknowledges heterogeneity in human-AI collaboration effects (citing Vaccaro et al.)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames this as studying 'perceptions, workflow fit, and sociotechnical considerations' rather than clinical effectiveness. It distinguishes between perceived value (what they measured) and actual clinical impact (not claimed). The paper states 'Our aim is not to evaluate clinical effectiveness.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-4-Turbo' for data generation and the appendix provides the model name. Temperature (0.5) and max tokens (1024) are specified in the appendix prompt configuration."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix A.2 for all four data modalities (physical activity, sedentary time, blood pressure, sleep) and the combined data prompt. The prompts include role descriptions, formatting instructions, column definitions, and data passed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The appendix specifies: Model: gpt-4-turbo, Temperature: 0.5, Max tokens: 1024."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system makes direct LLM API calls for summary generation and conversational interaction without agent loops, tool use, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes synthetic data generation: persona design with SCORE2 risk stratification, building on Henriksen et al. dataset, generated with ChatGPT using Python randomization functions, one year of multimodal PGHD across four modalities. Data verified by lead researcher and two HCPs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 is a dedicated 'Limitation' section with substantial discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: LLM not pre-trained on sensor data so summaries may be inaccurate, no strict time limits muting efficiency gains, N=16 underpowered for non-parametric tests, synthetic rather than real PGHD, HCPs working alone rather than with patients present, study conducted in non-native English language."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'Our aim is not to evaluate clinical effectiveness, but to generate insights into acceptable and effective interaction patterns.' It states findings reflect 'controlled conditions rather than deployment realities' and that triadic patient-HCP-AI interactions were not captured."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the synthetic PGHD, questionnaire responses, interview transcripts, nor chat logs are released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 (Procedure) details the 75-minute session structure: demographics questionnaire, MiniVLAT, demo familiarization, two experimental conditions with randomized order, conversational interface interaction, and semi-structured interview. Section 3.6 describes all instruments used."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states: 'We recruited participants via an email publicity sent to HCPs working in cardiac care at a university hospital, which was subsequently shared within their professional networks.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The study procedure is documented step by step (Figure 5). Audio transcription used OpenAI Whisper, verified by first author. Qualitative coding: two authors co-coded 4 participants, then first author coded remaining. Quantitative analysis methods specified (Wilcoxon, Spearman, linear mixed-effects). Post-hoc provenance analysis of LLM outputs is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grant information is found in the paper. Authors are affiliated with Ludwig Boltzmann Institute, LMU Munich, and LASIGE but no funding source is stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Ludwig Boltzmann Institute for Digital Health and Prevention (Salzburg), LMU Munich, and LASIGE/Universidade de Lisboa."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It is a user study evaluating HCPs' sensemaking with an LLM-augmented interface."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation. The study evaluates HCP perceptions, not model performance on test data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link or mention of pre-registration is found in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 states: 'Our study protocol received official approval from the relevant institutional ethics committee (blinded for review) prior to data collection.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 reports: 12 women, 4 men; M=31.4 years, SD=5.0; specialized in cardiovascular rehabilitation; M=9.1 years experience, SD=5.5, range 2-20; 12 regularly used PGHD; MiniVLAT M=9.47/12; AI usage patterns detailed."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The paper states participants were HCPs in cardiac care recruited via email but does not specify formal inclusion/exclusion criteria. It notes none participated in prior co-design activities, but this is the only stated criterion."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states conditions were presented in 'randomized order.' Personas were 'stratified by CVD risk: medium, high, very high' with 'three unique personas' per condition block. Figure 5 shows the randomized structure."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. The AI vs No-AI conditions were visually distinct (presence/absence of summaries), and participants were aware of which condition they were in. This is acknowledged implicitly but not discussed as a limitation."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported. The paper simply states 16 HCPs participated without mentioning whether any were excluded or dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or inference latency for the GPT-4-Turbo calls are reported, despite the system making multiple LLM calls per session."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for the LLM usage in the study."
       }
     }
   },
   "claims": [
     {
       "claim": "AI summaries reduced perceived time and effort for data sensemaking",
       "evidence": "15 of 16 participants reported summaries were valuable in reducing information overload. NASA-TLX showed ~3.9-point lower workload (AI: 24.53 vs No-AI: 27.40) but difference was not statistically significant (Section 4.1.1). Qualitative quotes from P12, P2, P13, P6 support perceived value.",
       "supported": "moderate"
     },
     {
       "claim": "AI summaries served as anchors for data exploration, changing how HCPs approached data review",
       "evidence": "Qualitative findings in Section 4.1.2 describe HCPs beginning with summaries rather than scanning charts, offloading mental effort. Quotes from P5 and P2 support this pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Conversational interfaces bridged data literacy gaps for HCPs",
       "evidence": "Section 4.2.3 provides quotes from P5 and P4 describing how the chatbot enabled analyses they could not perform themselves. P4 specifically notes 'we don't have that knowledge' regarding data analysis skills.",
       "supported": "moderate"
     },
     {
       "claim": "Higher trust in AI summaries correlated with greater confidence in final physical activity plans",
       "evidence": "Spearman r=.46, p=.001 (Section 4.3.2). Trust M=4.27, SD=0.98; confidence AI M=4.38, SD=0.67.",
       "supported": "strong"
     },
     {
       "claim": "LLM-generated statistics were broadly accurate with MAPD of 3.96% for holistic insights and 2.68% for chat logs",
       "evidence": "Post-hoc provenance analysis in Section 3.1 comparing LLM-derived averages against ground-truth values. Tables 4 and 5 provide modality-specific breakdowns. All 25 checked ranges were accurate.",
       "supported": "strong"
     },
     {
       "claim": "AI augmentation did not significantly increase workload or reduce usability",
       "evidence": "SUS scores: AI=90.63 vs No-AI=85.94, no significant difference. NASA-TLX: AI=24.53 vs No-AI=27.40, no significant difference on Wilcoxon signed-rank tests (Section 4.1.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample underpowered for quantitative claims",
       "detail": "N=16 is acknowledged as underpowered for detecting small or medium effects. The non-significant quantitative results may reflect insufficient power rather than absence of effect, yet the paper draws conclusions about workload and usability from these comparisons."
     },
     {
       "flag": "No blinding in within-subjects design",
       "detail": "Participants knew whether they were in the AI or No-AI condition (visible presence/absence of summaries). Demand characteristics and novelty effects could inflate perceived value of the AI condition. The demo familiarization mitigates but does not eliminate this concern."
     },
     {
       "flag": "Synthetic data limits ecological validity",
       "detail": "All PGHD was synthetically generated, and the study used personas rather than real patients. The paper acknowledges this but still draws design implications for real clinical workflows."
     },
     {
       "flag": "Recruitment from single institution",
       "detail": "All 16 HCPs were recruited from a single university hospital and its professional networks, potentially limiting diversity of perspectives. The sample skews young (23-42 years) and female (12/16)."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "relevance": "RCT measuring AI coding tool impact on developer productivity — directly relevant to AI programming productivity evidence base."
     },
     {
       "title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",
       "authors": ["Michelle Vaccaro", "Abdullah Almaatouq", "Thomas Malone"],
       "year": 2024,
       "doi": "10.1038/s41562-024-02024-1",
       "relevance": "Meta-analysis on human-AI collaboration effectiveness showing heterogeneous effects — key evidence on whether AI augmentation actually helps."
     },
     {
       "title": "Clinician-Facing AI in the Wild: Taking Stock of the Sociotechnical Challenges and Opportunities for HCI",
       "authors": ["Hubert D. Zając", "Dana Li", "Xiang Dai"],
       "year": 2023,
       "doi": "10.1145/3582430",
       "relevance": "Survey of sociotechnical challenges in clinical AI deployment, relevant to understanding real-world AI integration barriers."
     },
     {
       "title": "Vital Insight: Assisting Experts' Context-Driven Sensemaking of Multi-modal Personal Tracking Data Using Visualization and Human-In-The-Loop LLM Agents",
       "authors": ["Jiachen Li", "Xiwen Li", "Justin Steinberg"],
       "year": 2025,
       "arxiv_id": "2410.14879",
       "relevance": "Closely related work on LLM agents for expert sensemaking of multimodal personal tracking data."
     },
     {
       "title": "Narrating Fitness: Leveraging Large Language Models for Reflective Fitness Tracker Data Interpretation",
       "authors": ["Konstantin R. Strömel", "Stanislas Henry", "Tim Johansson"],
       "year": 2024,
       "doi": "10.1145/3613904.3642032",
       "relevance": "Prior work on LLMs generating narratives from fitness tracker data — direct precursor to this study's approach."
     },
     {
       "title": "MindfulDiary: Harnessing Large Language Model to Support Psychiatric Patients' Journaling",
       "authors": ["Taewan Kim", "Seolyeong Bae", "Hyun Ah Kim"],
       "year": 2024,
       "doi": "10.1145/3613904.3642937",
       "relevance": "LLM application in mental health data sensemaking, demonstrating AI-generated summaries for clinical insights."
     },
     {
       "title": "From Classification to Clinical Insights: Towards Analyzing and Reasoning About Mobile and Behavioral Health Data With Large Language Models",
       "authors": ["Zachary Englhardt", "Chengqian Ma", "Margaret E. Morris"],
       "year": 2024,
       "doi": "10.1145/3659604",
       "relevance": "LLMs for analyzing mobile and behavioral health data — directly relevant to AI-augmented clinical data interpretation."
     },
     {
       "title": "Adapted large language models can outperform medical experts in clinical text summarization",
       "authors": ["Dave Van Veen"],
       "year": 2024,
       "relevance": "Evidence that LLMs can outperform medical experts in summarization tasks, relevant to AI capability claims in healthcare."
     },
     {
       "title": "SensorLM: Learning the Language of Wearable Sensors",
       "authors": ["Yuwei Zhang", "Kumar Ayush", "Siyuan Qiao"],
       "year": 2025,
       "arxiv_id": "2506.09108",
       "relevance": "Fine-tuned model translating wearable sensor data into human-readable narratives — technical approach for PGHD interpretation."
     }
   ]
 }