scan.json (14404B)
1 { 2 "paper": { 3 "title": "Advancing methodological development of artificial intelligence in patient-centered comparative clinical effectiveness research: Patient-Centered Outcomes Research Institute's unique contribution to research done differently", 4 "authors": ["Jinghua Ou", "Erin Holve"], 5 "year": 2025, 6 "venue": "JAMIA Open", 7 "doi": "10.1093/jamiaopen/ooaf081" 8 }, 9 "checklist": { 10 "artifacts": { 11 "code_released": { 12 "applies": false, 13 "answer": false, 14 "justification": "This is a perspective/commentary piece describing a funding portfolio. There is no code to release." 15 }, 16 "data_released": { 17 "applies": false, 18 "answer": false, 19 "justification": "The paper states 'No new data were generated or analyzed in support of this research.' This is a commentary, not an empirical study." 20 }, 21 "environment_specified": { 22 "applies": false, 23 "answer": false, 24 "justification": "No computational experiments were conducted; this is a perspective piece." 25 }, 26 "reproduction_instructions": { 27 "applies": false, 28 "answer": false, 29 "justification": "No experiments to reproduce; this is a commentary describing a funding portfolio." 30 } 31 }, 32 "statistical_methodology": { 33 "confidence_intervals_or_error_bars": { 34 "applies": false, 35 "answer": false, 36 "justification": "No statistical analysis performed. This is a commentary/perspective piece." 37 }, 38 "significance_tests": { 39 "applies": false, 40 "answer": false, 41 "justification": "No comparative empirical claims requiring significance tests. This is a commentary." 42 }, 43 "effect_sizes_reported": { 44 "applies": false, 45 "answer": false, 46 "justification": "No empirical results reported. This is a perspective piece." 47 }, 48 "sample_size_justified": { 49 "applies": false, 50 "answer": false, 51 "justification": "No empirical study conducted." 52 }, 53 "variance_reported": { 54 "applies": false, 55 "answer": false, 56 "justification": "No experimental runs conducted." 57 } 58 }, 59 "evaluation_design": { 60 "baselines_included": { 61 "applies": false, 62 "answer": false, 63 "justification": "No evaluation or experiment conducted. This is a commentary describing funded projects." 64 }, 65 "baselines_contemporary": { 66 "applies": false, 67 "answer": false, 68 "justification": "No evaluation conducted." 69 }, 70 "ablation_study": { 71 "applies": false, 72 "answer": false, 73 "justification": "No system or method evaluated." 74 }, 75 "multiple_metrics": { 76 "applies": false, 77 "answer": false, 78 "justification": "No evaluation conducted." 79 }, 80 "human_evaluation": { 81 "applies": false, 82 "answer": false, 83 "justification": "No system outputs evaluated. This is a perspective piece." 84 }, 85 "held_out_test_set": { 86 "applies": false, 87 "answer": false, 88 "justification": "No evaluation conducted." 89 }, 90 "per_category_breakdown": { 91 "applies": false, 92 "answer": false, 93 "justification": "No quantitative results to break down." 94 }, 95 "failure_cases_discussed": { 96 "applies": false, 97 "answer": false, 98 "justification": "No system or method evaluated." 99 }, 100 "negative_results_reported": { 101 "applies": false, 102 "answer": false, 103 "justification": "No experiments conducted." 104 } 105 }, 106 "claims_and_evidence": { 107 "abstract_claims_supported": { 108 "applies": true, 109 "answer": true, 110 "justification": "The abstract claims PCORI has funded 'over 40 AI related projects' and describes a framework for patient-centered AI in CER. The body of the paper substantiates these claims by listing specific funded projects and describing the framework." 111 }, 112 "causal_claims_justified": { 113 "applies": false, 114 "answer": false, 115 "justification": "The paper makes no causal claims. It describes a funding portfolio and framework, using aspirational language ('have the potential to', 'aims to') rather than causal assertions." 116 }, 117 "generalization_bounded": { 118 "applies": true, 119 "answer": true, 120 "justification": "The paper is appropriately scoped to PCORI's specific funding portfolio and patient-centered CER. It does not make broad generalizations beyond this context." 121 }, 122 "alternative_explanations_discussed": { 123 "applies": false, 124 "answer": false, 125 "justification": "This is a commentary/perspective piece presenting no empirical results. There are no findings requiring alternative explanations." 126 } 127 }, 128 "setup_transparency": { 129 "model_versions_specified": { 130 "applies": false, 131 "answer": false, 132 "justification": "No models were run or evaluated in this paper. It is a perspective piece." 133 }, 134 "prompts_provided": { 135 "applies": false, 136 "answer": false, 137 "justification": "No prompting was used. This is a commentary." 138 }, 139 "hyperparameters_reported": { 140 "applies": false, 141 "answer": false, 142 "justification": "No experiments conducted." 143 }, 144 "scaffolding_described": { 145 "applies": false, 146 "answer": false, 147 "justification": "No agentic scaffolding used." 148 }, 149 "data_preprocessing_documented": { 150 "applies": false, 151 "answer": false, 152 "justification": "No data processing conducted. The paper explicitly states no new data were generated or analyzed." 153 } 154 }, 155 "limitations_and_scope": { 156 "limitations_section_present": { 157 "applies": true, 158 "answer": false, 159 "justification": "No limitations section is present. The paper does not discuss limitations of its framework or perspective." 160 }, 161 "threats_to_validity_specific": { 162 "applies": true, 163 "answer": false, 164 "justification": "No threats to validity discussed." 165 }, 166 "scope_boundaries_stated": { 167 "applies": true, 168 "answer": false, 169 "justification": "The paper does not explicitly state what its framework does NOT cover or what limitations exist in the portfolio approach described." 170 } 171 }, 172 "data_integrity": { 173 "raw_data_available": { 174 "applies": false, 175 "answer": false, 176 "justification": "No data collected or analyzed. This is a perspective piece." 177 }, 178 "data_collection_described": { 179 "applies": false, 180 "answer": false, 181 "justification": "No data collection performed." 182 }, 183 "recruitment_methods_described": { 184 "applies": false, 185 "answer": false, 186 "justification": "No participants recruited. This is a commentary." 187 }, 188 "data_pipeline_documented": { 189 "applies": false, 190 "answer": false, 191 "justification": "No data pipeline used." 192 } 193 }, 194 "conflicts_of_interest": { 195 "funding_disclosed": { 196 "applies": true, 197 "answer": true, 198 "justification": "The paper includes a Funding section stating 'None declared.'" 199 }, 200 "affiliations_disclosed": { 201 "applies": true, 202 "answer": true, 203 "justification": "Both authors are disclosed as affiliated with PCORI (Patient-Centered Outcomes Research Institute), which is the organization whose funding portfolio is being described." 204 }, 205 "funder_independent_of_outcome": { 206 "applies": true, 207 "answer": false, 208 "justification": "The authors work for PCORI and are writing about PCORI's funding portfolio. While they declare no funding, PCORI is their employer and the subject of the paper, creating a clear conflict: they have an institutional interest in portraying PCORI's portfolio favorably." 209 }, 210 "financial_interests_declared": { 211 "applies": true, 212 "answer": true, 213 "justification": "The paper includes a 'Conflicts of interest' section stating 'None declared.'" 214 } 215 }, 216 "contamination": { 217 "training_cutoff_stated": { 218 "applies": false, 219 "answer": false, 220 "justification": "No model evaluated on any benchmark." 221 }, 222 "train_test_overlap_discussed": { 223 "applies": false, 224 "answer": false, 225 "justification": "No model evaluated on any benchmark." 226 }, 227 "benchmark_contamination_addressed": { 228 "applies": false, 229 "answer": false, 230 "justification": "No model evaluated on any benchmark." 231 } 232 }, 233 "human_studies": { 234 "pre_registered": { 235 "applies": false, 236 "answer": false, 237 "justification": "No human subjects study conducted." 238 }, 239 "irb_or_ethics_approval": { 240 "applies": false, 241 "answer": false, 242 "justification": "No human subjects study conducted." 243 }, 244 "demographics_reported": { 245 "applies": false, 246 "answer": false, 247 "justification": "No human participants." 248 }, 249 "inclusion_exclusion_criteria": { 250 "applies": false, 251 "answer": false, 252 "justification": "No human participants." 253 }, 254 "randomization_described": { 255 "applies": false, 256 "answer": false, 257 "justification": "No human participants." 258 }, 259 "blinding_described": { 260 "applies": false, 261 "answer": false, 262 "justification": "No human participants." 263 }, 264 "attrition_reported": { 265 "applies": false, 266 "answer": false, 267 "justification": "No human participants." 268 } 269 }, 270 "cost_and_practicality": { 271 "inference_cost_reported": { 272 "applies": false, 273 "answer": false, 274 "justification": "This is a perspective/commentary piece, not a methods paper. No system was built or evaluated." 275 }, 276 "compute_budget_stated": { 277 "applies": false, 278 "answer": false, 279 "justification": "No computation performed." 280 } 281 } 282 }, 283 "claims": [ 284 { 285 "claim": "PCORI has funded more than 40 research projects through its methods research portfolio that improve use of AI and ML methods in clinical research.", 286 "evidence": "Stated in the body of the paper with references to specific PCORI project pages (refs 17-36).", 287 "supported": "moderate" 288 }, 289 { 290 "claim": "PCORI's requirement for patient engagement throughout the research lifecycle provides a blueprint for patient-centered AI.", 291 "evidence": "Described through PCORI's Foundational Expectations (ref 13) and examples of funded projects involving patient stakeholder panels, but this is an aspirational claim without empirical validation.", 292 "supported": "weak" 293 } 294 ], 295 "methodology_tags": ["theoretical"], 296 "key_findings": "This is a perspective/commentary piece from PCORI staff describing PCORI's funding portfolio of over 40 AI/ML methods projects in patient-centered comparative clinical effectiveness research. The paper outlines how PCORI's requirement for patient engagement throughout the research lifecycle can serve as a framework for patient-centered AI development. It highlights specific funded projects using LLMs for data extraction, predictive modeling, model evaluation, and ethical assessment in clinical research. No new empirical results are presented.", 297 "red_flags": [ 298 { 299 "flag": "Institutional self-promotion", 300 "detail": "Both authors are PCORI employees writing about PCORI's funding portfolio. The paper reads as organizational advocacy rather than independent scholarly analysis. No critical assessment of PCORI's approach or portfolio effectiveness is provided." 301 }, 302 { 303 "flag": "No evidence of portfolio effectiveness", 304 "detail": "The paper describes over 40 funded projects but provides no evidence that the portfolio has produced methodological improvements, improved patient outcomes, or advanced the state of the art. All claims about impact are aspirational." 305 }, 306 { 307 "flag": "No limitations acknowledged", 308 "detail": "Despite being a perspective piece advocating for a particular approach to AI in clinical research, the paper includes no discussion of limitations, potential drawbacks, or conditions under which the approach might fail." 309 } 310 ], 311 "cited_papers": [ 312 { 313 "title": "Emergent abilities of large language models", 314 "authors": ["J. Wei", "Y. Tay", "R. Bommasani"], 315 "year": 2022, 316 "arxiv_id": "2206.07682", 317 "relevance": "Foundational paper on LLM capabilities, relevant to understanding claims about AI potential." 318 }, 319 { 320 "title": "Are emergent abilities of large language models a mirage?", 321 "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"], 322 "year": 2024, 323 "relevance": "Challenges claims about emergent LLM abilities, relevant to methodological rigor in evaluating AI capabilities." 324 }, 325 { 326 "title": "Evaluation and mitigation of the limitations of large language models in clinical decision-making", 327 "authors": ["P. Hager", "F. Jungmann", "R. Holland"], 328 "year": 2024, 329 "relevance": "Evaluates LLM limitations in clinical settings, directly relevant to AI safety and reliability assessment." 330 }, 331 { 332 "title": "A large language model for electronic health records", 333 "authors": ["X. Yang", "A. Chen", "N. PourNejatian"], 334 "year": 2022, 335 "relevance": "GatorTron model for clinical NLP, relevant to LLM applications in healthcare research." 336 }, 337 { 338 "title": "A study of generative large language model for medical research and healthcare", 339 "authors": ["C. Peng", "X. Yang", "A. Chen"], 340 "year": 2023, 341 "relevance": "GatorTronGPT for medical research, relevant to LLM capability evaluation in clinical domains." 342 }, 343 { 344 "title": "Assessing the research landscape and clinical utility of large language models: a scoping review", 345 "authors": ["Y.J. Park", "A. Pillai", "J. Deng"], 346 "year": 2024, 347 "relevance": "Scoping review of LLM clinical utility, relevant as a survey of AI in healthcare research." 348 } 349 ] 350 }