scan.json (16098B)
1 { 2 "paper": { 3 "title": "Cracking the code: a scoping review to unite disciplines in tackling legal issues in health artificial intelligence", 4 "authors": ["Sophie Nunnelley", "Colleen M Flood", "Michael Da Silva", "Tanya Horsley", "Sarathy Kanathasan", "Bryan Thomas", "Emily Ann Da Silva", "Valentina Ly", "Ryan C Daniel", "Mohsen Sheikh Hassani", "Devin Singh"], 5 "year": 2025, 6 "venue": "BMJ Health Care Inform", 7 "doi": "10.1136/bmjhci-2024-101112" 8 }, 9 "checklist": { 10 "artifacts": { 11 "code_released": { 12 "applies": true, 13 "answer": false, 14 "justification": "No code or repository URL is provided. Custom Python software is mentioned for data analysis but not released." 15 }, 16 "data_released": { 17 "applies": true, 18 "answer": false, 19 "justification": "Data availability statement says 'Data are available upon reasonable request,' which does not count as released." 20 }, 21 "environment_specified": { 22 "applies": true, 23 "answer": false, 24 "justification": "No environment or dependency specifications provided for the custom Python software mentioned." 25 }, 26 "reproduction_instructions": { 27 "applies": true, 28 "answer": false, 29 "justification": "No reproduction instructions provided. The review methodology is described but no scripts or step-by-step instructions to replicate the analysis." 30 } 31 }, 32 "statistical_methodology": { 33 "confidence_intervals_or_error_bars": { 34 "applies": false, 35 "answer": false, 36 "justification": "Scoping review with descriptive statistics only; no experimental results requiring confidence intervals." 37 }, 38 "significance_tests": { 39 "applies": false, 40 "answer": false, 41 "justification": "Scoping review reporting frequencies and proportions; no comparative claims requiring significance tests." 42 }, 43 "effect_sizes_reported": { 44 "applies": false, 45 "answer": false, 46 "justification": "Not applicable to a scoping review that reports descriptive frequencies." 47 }, 48 "sample_size_justified": { 49 "applies": false, 50 "answer": false, 51 "justification": "Scoping review; sample size is determined by the search results, not a statistical design choice." 52 }, 53 "variance_reported": { 54 "applies": false, 55 "answer": false, 56 "justification": "No experimental runs; descriptive mapping review." 57 } 58 }, 59 "evaluation_design": { 60 "baselines_included": { 61 "applies": true, 62 "answer": false, 63 "justification": "No comparison against prior scoping reviews or surveys on the same topic." 64 }, 65 "baselines_contemporary": { 66 "applies": false, 67 "answer": false, 68 "justification": "No baselines included, so contemporaneity is not assessable." 69 }, 70 "ablation_study": { 71 "applies": false, 72 "answer": false, 73 "justification": "Not applicable to a scoping review." 74 }, 75 "multiple_metrics": { 76 "applies": false, 77 "answer": false, 78 "justification": "Not applicable to a scoping review." 79 }, 80 "human_evaluation": { 81 "applies": false, 82 "answer": false, 83 "justification": "Not applicable; this is a scoping review, not a system producing outputs to be evaluated." 84 }, 85 "held_out_test_set": { 86 "applies": false, 87 "answer": false, 88 "justification": "Not applicable to a scoping review." 89 }, 90 "per_category_breakdown": { 91 "applies": true, 92 "answer": true, 93 "justification": "Results are broken down by discipline (medicine, law, engineering, etc.) and by legal issue type, with figures 2-5 providing detailed breakdowns." 94 }, 95 "failure_cases_discussed": { 96 "applies": true, 97 "answer": true, 98 "justification": "The paper discusses missing voices (CS/engineering underrepresentation, Global South absence) and gaps in disciplinary coverage as failure cases in the literature." 99 }, 100 "negative_results_reported": { 101 "applies": true, 102 "answer": true, 103 "justification": "The paper highlights negative findings: minimal engagement from AI developers, absence of clinician-driven informed consent literature, and missing Global South voices." 104 } 105 }, 106 "claims_and_evidence": { 107 "abstract_claims_supported": { 108 "applies": true, 109 "answer": true, 110 "justification": "Abstract claims about disciplinary differences in frequency and nature of legal issue discussions are supported by the results section with percentage breakdowns and figures." 111 }, 112 "causal_claims_justified": { 113 "applies": false, 114 "answer": false, 115 "justification": "The paper makes descriptive claims about patterns in the literature, not causal claims." 116 }, 117 "generalization_bounded": { 118 "applies": true, 119 "answer": false, 120 "justification": "The paper acknowledges English/French language limitation but makes broad claims about 'missing voices' and 'disciplinary differences' without adequately bounding to the searched databases and time period. The title implies comprehensive coverage ('cracking the code') beyond what a 2012-2021 English/French search supports." 121 }, 122 "alternative_explanations_discussed": { 123 "applies": true, 124 "answer": false, 125 "justification": "The paper does not substantively discuss alternative explanations for the observed patterns. For example, CS/engineering authors may publish legal discussions in venues not indexed by the searched databases, but this is not explored." 126 } 127 }, 128 "setup_transparency": { 129 "model_versions_specified": { 130 "applies": false, 131 "answer": false, 132 "justification": "No AI models used in the methodology." 133 }, 134 "prompts_provided": { 135 "applies": false, 136 "answer": false, 137 "justification": "No prompting used." 138 }, 139 "hyperparameters_reported": { 140 "applies": false, 141 "answer": false, 142 "justification": "No AI models used." 143 }, 144 "scaffolding_described": { 145 "applies": false, 146 "answer": false, 147 "justification": "No agentic scaffolding used." 148 }, 149 "data_preprocessing_documented": { 150 "applies": true, 151 "answer": true, 152 "justification": "The paper documents the filtering pipeline: 18,168 screened records → 432 included, with a PRISMA flow diagram (Figure 1), defined inclusion/exclusion criteria (Stage 3), and detailed extraction procedures (Stage 4). Criteria at each stage are stated." 153 } 154 }, 155 "limitations_and_scope": { 156 "limitations_section_present": { 157 "applies": true, 158 "answer": false, 159 "justification": "No dedicated limitations or threats-to-validity section. The English/French limitation is mentioned briefly in the Discussion but there is no substantive limitations section." 160 }, 161 "threats_to_validity_specific": { 162 "applies": true, 163 "answer": false, 164 "justification": "No specific threats to validity are discussed. The language limitation is mentioned but not as part of a structured validity discussion." 165 }, 166 "scope_boundaries_stated": { 167 "applies": true, 168 "answer": false, 169 "justification": "The paper does not explicitly state what its results do NOT show. The temporal boundary (2012-2021) and language restriction are mentioned but specific scope limitations are not articulated." 170 } 171 }, 172 "data_integrity": { 173 "raw_data_available": { 174 "applies": true, 175 "answer": false, 176 "justification": "Data available only 'upon reasonable request,' not publicly available for independent verification." 177 }, 178 "data_collection_described": { 179 "applies": true, 180 "answer": true, 181 "justification": "Detailed description of database searches, search strategy development with trained librarians, peer review of search strategy (PRESS checklist), and date ranges. Full search terms referenced in supplemental materials." 182 }, 183 "recruitment_methods_described": { 184 "applies": false, 185 "answer": false, 186 "justification": "No human participants recruited; this is a literature review." 187 }, 188 "data_pipeline_documented": { 189 "applies": true, 190 "answer": true, 191 "justification": "The six-stage Arksey and O'Malley framework is described with PRISMA flow diagram showing counts at each stage (18,168 → 432). Extraction tool development, pilot testing, and coding procedures are documented." 192 } 193 }, 194 "conflicts_of_interest": { 195 "funding_disclosed": { 196 "applies": true, 197 "answer": true, 198 "justification": "Funding disclosed: CIHR (grant 452650), Alex Trebek Forum for Dialogue, Hospital for Sick Children Research Institute." 199 }, 200 "affiliations_disclosed": { 201 "applies": true, 202 "answer": true, 203 "justification": "All author affiliations are listed with numbered institutional affiliations." 204 }, 205 "funder_independent_of_outcome": { 206 "applies": true, 207 "answer": true, 208 "justification": "Funders are public research bodies (CIHR, academic institutions) with no financial stake in the review's findings. Paper explicitly states 'The funders had no other role in this study.'" 209 }, 210 "financial_interests_declared": { 211 "applies": true, 212 "answer": true, 213 "justification": "'Competing interests: None declared' is explicitly stated." 214 } 215 }, 216 "contamination": { 217 "training_cutoff_stated": { 218 "applies": false, 219 "answer": false, 220 "justification": "No pre-trained model evaluated on any benchmark." 221 }, 222 "train_test_overlap_discussed": { 223 "applies": false, 224 "answer": false, 225 "justification": "No pre-trained model evaluated on any benchmark." 226 }, 227 "benchmark_contamination_addressed": { 228 "applies": false, 229 "answer": false, 230 "justification": "No pre-trained model evaluated on any benchmark." 231 } 232 }, 233 "human_studies": { 234 "pre_registered": { 235 "applies": false, 236 "answer": false, 237 "justification": "No human participants; this is a scoping review of published literature." 238 }, 239 "irb_or_ethics_approval": { 240 "applies": false, 241 "answer": false, 242 "justification": "No human participants. Paper states 'Ethics approval: Not applicable.'" 243 }, 244 "demographics_reported": { 245 "applies": false, 246 "answer": false, 247 "justification": "No human participants." 248 }, 249 "inclusion_exclusion_criteria": { 250 "applies": false, 251 "answer": false, 252 "justification": "No human participants. (Paper inclusion/exclusion criteria are documented under data_preprocessing_documented.)" 253 }, 254 "randomization_described": { 255 "applies": false, 256 "answer": false, 257 "justification": "No human participants." 258 }, 259 "blinding_described": { 260 "applies": false, 261 "answer": false, 262 "justification": "No human participants." 263 }, 264 "attrition_reported": { 265 "applies": false, 266 "answer": false, 267 "justification": "No human participants." 268 } 269 }, 270 "cost_and_practicality": { 271 "inference_cost_reported": { 272 "applies": false, 273 "answer": false, 274 "justification": "Survey paper; no inference or computational method whose cost is relevant." 275 }, 276 "compute_budget_stated": { 277 "applies": false, 278 "answer": false, 279 "justification": "Survey paper; no significant computation." 280 } 281 } 282 }, 283 "claims": [ 284 { 285 "claim": "There has been exponential growth in literature discussing legal issues with health AI, with 950% growth between 2012-2016 and 2914% between 2016-2020.", 286 "evidence": "Figure 2 and Results section show the growth trajectory across the included 432 studies.", 287 "supported": "moderate" 288 }, 289 { 290 "claim": "Authors in medicine (36%) and law (28%) dominate the literature, while computer science (4%) and engineering (4%) authors are minimally represented.", 291 "evidence": "Figure 4 provides disciplinary distribution of 432 included studies.", 292 "supported": "strong" 293 }, 294 { 295 "claim": "Disciplines prioritise different legal issues: medicine focuses on privacy and safety/quality after regulatory efficiency, while law focuses on liability after regulatory efficiency.", 296 "evidence": "Results section with disciplinary breakdowns of legal issue frequency.", 297 "supported": "strong" 298 }, 299 { 300 "claim": "Medicine and law authors propose different solutions: medicine favors voluntary improvements (33%), law favors new legislation (34%).", 301 "evidence": "Figure 5 and accompanying text in Results section.", 302 "supported": "strong" 303 } 304 ], 305 "methodology_tags": ["meta-analysis", "qualitative"], 306 "key_findings": "This scoping review of 432 studies (from 18,168 screened) found exponential growth in literature on legal issues in health AI from 2012-2021. Medicine and law dominate authorship, while AI developers (CS/engineering) are notably absent from legal discussions. Disciplines diverge on priorities (medicine: privacy/safety; law: liability) and preferred solutions (medicine: voluntary measures; law: new legislation). The authors argue that effective AI governance requires bridging these disciplinary silos.", 307 "red_flags": [ 308 { 309 "flag": "No limitations section", 310 "detail": "For a scoping review, the absence of a dedicated limitations section is a notable omission. Language restriction (English/French), temporal cutoff (2021), and database selection could significantly affect findings but are not systematically discussed." 311 }, 312 { 313 "flag": "Data not publicly released", 314 "detail": "Extracted data from 432 studies is only 'available upon reasonable request' despite being from publicly available published literature. This limits reproducibility." 315 }, 316 { 317 "flag": "Discipline classification by corresponding author only", 318 "detail": "Categorizing papers by the corresponding author's faculty affiliation is a rough proxy that may misclassify interdisciplinary work or papers where the corresponding author is not the domain expert." 319 } 320 ], 321 "cited_papers": [ 322 { 323 "title": "AI in health and medicine", 324 "authors": ["P Rajpurkar", "E Chen", "O Banerjee"], 325 "year": 2022, 326 "relevance": "Major review of AI applications in healthcare, relevant to understanding the scope of health AI deployment." 327 }, 328 { 329 "title": "Regulatory considerations on artificial intelligence for health", 330 "year": 2023, 331 "relevance": "WHO guidance on AI regulation in health, directly relevant to AI governance and safety frameworks." 332 }, 333 { 334 "title": "Ethics and Law in Research on Algorithmic and Data-Driven Technology in Mental Health Care: Scoping Review", 335 "authors": ["P Gooding", "T Kariotis"], 336 "year": 2021, 337 "relevance": "Prior scoping review on legal/ethical issues in AI for mental health, relevant methodology comparison." 338 }, 339 { 340 "title": "Regulating the Safety of Health-Related Artificial Intelligence", 341 "authors": ["M Da Silva", "CM Flood", "A Goldenberg"], 342 "year": 2022, 343 "relevance": "Examines regulatory frameworks for health AI safety, directly relevant to AI safety governance." 344 }, 345 { 346 "title": "Generative AI and large language models in health care: pathways to implementation", 347 "authors": ["MM Raza", "KP Venkatesh", "JC Kvedar"], 348 "year": 2024, 349 "relevance": "Discusses LLM implementation pathways in healthcare, relevant to understanding deployment challenges." 350 } 351 ] 352 }