ai-research-survey

scan.json (19497B)

---

 {
   "paper": {
     "title": "Concordance of randomised controlled trials for artificial intelligence interventions with the CONSORT-AI reporting guidelines",
     "authors": ["Alexander P. L. Martindale", "Carrie D. Llewellyn", "Richard O. de Visser", "Benjamin Ng", "Victoria Ngai", "Aditya U. Kale", "Lavinia Ferrante di Ruffano", "Robert M. Golub", "Gary S. Collins", "David Moher", "Melissa D. McCradden", "Lauren Oakden-Rayner", "Samantha Cruz Rivera", "Melanie Calvert", "Christopher J. Kelly", "Cecilia S. Lee", "Christopher Yau", "An-Wen Chan", "Pearse A. Keane", "Andrew L. Beam", "Alastair K. Denniston", "Xiaoxuan Liu"],
     "year": 2024,
     "venue": "Nature Communications",
     "doi": "10.1038/s41467-024-45355-3"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are provided. The Data availability section states materials are 'available upon request to the corresponding author.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Supplementary Data 1 and 2 are provided with the paper, containing the full list of included RCTs and concordance data. The paper states 'All data used in this study is referenced and publicly available. Supplementary information has been provided.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper mentions using SPSS Version 25.0, Covidence, and R Statistical Software v4.1.1, but no requirements file or reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The Methods section describes the methodology but does not provide a reproducible workflow with specific commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "IQRs are reported throughout for concordance values, e.g., 'Median concordance with CONSORT-AI reporting was 90% (IQR 77–94%).' IQR serves as an uncertainty measure for the median."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Spearman's Rank-Order Correlation was used to test whether concordance changed with publication date: 'Spearman's r = −0.21, p = 0.091.' P-values under 0.05 were considered significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Concordance differences are reported with enough context: e.g., CONSORT-AI mandated journals had 100% concordance vs 90% (IQR 76–94%) for non-mandated. Spearman's r = −0.21 is an effect size measure."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The 65 RCTs represent all eligible studies found, which is appropriate for a systematic review, but the paper acknowledges 'this exploratory analysis was limited by the small number of studies' without formal justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "IQRs are reported for all concordance metrics, providing spread measures. E.g., 'Median sample size across all included RCTs was 186 (IQR 56-654).'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares concordance across subgroups: those using CONSORT-AI vs CONSORT 2010 vs no guidelines (Table 2), and journals mandating vs not mandating guidelines (Table 4). Previous systematic reviews are also compared in the Discussion."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against three prior systematic reviews (refs 84-86) that used CONSORT-AI to evaluate RCTs, noting methodological differences and earlier search dates."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Concordance is reported for all CONSORT-AI items combined, AI-specific items only, and non-AI-specific items separately. Per-item concordance percentages are also provided."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review assessing reporting completeness; there is no system output to evaluate with human judges."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a systematic review; there is no train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-item concordance for all 14 AI-specific CONSORT-AI items. Results are also broken down by geography, guideline usage, and journal mandate status."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion identifies specific poorly reported items (algorithm version at 20%, code accessibility at 42%, protocol access at 31%) and discusses why these items were underreported."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that no significant correlation was found between publication date and concordance (Spearman's r = −0.21, p = 0.091), and that only 3/52 journals mandated CONSORT-AI."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (90% median concordance, only 10 RCTs explicitly used CONSORT-AI, only 3/52 journals endorsed it, algorithm version and code accessibility poorly reported) are all supported by the Results section and tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal links, e.g., 'This may point towards a higher level of editorial scrutiny in journals which promote better reporting practices.' This is observational data and the association could be confounded by journal quality, author expertise, etc. The paper uses hedging language ('may') but does not discuss specific confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its scope to English-language RCTs published after September 2020, and acknowledges limitations of its search strategy including potential incomplete retrieval and exclusion of non-English RCTs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion considers alternative explanations: high concordance without CONSORT-AI awareness may reflect general clinical trial standards; low algorithm version reporting may reflect commercial considerations; the narrow date range limits correlation analysis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review; no AI models are used or evaluated by the authors."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting or LLMs used in this systematic review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in the conduct of this review."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The PRISMA flow diagram (Fig. 1) documents the full pipeline: 8263 records imported → 5111 after deduplication → 332 full-text review → 65 included, with specific exclusion counts and reasons at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a substantive limitations discussion within the Discussion section, covering incomplete study retrieval, timing of CONSORT-AI adoption, search term confounding, and exclusion of non-English RCTs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: search terms for AI/ML confound item 1a,b(i) concordance assessment; RCTs may have been submitted before CONSORT-AI was known; trial registry indexing errors could cause missed studies; non-English exclusion biases the sample given the diverse geographical spread."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers only English-language RCTs submitted after September 2020, excludes conference abstracts and protocols, and notes the exploratory correlation analysis was limited by small N and narrow date range."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Supplementary Data 1 contains the full list of included RCTs; Supplementary Data 2 contains concordance data for non-AI-specific items. Further materials are available upon request."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The Methods section describes the search strategy (MEDLINE, Embase, Cochrane, clinical trial registries), search date (19 September 2022), date range (from 9 September 2020), and data extraction procedure with two independent reviewers."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. This is a systematic review of published papers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The PRISMA flow diagram documents each stage with counts: 8263 imported → 3152 duplicates removed → 4779 excluded at screening → 267 excluded at full-text with itemized reasons → 65 included. Data extraction by two independent reviewers with conflict resolution described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is published in Nature Communications which requires funding and competing interests disclosures. The affiliations and acknowledgments sections are present (though specific funding details would be in the full published version's declarations)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are fully listed. Notably, several authors (Liu, Denniston, Cruz Rivera, Calvert, Collins, Moher) are authors of the original CONSORT-AI guidelines (ref 8), and this connection is acknowledged in the Methods ('carried out in conjunction with CONSORT-AI authors')."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Several authors developed CONSORT-AI itself (ref 8). They have a reputational interest in CONSORT-AI being adopted and shown to be useful. This potential conflict is not explicitly discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is visible in the extracted text. The paper does not include an explicit declaration of financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review; no pre-trained models are evaluated on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable; no model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable; no model evaluation on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review of published papers, not a study with human participants. However, the review protocol was registered on OSF (doi.org/10.17605/OSF.IO/CRF3Q)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a systematic review of published papers."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study selection criteria for papers are documented in the Methods section."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review; no computational method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review; no significant compute was required."
       }
     }
   },
   "claims": [
     {
       "claim": "Median concordance with CONSORT-AI reporting was 90% (IQR 77–94%) across 65 included AI RCTs.",
       "evidence": "Results section and Table 2 report this figure based on assessment of all 51 CONSORT-AI items across 65 RCTs.",
       "supported": "strong"
     },
     {
       "claim": "Only 10 of 65 RCTs (15%) explicitly reported use of CONSORT-AI.",
       "evidence": "Results section states this directly, with breakdown in Table 2 showing subgroup concordance.",
       "supported": "strong"
     },
     {
       "claim": "Algorithm version (item 5(i)) was the most poorly reported AI-specific item at only 20% concordance.",
       "evidence": "Table 3 provides per-item concordance for all 14 AI-specific items.",
       "supported": "strong"
     },
     {
       "claim": "Only 3 of 52 journals explicitly endorsed or mandated CONSORT-AI.",
       "evidence": "Results section on journal mandates states this, identifying The Lancet Digital Health, The Lancet Gastroenterology, and Ophthalmology Science.",
       "supported": "strong"
     },
     {
       "claim": "There was no significant correlation between date of publication and CONSORT-AI concordance.",
       "evidence": "Spearman's r = −0.21, p = 0.091, acknowledged as limited by small N and narrow date range.",
       "supported": "moderate"
     },
     {
       "claim": "RCTs in journals mandating CONSORT 2010 had higher overall concordance (92%) than those without mandates (82%).",
       "evidence": "Table 4 provides the breakdown with IQRs. However, no formal significance test is reported for this comparison.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic review of 65 AI RCTs published after CONSORT-AI's release found generally high reporting concordance (median 90%), but identified persistent gaps in algorithm version reporting (20%) and code accessibility (42%). Only 10/65 RCTs explicitly used CONSORT-AI and only 3/52 publishing journals mandated it. Journal endorsement of reporting guidelines was associated with higher concordance, suggesting editorial oversight plays a meaningful role in reporting quality.",
   "red_flags": [
     {
       "flag": "Authors evaluated their own guideline",
       "detail": "Several authors of this systematic review (Liu, Denniston, Cruz Rivera, Calvert, Collins, Moher) are the developers of CONSORT-AI itself (ref 8). They have a reputational interest in CONSORT-AI being shown to be useful and adopted. This conflict is not explicitly discussed."
     },
     {
       "flag": "No formal comparison tests between subgroups",
       "detail": "The paper reports concordance differences between subgroups (e.g., journals mandating vs not mandating guidelines) but does not apply formal statistical tests to these comparisons, only to the time trend. The subgroup differences could be coincidental."
     }
   ],
   "cited_papers": [
     {
       "title": "Reporting guidelines for clinical trial reports for interventions involving artificial intelligence: the CONSORT-AI extension",
       "authors": ["X. Liu", "S. Cruz Rivera", "D. Moher", "M. J. Calvert", "A. K. Denniston"],
       "year": 2020,
       "doi": "10.1038/s41591-020-1034-x",
       "relevance": "The CONSORT-AI reporting guideline itself, directly relevant to AI methodology standards and reporting quality."
     },
     {
       "title": "A comparison of deep learning performance against health-care professionals in detecting diseases from medical imaging: a systematic review and meta-analysis",
       "authors": ["X. Liu"],
       "year": 2019,
       "relevance": "Meta-analysis comparing AI to clinicians in medical imaging, relevant to evaluating AI system claims and methodology."
     },
     {
       "title": "CONSORT 2010 Statement: updated guidelines for reporting parallel group randomised trials",
       "authors": ["K. F. Schulz", "D. G. Altman", "D. Moher"],
       "year": 2010,
       "relevance": "The foundational RCT reporting standard that CONSORT-AI extends, relevant to methodology quality assessment frameworks."
     },
     {
       "title": "Reducing waste from incomplete or unusable reports of biomedical research",
       "authors": ["P. Glasziou"],
       "year": 2014,
       "relevance": "Motivates why reporting quality matters — incomplete reporting leads to research waste, directly relevant to the survey's methodology quality theme."
     }
   ]
 }