ai-research-survey

scan.json (23163B)

---

 {
   "paper": {
     "title": "Comparing code-free and bespoke deep learning approaches in ophthalmology",
     "authors": [
       "Carolyn Yu Tung Wong",
       "Ciara O'Byrne",
       "Priyal Taribagil",
       "Timing Liu",
       "Fares Antaki",
       "Pearse Andrew Keane"
     ],
     "year": 2024,
     "venue": "Graefe's Archive for Clinical and Experimental Ophthalmology",
     "doi": "10.1007/s00417-024-06432-x"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, analysis scripts, or supplementary code are mentioned anywhere in the paper. As a review paper, analysis or data extraction scripts could have been released but were not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or structured extraction data is released. The review relies on published literature but does not release its own extracted data tables or search results in a reusable format."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative mini review paper with no computational experiments. There is no software environment to specify."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The search methodology is briefly described (PubMed search with keywords 'autoML' AND 'ophthalmology') but there are no detailed instructions for reproducing the review process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative review that does not conduct any statistical analyses. It reports performance metrics from reviewed studies but does not perform its own statistical computations."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are applicable; this is a narrative review comparing studies qualitatively, not performing meta-analysis."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are computed by the authors. The paper reports raw performance metrics from the reviewed studies but does not calculate any pooled effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No statistical experiments are conducted by the authors. Sample size justification is not applicable to this narrative review."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are performed. This is a narrative review, so variance across runs is not applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The review does not compare itself against prior reviews or surveys on the same topic. It does compare CFDL vs bespoke DL studies, but this is the content of the review, not a baseline comparison for the review itself as a methodology."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "As a narrative mini review, there is no experimental evaluation with baselines. The question of contemporary baselines does not apply."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method with components to ablate. This is a review paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted by the authors. This is a narrative review."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are produced that would require human evaluation. This is a review paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted. This is a narrative review paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The review is organized by task category (DR screening, retinal multi-disease classification, surgical video classification, oculomics, resource management) with detailed per-task tables (Tables 1-5) showing breakdowns of each study's performance, limitations, and findings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations and failure modes of reviewed studies throughout, including lack of external validation, small datasets, class imbalance, limited generalizability, and black-box nature. The Limitations section explicitly discusses biases and challenges."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings: 'discussions of CFDL were mostly done mono-dimensionally' and 'positive conclusions drawn on CFDL's benefits were largely based on the system-derived performance results' without holistic comparison. It also reports performance drops during external validation (e.g., Korot et al.'s accuracy dropped from 86.5% to 78.6%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'studies were optimistic towards CFDL's advantages over bespoke DL' but discussions were 'mono-dimensional and had wide applicability gaps.' Both claims are supported by the detailed per-task analyses in the Results and Discussion sections."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make causal claims. It is a narrative review that reports findings from existing studies and argues for a multidimensional evaluation framework, without claiming causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to ophthalmological tasks and notes limitations: 'Our model-to-model comparison per task is subject to biases because of the different datasets used' and acknowledges the review covers only 10 studies across 5 specific tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section considers that apparent CFDL advantages may be artifacts of different datasets, lack of external validation, and mono-dimensional evaluation. The Limitations section discusses dataset biases, single-centre designs, and small dataset sizes as alternative explanations for reported performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used by the authors. This is a review paper that discusses models used in the reviewed studies."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a review paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted by the authors. This is a review paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a review paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The search process is described at a high level (PubMed search with 'autoML' AND 'ophthalmology', then subsequent search for equivalent DL studies), with a flow diagram in Fig. 1. However, the actual filtering criteria beyond language and article type are vague. No clear criteria for how 'equivalent DL studies' were identified for each task. The paper says 'We identified ten relevant studies' without specifying how many were initially found or the screening criteria at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present near the end of the paper, discussing biases from different datasets, lack of external validation, black-box challenges, and single-centre study designs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: biases from different datasets used for CFDL vs bespoke models, lack of routine external validation across reviewed studies, uncertain effectiveness of external validation against adversarial attacks, single-centre observational study designs, and small dataset sizes with class imbalance."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what is outside its scope. It focuses on 5 ophthalmological tasks but does not explicitly list what tasks or aspects it did not cover. The limitations discuss biases but do not draw clear boundaries around what claims are and are not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data, extracted metrics, or search results are made available for independent verification. The structured comparison data in the tables is not released in a machine-readable format."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The Methods section describes the search process: MEDLINE/PubMed search on June 25, 2023, using keywords 'autoML' AND 'ophthalmology', followed by a second search for equivalent DL studies. Inclusion criteria (English, full text available) and exclusion criteria (reviews, editorials, protocols, case reports/series) are stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a review of published literature."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly describes the search strategy and references Fig. 1 for the search process, but the pipeline from initial search results to the final 10 studies is not documented with counts at each stage. There is no PRISMA-style reporting of how many studies were found initially, screened, and excluded at each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section listing grants or sponsors is provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Institute of Ophthalmology (UCL), Moorfields Eye Hospital NHS Foundation Trust, Chinese University of Hong Kong, CHUM School of AI in Healthcare, and NIHR Moorfields Biomedical Research Centre. Notably, author Fares Antaki is co-author on some of the reviewed CFDL studies (Antaki et al. [18])."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be assessed. The absence of a funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Declarations section states: 'The authors declare no competing interests.' A competing interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a review paper that does not evaluate any pre-trained model on a benchmark. Contamination criteria do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a review paper, not a benchmark evaluation. Contamination criteria do not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a review paper, not a benchmark evaluation. Contamination criteria do not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a review of published studies."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper explicitly states: 'This article does not contain any studies with human participants or animals performed by any of the authors.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this review."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper. It discusses costs of CFDL in general (e.g., 'processing up to 35,000 images with less than US$100') but does not have its own method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper with no computational experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CFDL has shown comparable performance to bespoke DL models across five ophthalmological tasks (DR screening, multi-retinal disease classification, surgical video classification, oculomics, resource management).",
       "evidence": "Tables 1-5 and the Results section present per-task comparisons. For example, the CFDL DR screening model achieved ACC 97% and F1 96%, while bespoke DL achieved AUROC 0.99 and SN 93.86%. The CFDL surgical video classifier achieved ACC 96% vs. bespoke DL's 84%.",
       "supported": "moderate"
     },
     {
       "claim": "Discussions of CFDL's advantages over bespoke DL in the reviewed literature are largely mono-dimensional, focusing on performance metrics without considering implementation factors like patient acceptance and cost-effectiveness.",
       "evidence": "The Discussion section states: 'positive conclusions drawn on CFDL's benefits were largely based on the system-derived performance results' and 'discussions of CFDL were mostly done mono-dimensionally, seldomly discussing other implementation demands of AI.' This is supported by the observation that none of the 5 CFDL studies included patient attitude information (p. 2794).",
       "supported": "strong"
     },
     {
       "claim": "CFDL and bespoke DL are unique in their assets and irreplaceable with each other; their benefits are differentially valued on a case-to-case basis.",
       "evidence": "The Conclusion states this directly, supported by the per-task discussion showing CFDL advantages in some contexts (low-cost prototyping, resource management) and bespoke DL advantages in others (interpretability for diagnostic tasks). However, this is based on a narrative comparison of only 10 studies.",
       "supported": "moderate"
     },
     {
       "claim": "External validation has not been routinely performed across the reviewed studies, making claims of model robustness liable to biases.",
       "evidence": "The Limitations section notes this, and the tables confirm: Nunez et al. (Table 1), Abitbol et al. (Table 2), Yeh et al. (Table 3), Munk et al. (Table 4), and both resource management studies (Table 5) had 'NR' (not reported) for externally validated performance metrics.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This mini review compares code-free deep learning (CFDL) and bespoke deep learning across five ophthalmological tasks (DR screening, retinal disease classification, surgical video classification, oculomics, and resource management), finding that CFDL shows comparable performance to bespoke DL in the reviewed studies. The authors argue that existing evaluations are mono-dimensional, focusing only on model performance metrics while neglecting implementation considerations such as developer intent, patient acceptance, cost-effectiveness, and model interpretability. The paper concludes that CFDL and bespoke DL serve complementary roles and that future work should adopt a multidimensional assessment framework.",
   "red_flags": [
     {
       "flag": "Author conflict of interest not acknowledged",
       "detail": "Fares Antaki, a co-author of this review, is also a co-author of Antaki et al. [18], one of the 5 CFDL studies being reviewed. This conflict is not disclosed or discussed, despite the review assessing CFDL's promise. The review generally presents CFDL favorably."
     },
     {
       "flag": "Very narrow search strategy",
       "detail": "The literature search used only 'autoML' AND 'ophthalmology' in PubMed, which is extremely narrow and likely missed relevant CFDL studies that use different terminology (e.g., 'no-code deep learning', 'automated deep learning', 'transfer learning platform'). Only 10 studies were included."
     },
     {
       "flag": "Non-systematic review presented with systematic language",
       "detail": "The paper describes a search strategy and uses tables to structure its comparison, but lacks key systematic review elements: no PRISMA checklist, no risk-of-bias assessment of included studies, no protocol registration, and no documentation of study counts at each screening stage."
     },
     {
       "flag": "Comparisons across different datasets",
       "detail": "The review compares CFDL and bespoke DL models that were trained on different datasets (acknowledged in Limitations), making performance comparisons unreliable. This is a fundamental methodological limitation that undermines the paper's central comparison."
     },
     {
       "flag": "No quality assessment of reviewed studies",
       "detail": "The review does not apply any quality assessment tool (e.g., PROBAST, QUADAS-2) to the included studies. Without structured quality assessment, the review may be laundering weak methodology from its source studies."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated deep learning in ophthalmology: AI that can build AI",
       "authors": ["C. O'Byrne", "A. Abbas", "E. Korot", "P.A. Keane"],
       "year": 2021,
       "relevance": "Directly relevant review of code-free/automated deep learning approaches in ophthalmology, establishing the CFDL concept evaluated in this paper."
     },
     {
       "title": "Automated machine learning: review of the state-of-the-art and opportunities for healthcare",
       "authors": ["J. Waring", "C. Lindvall", "R. Umeton"],
       "year": 2020,
       "relevance": "Review of automated machine learning (AutoML) in healthcare, establishing the broader context for code-free AI development tools."
     },
     {
       "title": "Code-free deep learning for multi-modality medical image classification",
       "authors": ["E. Korot", "Z. Guan", "D. Ferraz"],
       "year": 2021,
       "relevance": "Demonstrates code-free deep learning can process large image datasets cheaply (35,000 images for <$100), establishing cost-effectiveness claims for CFDL."
     },
     {
       "title": "Evaluating an automated machine learning model that predicts visual acuity outcomes in patients with neovascular age-related macular degeneration",
       "authors": ["A. Abbas", "C. O'Byrne", "D.J. Fu"],
       "year": 2022,
       "relevance": "Empirical evaluation of AutoML model performance in ophthalmology, directly relevant to the survey's scope of comparing automated vs bespoke approaches."
     },
     {
       "title": "Development and international validation of custom-engineered and code-free deep-learning models for detection of plus disease in retinopathy of prematurity: a retrospective study",
       "authors": ["S.K. Wagner", "B. Liefers", "M. Radia"],
       "year": 2023,
       "relevance": "International validation study comparing code-free and custom DL models, providing evidence on CFDL matching bespoke DL performance."
     },
     {
       "title": "Democratizing artificial intelligence: how no-code AI can leverage machine learning operations",
       "authors": ["L. Sundberg", "J. Holmström"],
       "year": 2023,
       "relevance": "Examines the democratization of AI through no-code platforms, relevant to the broader theme of making AI accessible to non-programmers."
     },
     {
       "title": "The importance of being external. Methodological insights for the external validation of machine learning models in medicine",
       "authors": ["F. Cabitza", "A. Campagner", "F. Soares"],
       "year": 2021,
       "relevance": "Discusses methodology for external validation of ML models in medicine, directly relevant to the paper's emphasis on validation gaps in CFDL studies."
     },
     {
       "title": "Cost-effectiveness of artificial intelligence as a decision-support system applied to the detection and grading of melanoma, dental caries, and diabetic retinopathy",
       "authors": ["J. Gomez Rossi", "N. Rojas-Perilla", "J. Krois", "F. Schwendicke"],
       "year": 2022,
       "relevance": "Cost-effectiveness analysis of AI decision-support systems relevant to the paper's argument that CFDL evaluations should consider cost-effectiveness."
     }
   ]
 }