ai-research-survey

scan.json (21485B)

---

 {
   "paper": {
     "title": "Cocreating an Automated mHealth Apps Systematic Review Process With Generative AI: Design Science Research Approach",
     "authors": ["Guido Giunti", "Colin P Doherty"],
     "year": 2024,
     "venue": "JMIR Medical Education",
     "doi": "10.2196/48949"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is referenced: https://github.com/guidogiunti/ChatGPT-SR-script (reference [32]). The working script version is stated to be available there."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release the CSV output files, the keywords.csv file, or the comparison data from prior systematic reviews. Only the script itself is on GitHub."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python and several libraries (requests, bs4) but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The cocreation process is described narratively, but there is no reproducibility section explaining how to replicate the study."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study / DSR paper with no quantitative experimental results that would require confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no comparative claims based on quantitative data that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effects are measured. The paper is a qualitative feasibility study."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a single-case DSR study, not a quantitative experiment with a sample requiring justification."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No repeated experimental runs are conducted. The cocreation process was performed once by a single author."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares the ChatGPT-generated script output to the output from the prior 42matters-based script used in background studies. The Evaluation section notes differences in the CSV output columns."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baseline is the authors' own prior script using 42matters, described as 'no longer functional.' No comparison to contemporary AI-assisted systematic review tools or alternative LLM approaches is made."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system is a single cocreation process without separable components that could be ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation mentions time (4 hours 39 minutes) and a qualitative comparison of output columns, but no formal metrics are defined or reported systematically."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Although the authors qualitatively assessed the output, there is no structured human evaluation with defined criteria, multiple evaluators, or inter-rater reliability."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation. There is no test set to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation does not break down results by category (e.g., types of errors, types of apps found). Only aggregate observations are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple failures: code errors (ModuleNotFoundError), timeout issues, PubMed search problems, missing metadata columns, and the script's reliance on exact title matching for PubMed. The output was deemed 'unsuitable as a final output.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the ChatGPT-generated script produced an output that was 'useful as an intermediate outcome but was deemed unsuitable as a final output,' and that the Medical Evidence column only contained 'Yes' or 'No' based on title matching, missing important metadata."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the study 'demonstrates the potential of using generative AI to automate the time-consuming process' and notes limitations. The paper provides a narrative account supporting these hedged claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'this approach could be particularly useful for researchers with limited coding skills' and implies the cocreation process led to a functional script. These are causal-adjacent claims, but the study design (single case, single user, no controls) is inadequate for causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusions generalize to 'researchers with limited coding skills' and 'the time-consuming process of conducting systematic reviews' broadly, but the study tested only one researcher, one topic (MS mHealth apps), one AI (ChatGPT 3.5), and one platform (Google Play Store). The title says 'automated mHealth apps systematic review process' which overstates what was achieved."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, the author's actual coding experience (rated 4-8 across various skills) may have substantially aided the process, but this is not examined as a confound."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'ChatGPT 3.5, as of June 2023' but does not specify the exact model version (e.g., gpt-3.5-turbo-0613). 'ChatGPT 3.5' is a marketing name, not a versioned API model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes several actual prompts used during the cocreation process, including the triggering prompt (full text quoted) and several follow-up prompts. While not every prompt is included, the key prompts are provided verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters are reported (temperature, top-p, max tokens). The interaction appears to have been through the ChatGPT web interface, but no settings are mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The interaction is a simple conversational exchange through the ChatGPT web interface."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not clearly document how the background studies' data was preprocessed for comparison, or how the keywords.csv file was constructed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present, discussing DSR methodology limitations, subjectivity bias, and data quality issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations are mostly generic to DSR methodology ('subjectivity is a common bias present in DSR'). The most specific limitation mentioned is about the quality of search results from training data, but this is still relatively vague. No threats specific to this particular study's findings are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no explicit scope boundaries like 'this does not demonstrate that non-coders could achieve the same results' or 'this does not apply to systematic reviews beyond mHealth apps.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The full chat transcript with ChatGPT is not available. The CSV output files are not released. Only the final script is on GitHub. The reader cannot independently verify the cocreation process."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection process (cocreation with ChatGPT) is described in the Results section with specific prompts, responses, and the iterative debugging process. The background studies used as comparison points are also referenced."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study involved a single author interacting with ChatGPT."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from ChatGPT interaction to final output is described narratively but lacks detail on intermediate steps. For example, how many iterations occurred total, how the comparison to prior study outputs was structured, and what specific criteria were used for the evaluation are not fully documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgments section: EU Horizon 2020 Marie Skłodowska-Curie grant (101034252), Science Foundation Ireland grant (16/RC/3948), and Business Finland's More Stamina Research to Business project."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Trinity College Dublin, University of Oulu, FutureNeuro SFI Research Centre, and St James Hospital."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (EU Horizon 2020, SFI, Business Finland) are public research funding agencies that do not have a financial interest in whether ChatGPT is useful for systematic reviews."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "A 'Conflicts of Interest' section is present and states 'None declared.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It uses ChatGPT as a code generation assistant in a qualitative case study."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The paper is a qualitative case study of code cocreation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used. This is a qualitative case study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The paper explicitly states 'No ethics board review is needed as the work does not use patient data or involve human participants.'"
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethical Considerations section explicitly states ethics review is not needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The single author who conducted the cocreation is described via Table 1 (skills background), but this is not a participant demographics report."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions to randomize."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports the total time (4 hours 39 minutes) but does not report any monetary cost, token usage, or number of API calls/messages exchanged with ChatGPT."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The hardware used, API costs, or total compute resources are not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Generative AI (ChatGPT 3.5) can be used to cocreate an automated script for systematic reviews of mHealth apps.",
       "evidence": "The Results section describes the full cocreation process, resulting in a working Python script that crawls the Google Play Store and cross-references with PubMed. The script is available on GitHub (reference [32]).",
       "supported": "moderate"
     },
     {
       "claim": "The cocreation process took 4 hours and 39 minutes total.",
       "evidence": "Stated in the Evaluation section and in the Highlights: 'The overall cocreation process exercise had a total duration of 4 hours and 39 minutes.'",
       "supported": "moderate"
     },
     {
       "claim": "No knowledge of Python scripting was required by the author.",
       "evidence": "Listed in Highlights section. However, Table 1 shows the author has intermediate-to-advanced skills in multiple programming-adjacent domains (JavaScript beginner, PHP beginner, HTML5 intermediate), undermining this claim. The process was conducted 'as if no coding skill was present,' but the author rated himself 4-7 on relevant coding skills.",
       "supported": "weak"
     },
     {
       "claim": "The output was useful as an intermediate outcome but insufficient to replace manual systematic review steps.",
       "evidence": "The Evaluation section states the CSV lacked metadata columns and the Medical Evidence column used only title matching in PubMed. The output 'was deemed unsuitable as a final output.'",
       "supported": "strong"
     },
     {
       "claim": "This approach could be particularly useful for researchers with limited coding skills.",
       "evidence": "Stated in the abstract and conclusions, but supported only by a single case study involving a researcher who actually has some coding skills. No evidence from actual non-coders is presented.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "A physician used ChatGPT 3.5 to co-create a Python script that automates part of a systematic review of mHealth apps, completing the process in about 4.5 hours. The resulting script could crawl the Google Play Store and cross-reference apps with PubMed, but the output was deemed insufficient as a final product, requiring further refinement. The paper provides a useful narrative account of the cocreation process including error handling and debugging, but the evaluation is minimal and the generalizability claims are not well-supported by a single-case study.",
   "red_flags": [
     {
       "flag": "N=1 case study with broad generalization claims",
       "detail": "The entire study is based on a single researcher's interaction with a single AI tool on a single topic. The conclusions generalize to 'researchers with limited coding skills' and systematic reviews broadly, far exceeding what a single case study can support."
     },
     {
       "flag": "Author's coding skills undermine 'no coding required' claim",
       "detail": "Table 1 shows the author rates himself 4-8 on various programming-related skills. The study was conducted 'as if no coding skill was present,' but the author's actual experience likely influenced prompting quality and debugging ability. This is not examined as a confound."
     },
     {
       "flag": "Minimal evaluation with no formal metrics",
       "detail": "The evaluation consists of a qualitative comparison to a prior script's output with no formal metrics, no precision/recall analysis of the app search results, and no comparison to alternative approaches."
     },
     {
       "flag": "No chat transcript or detailed interaction log released",
       "detail": "While some prompts are included in the paper, the full interaction log with ChatGPT is not available, making it impossible to fully verify or reproduce the cocreation process."
     }
   ],
   "cited_papers": [
     {
       "title": "Better together? An evaluation of AI-supported code translation",
       "authors": ["J Weisz", "M Muller", "S Ross", "F Martinez", "S Houde", "M Agarwal"],
       "year": 2022,
       "doi": "10.1145/3490099.3511157",
       "relevance": "Evaluates AI-supported code translation, relevant to understanding AI-assisted programming productivity."
     },
     {
       "title": "ChatGPT is fun, but not an author",
       "authors": ["HH Thorp"],
       "year": 2023,
       "doi": "10.1126/science.adg7879",
       "relevance": "Discusses ethical implications of generative AI in scientific authorship, relevant to AI in research methodology."
     },
     {
       "title": "How does ChatGPT perform on the United States Medical Licensing Examination? The implications of large language models for medical education and knowledge assessment",
       "authors": ["A Gilson", "CW Safranek", "T Huang"],
       "year": 2023,
       "doi": "10.2196/45312",
       "relevance": "Evaluates ChatGPT's capabilities on medical exams, relevant to LLM capability assessment methodology."
     },
     {
       "title": "Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models",
       "authors": ["TH Kung", "M Cheatham", "A Medenilla"],
       "year": 2023,
       "doi": "10.1371/journal.pdig.0000198",
       "relevance": "Another evaluation of ChatGPT on medical licensing exams, relevant to LLM benchmark evaluation methodology."
     },
     {
       "title": "The role of ChatGPT, generative language models, and artificial intelligence in medical education: a conversation with ChatGPT",
       "authors": ["G Eysenbach"],
       "year": 2023,
       "doi": "10.2196/46885",
       "relevance": "Explores generative AI's role in medical education, relevant to understanding AI-assisted processes in research."
     },
     {
       "title": "A full systematic review was completed in 2 weeks using automation tools: a case study",
       "authors": ["J Clark", "P Glasziou", "C Del Mar", "A Bannach-Brown", "P Stehlik", "AM Scott"],
       "year": 2020,
       "doi": "10.1016/j.jclinepi.2020.01.008",
       "relevance": "Prior work on automating systematic reviews, directly relevant to AI-assisted research methodology."
     },
     {
       "title": "Reducing systematic review burden using Deduklick: a novel, automated, reliable, and explainable deduplication algorithm to foster medical research",
       "authors": ["N Borissov", "Q Haas", "B Minder"],
       "year": 2022,
       "doi": "10.1186/s13643-022-02045-9",
       "relevance": "AI-based automation of systematic review deduplication, relevant to AI-assisted research tools."
     }
   ]
 }