ai-research-survey

scan.json (25655B)

---

 {
   "paper": {
     "title": "Synergizing Human Expertise and AI Efficiency with Language Model for Microscopy Operation and Automated Experiment Design",
     "authors": [
       "Yongtao Liu",
       "Marti Checa",
       "Rama K. Vasudevan"
     ],
     "year": 2024,
     "venue": "Machine Learning: Science and Technology",
     "arxiv_id": "2401.13803",
     "doi": "10.1088/2632-2153/ad52e9"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "ChatGPT4, when fed an API guideline document (AEcroscoPy), can convert natural-language experimental descriptions into executable Python scripts for scanning probe microscopy operations, including multi-step workflows and custom scan trajectories. The LLM can also summarize experiment logs and assist in reproducing experiments from literature. However, the paper finds that GPT4's scientific interpretation capability is very limited, and it cannot design advanced experiments from domain literature without significant human guidance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the AEcroscoPy package (ref 43) but does not provide a repository URL or archive for the code, prompts, or scripts used in this study. The ChatGPT4 conversation logs and generated scripts are shown only in figures and supplementary materials within the paper itself."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The microscopy data (BEPFM HDF files, PFM images) and ChatGPT4 conversation logs used in the study are not made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper does not list library versions, Python version, or any dependency information for reproducing the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. A reader would need to guess how to set up the ChatGPT4 sessions, what guideline document to feed, and how to replicate the specific queries."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports no quantitative metrics at all. All evaluations of LLM performance are qualitative descriptions (e.g., 'correctly suggested,' 'successfully accomplished'). No numerical results with uncertainty are provided."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no quantitative comparative claims between systems. All evaluations are qualitative demonstrations of a single system (ChatGPT4 + AEcroscoPy), so significance tests are structurally inapplicable."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes or quantitative measurements of performance are reported. The paper could have measured success rates, time savings, or code correctness rates but did not."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests a small number of tasks (approximately 8-10 distinct queries across different categories) with no justification for why this number is sufficient to support its claims about LLM utility."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No repeated trials are conducted. Each task is tested once. There is no assessment of whether the LLM would produce consistent outputs across multiple runs of the same query."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper does not compare ChatGPT4 against other LLMs, against non-LLM code generation approaches, or against human-only programming time. The evaluation consists solely of the authors' subjective assessment of ChatGPT4 outputs."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines of any kind are included, so contemporariness cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system is a single component (ChatGPT4 prompted with API documentation). There are no modular components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are used at all. Evaluation is entirely qualitative — the authors inspect generated code and describe whether it is 'correct,' 'insightful,' or 'reasonable.'"
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "While the authors informally assess the LLM outputs, there is no formal human evaluation with defined criteria, inter-rater reliability, or structured assessment protocol."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no test set concept. Tasks are ad-hoc demonstrations chosen by the authors."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes results by task category: individual command conversion, multifaceted workflow programming, experiment summarization/reproduction, data analysis, and scientific interpretation. Each category is discussed separately with distinct examples."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several failure cases: the spiral scan with DC voltage where AE-GPT 'erroneously includes the tip voltage parameters directly into the spiral function'; the 'Necessity of Science-LLM for Scientific Interpretation' section noting that 'AE-GPT's ability in scientific interpretation is very limited'; and Supplemental Note-II showing designed experiments 'do not harness knowledge from the provided literatures.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT4 'suffers from inability to extend beyond basic analyses or more in-depth technical experimental design,' that scientific interpretation is 'mostly just the definition of certain scientific concept,' and that advanced experiment design from literature fails. These are genuine negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are hedged and qualitatively supported: 'LLM can be especially useful in converting ideations' is demonstrated through several examples, 'GPT4 is capable of analyzing microscopy images in a generic sense' is shown in Figure 6, and 'GPT4 suffers from inability to extend beyond basic analyses' is discussed in the Necessity of Science-LLM section. The claims are appropriately tentative."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the integration 'enables more efficient and reproducible experiments' and 'can potentially significantly increase the pace of scientific research.' These causal claims about efficiency and acceleration are not supported by any controlled comparison or quantitative evidence — only by qualitative demonstrations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests ChatGPT4 on one custom API (AEcroscoPy) and two commercial APIs for SPM, but the title and conclusions generalize to 'Microscopy Operation and Automated Experiment Design' broadly. The conclusions claim the synergy 'will open new door for accelerating scientific research' — far beyond the SPM domain tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the paper does not consider whether the tasks succeeded because GPT4's training data included similar API documentation and code examples, or whether the success is specific to well-documented APIs with Python interfaces."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures whether GPT4 generates seemingly correct code snippets and uses this as evidence that LLMs 'enable more efficient and reproducible experiments' and 'accelerate scientific research.' The gap between generating code that looks correct in a conversational setting and actually accelerating real research workflows is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'ChatGPT4' throughout without specifying an exact model version, snapshot date, or API version (e.g., gpt-4-0613). Model behavior varies across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The actual prompts (user queries) and ChatGPT4 responses are shown in Figures 2, 3, S1-S9, and supplemental notes. The user inputs are the actual text used, not just natural-language descriptions of what was asked."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, or other ChatGPT4 API settings are mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is straightforward in-context learning: feeding an API guideline document into ChatGPT4 and then issuing queries. There is no tool use, retry logic, memory management, or multi-step agent architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the AEcroscoPy guideline was 'fed' into ChatGPT4 but does not document how (via system prompt, file upload, or pasting), whether the full guideline or excerpts were used, or how the microscopy data (HDF files) were prepared for ChatGPT4 analysis."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The 'Necessity of Science-LLM for Scientific Interpretation' section functions as a limitations discussion, detailing that AE-GPT's scientific interpretation is 'very limited' and that it lacks the ability to design advanced experiments from literature. The conclusions further note that 'experimental design capabilities of AE-GPT are generic, and the data analysis especially the interpretation of scientific data is basic.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not consider issues such as whether outputs might look correct but contain subtle errors, whether results are reproducible across ChatGPT4 sessions, or whether the authors' domain expertise biased their assessment of code correctness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what it is NOT claiming (e.g., that this does not prove LLMs can replace expert microscopists, or that results are specific to ChatGPT4 and may not hold for other LLMs). The conclusions trend toward broad generalizations rather than bounded claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. ChatGPT4 conversation logs, generated scripts, microscopy data files, and the AEcroscoPy guideline document used are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the process: the AEcroscoPy guideline was fed to ChatGPT4 as a learning resource, then specific queries were issued for each task category (command conversion, workflow programming, experiment reproduction, data analysis). The process for each task is narrated in the text."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates an LLM on author-designed tasks, not a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from feeding the guideline to ChatGPT4 through to evaluating outputs is not formally documented. It is unclear how the evaluation tasks were selected, whether all queries are reported (or only successful ones), and how code correctness was verified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgements: 'This research was supported by the Center for Nanophase Materials Sciences (CNMS), which is a US Department of Energy, Office of Science User Facility at Oak Ridge National Laboratory.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with the Center for Nanophase Materials Sciences, Oak Ridge National Laboratory, which is clearly stated. The authors developed AEcroscoPy, the custom API being used, and this is disclosed via references."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (US DOE) has no financial interest in whether ChatGPT4 works well for microscopy automation. The funding supports the research facility broadly, not a specific outcome regarding LLM utility."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes an explicit conflict of interest statement: 'The authors declare no conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It tests ChatGPT4's ability to generate code for a specific API given its documentation, which is a code generation task rather than a benchmark evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The tasks are novel, author-designed queries about a custom API (AEcroscoPy), not standardized benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The study is a qualitative demonstration of LLM utility with custom tasks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported — no API costs, tokens consumed, or time per query are mentioned despite using a commercial LLM (ChatGPT4) for all experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total cost of ChatGPT4 usage for the experiments is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs (ChatGPT4) can convert human-language experimental workflow descriptions into executable Python code for microscope APIs.",
       "evidence": "Demonstrated through multiple examples: individual command conversion (Figures 3, S1), BE line scan workflow (Figure 3a), custom trajectory design (Figure 3b-c), and multifaceted workflow programming (Figure 4, Figures S3-S5). The generated code uses correct API functions in proper sequence.",
       "supported": "moderate"
     },
     {
       "claim": "AE-GPT can summarize experiment details from logger files and reproduce experiments from both logs and published literature.",
       "evidence": "Demonstrated in Figure 5: AE-GPT translates a logger file to natural language (Figure 5a) and extracts BE-PFM parameters from a published paper to generate reproduction code (Figure 5b, Figures S7-S8).",
       "supported": "moderate"
     },
     {
       "claim": "GPT4 can analyze BEPFM data including identifying parameters, plotting results, finding maximum intensity spectra, and calculating roughness.",
       "evidence": "Shown in Figure 6 and Figure S9 where AE-GPT processes HDF data files and performs basic analyses upon request.",
       "supported": "weak"
     },
     {
       "claim": "GPT4's scientific interpretation ability is very limited and it cannot design advanced experiments from literature.",
       "evidence": "Discussed in 'Necessity of Science-LLM for Scientific Interpretation' section and Supplemental Notes I and II: 'The interpretation is mostly just the definition of certain scientific concept' and 'the designed experiments do not harness knowledge from the provided literatures.'",
       "supported": "moderate"
     },
     {
       "claim": "The integration of LLMs and APIs 'can potentially significantly increase the pace of scientific research.'",
       "evidence": "No quantitative evidence provided. Supported only by the qualitative demonstrations of code generation, which show ChatGPT4 can produce functional code snippets but do not measure actual research acceleration.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "The entire paper relies on qualitative assessment by the authors. No success rates, time comparisons, code correctness metrics, or any numerical evaluation is provided. The reader must trust the authors' subjective judgment that outputs are 'correct' or 'insightful.'"
     },
     {
       "flag": "Cherry-picked demonstrations",
       "detail": "It is unclear whether all queries attempted are reported. The paper may show only successful examples while omitting failures. The selection of test tasks appears ad-hoc with no systematic sampling strategy."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The paper's claims about 'accelerating scientific research,' 'enabling effective sharing of experimental protocols,' and serving as 'FAIR infrastructure' are far-reaching relative to the evidence, which consists of a handful of qualitative code generation demonstrations on one custom and two commercial APIs."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors developed AEcroscoPy and are evaluating ChatGPT4's ability to use their own API. They are uniquely positioned to judge code correctness but also uniquely biased — they may unconsciously rate outputs more favorably or design queries that align with what ChatGPT4 can do."
     },
     {
       "flag": "No reproducibility of LLM outputs",
       "detail": "ChatGPT4 outputs are non-deterministic. No temperature settings are reported, no repeated trials are conducted, and no analysis of output consistency is performed. The specific model version is not stated, making future reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM model used throughout this study for code generation and scientific analysis tasks."
     },
     {
       "title": "Natural language processing models that automate programming will transform chemistry research and teaching",
       "authors": ["G. M. Hocky", "A. D. White"],
       "year": 2022,
       "relevance": "Early argument for LLMs automating scientific programming, directly relevant to the survey's scope on LLM-assisted code generation for research."
     },
     {
       "title": "Opportunities for Retrieval and Tool Augmented Large Language Models in Scientific Facilities",
       "authors": ["M. H. Prince"],
       "year": 2023,
       "arxiv_id": "2312.01291",
       "relevance": "Directly relevant work on LLM tool augmentation for scientific instrumentation, closely related to the agentic AI workflow theme."
     },
     {
       "title": "14 examples of how LLMs can transform materials science and chemistry: a reflection on a large language model hackathon",
       "authors": ["K. M. Jablonka"],
       "year": 2023,
       "relevance": "Explores diverse LLM applications in materials science including property prediction and code generation, relevant to LLM capability assessment."
     },
     {
       "title": "Summary of chatgpt-related research and perspective towards the future of large language models",
       "authors": ["Y. Liu"],
       "year": 2023,
       "relevance": "Survey of ChatGPT research applications relevant to understanding the landscape of LLM capability claims."
     },
     {
       "title": "AEcroscoPy: A software-hardware framework empowering microscopy toward automated and autonomous experimentation",
       "authors": ["Y. Liu"],
       "year": 2023,
       "arxiv_id": "2312.10281",
       "relevance": "The custom API framework used in this study, relevant as an example of programmatic scientific instrument control that LLMs interface with."
     },
     {
       "title": "Galactica: A large language model for science",
       "authors": ["R. Taylor"],
       "year": 2022,
       "arxiv_id": "2211.09085",
       "relevance": "Domain-specific scientific LLM relevant to the paper's argument that fine-tuned models would outperform general LLMs for scientific tasks."
     },
     {
       "title": "SciBERT: A pretrained language model for scientific text",
       "authors": ["I. Beltagy", "K. Lo", "A. Cohan"],
       "year": 2019,
       "arxiv_id": "1903.10676",
       "relevance": "Domain-specific scientific language model, cited as evidence that specialized LLMs could address limitations found in general-purpose models for scientific interpretation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Researchers operating scanning probe microscopes could adopt this approach of feeding API documentation to ChatGPT4 for workflow generation, though it requires specific hardware and API access."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the widely expected finding that LLMs can generate code from documentation and help with API usage, with no surprising or contrarian elements."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or safety concerns are raised; the paper is about assisting scientific instrumentation."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward demonstration paper."
     },
     "demo_ability": {
       "score": 1,
       "justification": "AEcroscoPy is published as a package but the LLM integration has no standalone demo; reproducing requires physical microscopy equipment."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses ChatGPT4 (well-known) but the lab (ORNL), venue (ML:ST), and application domain (SPM) are niche."
     }
   }
 }