ai-research-survey

scan.json (22025B)

---

 {
   "paper": {
     "title": "AI for NONMEM Coding in Pharmacometrics Research and Education: Shortcut or Pitfall?",
     "authors": [
       "Wenhao Zheng",
       "Wanbing Wang",
       "Carl M.J. Kirkpatrick",
       "Cornelia B. Landersdorfer",
       "Huaxiu Yao",
       "Jiawei Zhou"
     ],
     "year": 2025,
     "venue": "CPT: Pharmacometrics & Systems Pharmacology",
     "arxiv_id": "2507.08144",
     "doi": "10.1002/psp4.70125"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/zhoujw14/AI_PMx stated as 'Supplementary Materials can be found https://github.com/zhoujw14/AI_PMx'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The 13 NONMEM coding tasks (prompts) and all AI agent responses are provided in the Supplementary Materials via the GitHub repository. The tasks themselves constitute the evaluation data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The paper does not describe API versions, library versions, or any reproducibility setup beyond naming the models used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The supplementary materials contain prompts and responses but there is no README or instructions describing how to replicate the evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports median and IQR for scores across 13 tasks but does not provide confidence intervals or error bars. IQR describes data spread but is not a measure of uncertainty around an estimate."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims o1 'demonstrated the best overall performance' and that the optimized prompt 'enhanced coding accuracy across all AI agents' but provides no statistical significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper compares median scores but does not provide standardized effect sizes or percentage improvements with baseline context for the optimized prompt improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 13 tasks is not justified. There is no discussion of whether 13 tasks is sufficient to draw reliable conclusions about AI agent performance, no power analysis, and no acknowledgment that the small number limits statistical inference."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. The paper reports medians and IQR across the 13 tasks but these reflect task-level variation in a single run per model-task pair. Each model-task combination appears to be run once with no repeated runs, so there is no measure of run-to-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven AI agents are compared against each other, and the original prompt serves as a baseline for the optimized prompt condition. Prior work (refs 5-7) evaluating ChatGPT and Gemini is referenced as comparison context."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o1, o3-mini) are all recent OpenAI models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The optimized prompt is compared to the original prompt, but there is no ablation study examining which components of the optimized prompt contribute to improvement. The prompt embeds multiple rubric criteria and no individual component analysis is performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: the 3-level scoring rubric with a maximum of 6 points. No additional metrics such as syntactic validity, compilation success rate, or execution success are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The scoring rubric is applied through expert evaluation. The rubric was 'refined and validated through expert consensus' and the codes are scored by domain experts against the rubric criteria."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation of tasks into development and test sets. The same 13 tasks are used both for developing the rubric and evaluating final performance. The rubric was 'developed iteratively using several preliminary test tasks' but the final evaluation uses all 13 tasks without a held-out split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows per-task performance for each AI agent comparing original vs. optimized prompt. Individual task scores are available, not just aggregated averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Impact Assessment section discusses specific failure modes: 'AI agents may fail to implement differential equations correctly or misapply model compartments. They may also struggle with proper initialization of model parameters or aligning dosing/event records in the dataset.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that several models performed poorly (e.g., gpt-4.1-nano and gpt-4o-mini had lower scores), and notes limitations of AI agents for complex models. The paper honestly states that 'current AI agents show limitations' for complex modeling tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims o1 and gpt-4.1 achieved the best performance and perfect scores with the optimized prompt. These claims are supported by the results in Figure 1 and the reported scores (median 5, IQR 2.5-6 for o1; perfect 6.0 with optimized prompt for both)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about the optimized prompt improving performance ('significantly improved code accuracy', 'enhanced coding accuracy across all AI agents'). However, the study design does not control for confounds such as prompt length, specificity, or order effects, and each condition is run only once per model-task pair."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame the work broadly ('AI for NONMEM Coding') but only 7 OpenAI models are tested. No non-OpenAI models (Claude, Gemini, Llama, etc.) are evaluated despite being commonly available. The paper does not explicitly bound its conclusions to OpenAI models only."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed performance differences. For example, the improvement from the optimized prompt could be due to prompt length, added domain context, or specific formatting rather than the rubric criteria. No such alternatives are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists model marketing names (gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o1, o3-mini) but does not specify exact API versions, snapshot dates, or version identifiers (e.g., 'gpt-4.1-2025-04-14'). Marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that 'Responses generated by all seven AI agents, as well as their responses using the optimized prompt, are provided in the Supplementary Materials' at the GitHub repository. The optimized prompt is explicitly said to be 'provided in the Supplementary Materials.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters such as temperature, top-p, max tokens, or other API settings are reported. These significantly affect LLM output quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper sends single prompts to AI models and evaluates the responses directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The 13 tasks are clearly enumerated in Table 1 with exact task descriptions. The scoring rubric development process is described: 'developed iteratively using several preliminary test tasks and was informed by common AI-generated NONMEM coding errors... refined and validated through expert consensus.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper acknowledges some limitations of AI agents in the Impact Assessment section but does not discuss the study's own methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity of this study are discussed. The paper discusses limitations of AI agents themselves but not limitations of the evaluation methodology (e.g., small task set, single vendor, no inter-rater reliability for rubric scoring, single-run evaluation)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its conclusions to OpenAI models, does not acknowledge that 13 tasks may be insufficient for generalizable claims, and does not state that results may not transfer to other LLM providers."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/zhoujw14/AI_PMx) contains the AI-generated responses for all tasks, which constitutes the raw data of this study."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: 13 coding tasks were defined (Table 1), each sent to 7 AI agents, with both original and optimized prompts. The scoring rubric (Table 2) is provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study evaluates AI model outputs on defined tasks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from collecting AI responses to scoring them is not fully documented. It is unclear who scored the responses, whether multiple scorers were used, or how disagreements (if any) were resolved. The scoring process lacks transparency."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'This work is funded by a PharmAlliance Early Career Research Award and University of North Carolina at Chapel Hill.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed. The authors are from UNC Chapel Hill and Monash University. They are evaluating OpenAI products and have no apparent affiliation with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (PharmAlliance and UNC Chapel Hill) are academic institutions with no apparent financial interest in whether OpenAI models perform well at NONMEM coding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'The authors report no conflicts of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 7 models evaluated. This is relevant because NONMEM coding examples may exist in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 13 NONMEM coding tasks or similar tasks appeared in the models' training data. NONMEM coding examples are publicly available online and could be in the training sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of benchmark contamination. The NONMEM coding patterns tested are standard pharmacometrics tasks that likely exist in training corpora, but this is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The study evaluates AI model outputs on coding tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or latency are reported despite calling 7 different OpenAI models across 13 tasks with two prompt conditions (182 API calls minimum)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or total API spend is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "OpenAI o1 demonstrated the best overall performance among seven AI agents, with a mean score of 5 (IQR 2.5-6) using the original prompt and perfect 6.0 on all 13 tasks with the optimized prompt.",
       "evidence": "Results section and Figure 1 show o1's median and IQR scores. With the optimized prompt, o1 achieved 6.0 on all 13 tasks.",
       "supported": "moderate"
     },
     {
       "claim": "gpt-4.1 ranked second, scoring 5 (IQR 2.5-5) with the original prompt and also achieving a perfect 6.0 on all 13 tasks with the optimized prompt.",
       "evidence": "Results section and Figure 1. gpt-4.1 matched o1's perfect performance with the optimized prompt.",
       "supported": "moderate"
     },
     {
       "claim": "The optimized prompt enhanced coding accuracy across all AI agents, with greater improvement seen in more complex models.",
       "evidence": "Results section and Figure 2 show task-specific improvements. Indirect response and PK models with absorption lag time showed greater improvement.",
       "supported": "weak"
     },
     {
       "claim": "AI agents performed well in writing basic NONMEM model structures, providing a useful foundation for pharmacometrics model coding.",
       "evidence": "Results section: high scores on simpler tasks (e.g., one-compartment models). Impact Assessment acknowledges limitations for complex models.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The study benchmarked 7 OpenAI models on 13 NONMEM pharmacometrics coding tasks using a 3-level scoring rubric (max 6 points). OpenAI o1 and gpt-4.1 achieved the best performance, both reaching perfect scores on all tasks when given an optimized prompt embedding the rubric criteria. AI agents performed well on basic model structures but struggled with complex models involving differential equations, parameter initialization, and dataset alignment. The paper argues AI can support pharmacometrics education but warns against overreliance without expert oversight.",
   "red_flags": [
     {
       "flag": "Single vendor evaluation",
       "detail": "All 7 models evaluated are from OpenAI. Despite the broad framing of 'AI for NONMEM Coding', no models from Anthropic, Google, Meta, or other providers are tested, severely limiting generalizability claims."
     },
     {
       "flag": "No inter-rater reliability for rubric scoring",
       "detail": "The scoring rubric is applied by experts but there is no mention of multiple scorers, inter-rater agreement, or blinding. It is unclear whether a single person scored all outputs or whether consistency was verified."
     },
     {
       "flag": "Single-run evaluation per model-task pair",
       "detail": "Each model appears to be run once per task. LLM outputs are stochastic and can vary significantly across runs. No repeated sampling or temperature control is mentioned."
     },
     {
       "flag": "Very small task set",
       "detail": "Only 13 coding tasks are used to evaluate performance. This is too few for robust statistical conclusions about model ranking, especially given that all results rely on this small sample."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Claims about which model is 'best' and that the optimized prompt 'significantly improved' accuracy are made without any statistical tests on a sample of 13 tasks."
     },
     {
       "flag": "Missing hyperparameter reporting",
       "detail": "Temperature, top-p, and other sampling parameters are not reported. These critically affect output quality and reproducibility. Without this information, results cannot be reliably replicated."
     }
   ],
   "cited_papers": [
     {
       "title": "On the trustworthiness of generative foundation models: Guideline, assessment, and perspective",
       "authors": ["Y. Huang"],
       "year": 2025,
       "arxiv_id": "2502.14296",
       "relevance": "Provides guidelines for assessing trustworthiness of generative AI models, directly relevant to AI evaluation methodology."
     },
     {
       "title": "Agents for Change: Artificial Intelligent Workflows for Quantitative Clinical Pharmacology and Translational Sciences",
       "authors": ["M.H. Shahin", "S. Goswami", "S. Lobentanzer", "B.W. Corrigan"],
       "year": 2025,
       "relevance": "Discusses AI agent workflows in pharmacology, relevant to agentic AI applications in specialized scientific domains."
     },
     {
       "title": "Leveraging large language models in pharmacometrics: evaluation of NONMEM output interpretation and simulation capabilities",
       "authors": ["H.J. Cha", "K. Choe", "E. Shin", "M. Ramanathan", "S. Han"],
       "year": 2025,
       "relevance": "Directly evaluates LLMs for pharmacometrics tasks including NONMEM, a closely related benchmark study."
     },
     {
       "title": "ChatGPT in pharmacometrics? Potential opportunities and limitations",
       "authors": ["M.E. Cloesmeijer", "A. Janssen", "S.F. Koopman", "M.H. Cnossen", "R.A. Math\u00f4t"],
       "year": 2024,
       "relevance": "Early evaluation of ChatGPT for pharmacometrics, establishing baseline for LLM coding capability in this domain."
     },
     {
       "title": "Evaluation of ChatGPT and Gemini large language models for pharmacometrics with NONMEM",
       "authors": ["E. Shin", "Y. Yu", "R.R. Bies", "M. Ramanathan"],
       "year": 2024,
       "relevance": "Evaluates ChatGPT and Gemini for NONMEM coding, directly comparable benchmark study with different models."
     },
     {
       "title": "Check My Work?: Measuring Sycophancy in a Simulated Educational Context",
       "authors": ["C. Arvin"],
       "year": 2025,
       "arxiv_id": "2506.10297",
       "relevance": "Relevant to the paper's discussion of AI in education and the risks of sycophantic AI behavior in learning contexts."
     }
   ]
 }