ai-research-survey

scan.json (28281B)

---

 {
   "paper": {
     "title": "Picking the Right Specialist: Attentive Neural Process-based Selection of Task-Specialized Models as Tools for Agentic Healthcare Systems",
     "authors": ["Pramit Saha", "Joshua Strong", "Mohammad Alsharid", "Divyanshu Mishra", "J. Alison Noble"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.14901"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ToolSelect, an Attentive Neural Process-based tool selector, consistently outperforms 10 SOTA routing baselines across four chest X-ray task families (disease diagnosis, report generation, visual grounding, VQA). The paper demonstrates substantial per-instance complementarity among specialist models — e.g., Oracle F1 is 58.00% on Open-I vs 13.35% for the best single model — and shows that ToolSelect closes a large fraction of this gap (43.80% F1). The paper introduces ToolSelectBench with 1448 queries and a diverse pool of 55 specialist models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or GitHub link is provided in the paper. The supplementary material includes model predictions and benchmark data but no source code for the ToolSelect method."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "§3.1: 'All benchmark samples, together with the individual predictions of 55 candidate models, are attached as supplementary material for transparency and reproducibility.' The evaluation datasets (Open-I, VinDr-CXR, ReX-VQA) are also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "§3.3 lists key architecture choices (ViT-B/16, CheXbert) and hyperparameters but provides no requirements.txt, Dockerfile, or detailed software dependency listing with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts to replicate experiments are mentioned in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-4 report only point estimates for all metrics (accuracy, F1, precision, recall, AUC, IoU, etc.). No confidence intervals, error bars, or ± notation appear anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'ToolSelect consistently outperforms' baselines based solely on comparing raw numbers across Tables 1-4. No statistical significance tests (p-values, bootstrap tests, etc.) are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "§4.1: 'improving over the best baseline by +31.82 F1 points (vs 11.98%)' and '+35.44 F1 points (vs 24.44%)'. §4.2: 'improving over Random by +12.46 points.' Absolute improvements with baseline context are reported throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 550 diagnosis, 156 report generation, 394 grounding, and 348 VQA queries. No justification for these sizes or power analysis is provided. The report generation set (156 queries) is notably small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 appear to be from single runs. No standard deviations, variance across seeds, or multiple-run spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "§3.4 describes 10+ baselines: KNNRouter, SVMRouter, MLPRouter, MFRouter, EloRouter, RouterDC, AutoMix, Hybrid LLM, GraphRouter, and CausalLLM Router. Also includes Random and Oracle bounds."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include RouterDC (Chen et al., 2024a, NeurIPS 2024), GraphRouter (Feng et al., ICLR), RouteLLM (Ong et al., 2024), and ToolACE (Liu et al., 2025). These are recent and representative."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§4.2: 'See Suppl. E.1 for ablation on reference point / aggregation.' An ablation study exists in the supplementary material, though only referenced briefly in the main text."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive multi-metric evaluation: disease diagnosis uses Acc/F1/P/R/AUC (Table 1); report generation uses SemBScore/F1-RadGraph/ROUGE-L/METEOR/RateScore (Table 2); grounding uses mAP@0.25/mAP@0.5/Mean IoU (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated. For report generation and clinical grounding tasks, human expert evaluation of clinical faithfulness would strengthen the claims, but none is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "§3.1: 'we follow the standard dataset splits: the training split is used for training the selector, and performance is reported on the corresponding test split.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-phrase performance across 11 clinical findings for visual grounding. Table 1 breaks down results by Open-I vs VinDr datasets. Results are shown separately for all 4 task families."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§4.1 discusses failure modes: 'R-All attains the highest accuracy but has the lowest F1.' §4.3 discusses 'Hard phrases and weak specialists' with specific performance details. §4.4 notes baseline routers that underperform single models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "ToolSelect outperforms all baselines in every table on every primary metric. No failed configurations, abandoned approaches, or design choices that hurt performance are reported. The supplementary ablation is referenced but no negative findings are mentioned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: (1) 'ToolSelect consistently outperforms 10 SOTA methods across four different task families' — supported by Tables 1-4; (2) introduces ToolSelectBench with 1448 queries — confirmed in §3.1. Claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited to 'ToolSelect improves over baselines' which is supported by controlled comparisons holding the specialist pool constant while varying only the selection method. The ablation study (Suppl. E.1) tests component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Agentic Healthcare Systems' broadly but all experiments are on chest X-ray only — one imaging modality, one anatomical region. The conclusion references 'multimodal clinical agentic pipelines' without bounding generalization to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for ToolSelect's gains are discussed. For instance, the ANP's advantage could stem from the behavioral reference sets rather than the attention mechanism, or compute budget differences between baselines could affect results. Neither is explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures automated metrics (F1, IoU, F1-RadGraph) but frames results as 'clinically aligned reports' (§4.2) and 'clinical performance' (§2.1). The gap between automated proxies and actual clinical utility is not acknowledged. F1-RadGraph measures structured clinical entity overlap, not clinical decision quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "§3.2 specifies model names with sizes/variants: 'CheXAgent-8B', 'Qwen3-VL-2B/4B/8B', 'LLaVA-1.5-7B/13B', 'LLaMA-3.2-Vision-11B', 'EVA-X Tiny/Small/Base', etc. These are specific enough to identify the models used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The system uses a ReAct-style LLM orchestrator (Figure 3) that maps queries to tasks. No actual prompt text is provided for the LLM agent core or the task selector. Only natural language descriptions of the pipeline are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§3.3: 'AdamW (learning rate 3 × 10−5, weight decay 10−4), up to 50 epochs with early stopping (patience = 10, min delta = 10−4). Batch size 16. λH = 0.05, dropout 0.1, reference set Bt ∈ [16, 64].' Key hyperparameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides an architecture overview of the agentic framework showing the ReAct-style loop with LangChain short-term memory and the tool model zoo. §2.1-2.3 describe the selector architecture in mathematical detail. Figure 4 provides detailed ToolSelect architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The construction of ToolSelectBench queries is insufficiently documented. §3.1 states benchmark sizes (550/156/394/348 queries) and mentions using standard splits and 1% sampling for VQA, but the query construction criteria and any filtering applied to arrive at these specific counts are not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has an 'Impact Statement' section discussing deployment concerns but no dedicated 'Limitations' or 'Threats to Validity' section with substantive discussion of methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Impact Statement contains generic statements like 'improper deployment without clinical oversight could pose risks' but no study-specific threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the scope to chest X-ray or acknowledge that generalization to other imaging modalities, clinical domains, or real-world deployment settings is untested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "§3.1: 'All benchmark samples, together with the individual predictions of 55 candidate models, are attached as supplementary material.' This allows independent verification of the routing results from raw predictions."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3.1-3.2 describe the data sources (Open-I, VinDr-CXR, ReX-VQA), the model collection process (publicly available models, fine-tuned models), training datasets, and architectures for each of the 55 specialist models."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public medical imaging benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the label-space alignment (§2.3) and cost function definitions (§2.1) are mathematically described, the concrete pipeline from raw datasets to the final 1448 benchmark queries — including any filtering, exclusion, or sampling criteria at each stage — is not fully documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the provided paper text. Funding sources are not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Department of Engineering Science, University of Oxford (Saha, Strong, Mishra, Noble) and Department of Computer Science, Khalifa University (Alsharid)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. The absence of funding disclosure makes this NO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained VLMs (Qwen3-VL, LLaVA, LLaMA-3.2-Vision) on benchmark datasets but does not state the training data cutoff dates for any of these models. It is unclear whether these models could have seen the test data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained VLMs (which are used as tool candidates) may have been trained on data overlapping with the Open-I, VinDr-CXR, or ReX-VQA test sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Open-I and VinDr-CXR are publicly available datasets. Large VLMs like Qwen3-VL and LLaVA may have encountered these during pre-training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-query cost is reported. The system involves running 55 specialist models plus a selector but the computational cost of this pipeline is not discussed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "§3.3 mentions training for 50 epochs with early stopping but no GPU hours, hardware specifications, total training time, or total compute budget are stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from a single training run of the selector."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated anywhere. Results in Tables 1-4 are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported (§3.3) but no information is given about how they were selected — number of configurations tried, search method, or compute spent on tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "§3.3: 'early stopping (patience = 10, min delta = 10−4)' — model selection is based on validation performance with a principled early stopping criterion, not cherry-picked from test results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baseline routing methods and compare them against their own ToolSelect. No acknowledgment that their implementations of baselines may systematically underperform compared to original authors' implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis. Baselines range from lightweight heuristics (EloRouter) to learned methods (GraphRouter with GNN) but compute costs are not compared. ToolSelect uses ANP cross-attention which likely costs more than simpler baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the automated metrics (F1, IoU, F1-RadGraph) actually measure clinical utility. The paper frames results as clinical performance improvement but doesn't question whether the benchmarks capture what matters clinically."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All routing methods are compared within the same agentic framework (Figure 3). The specialist model pool is held constant across all methods. Only the selection mechanism varies, so the scaffold confound is controlled by design."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The VLMs (Qwen3-VL, LLaVA, LLaMA-3.2-Vision) were trained on web-scale data that may include the benchmark datasets which predate them (Open-I from 2014, VinDr-CXR from 2022)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The selector uses behavioral reference sets from the same distribution as the test data; whether this introduces any information leakage from reference to test sets is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No verification that training and test splits for the selector are fully independent, or that reference set examples don't share structural similarities with test queries."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "ToolSelect consistently outperforms 10 SOTA routing baselines across all four task families.",
       "evidence": "Tables 1-4: ToolSelect achieves best routing performance on disease diagnosis (43.80% F1 Open-I, 59.88% VinDr), report generation (24.17% F1-RadGraph), visual grounding (50.08 Mean IoU), and VQA (72.01% accuracy).",
       "supported": "strong"
     },
     {
       "claim": "Per-instance Oracle selection reveals large headroom beyond any single specialist model.",
       "evidence": "Table 1: Oracle F1 58.00% vs best single-model F1 13.35% on Open-I. Table 4: Oracle 96.49% vs best single 63.22% on VQA. Table 3: Oracle IoU 56.20 vs best single 40.66.",
       "supported": "strong"
     },
     {
       "claim": "No single specialist model is uniformly reliable under dataset shift; the best model depends on the metric and dataset.",
       "evidence": "§4.1: R-All has highest accuracy (84.08% Open-I, 75.66% VinDr) but lowest F1 (6.61%, 4.70%). D-RSNA has 64.99% recall on VinDr but degrades on Open-I. Table 3 shows different grounding specialists excel on different phrases.",
       "supported": "strong"
     },
     {
       "claim": "Classical and heuristic routers provide limited improvement and can trail strong individual models.",
       "evidence": "§4.4: 'KNN 41.52%, Graph 47.37%' underperform the best single model (NV 63.22%). §4.1: best baseline F1 on Open-I is only 11.98% (MF Router).",
       "supported": "strong"
     },
     {
       "claim": "ToolSelect produces more clinically aligned reports than baseline routing methods.",
       "evidence": "Figure 5 shows two qualitative examples where ToolSelect selects better-suited specialists. §4.2: ToolSelect achieves 24.17% F1-RadGraph vs 22.40% for best baseline.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across Tables 1-4 are single-run point estimates with no confidence intervals, standard deviations, or multiple-seed results. For a learning-based method (ToolSelect), training variance could substantially affect results. Claims of 'consistently outperforms' are unsupported without uncertainty quantification."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims are made by comparing raw numbers without any statistical tests. The differences between ToolSelect and baselines could potentially fall within random variation, especially for the report generation task with only 156 queries."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a limitations section. The Impact Statement addresses deployment ethics but does not discuss methodological limitations such as the chest-X-ray-only evaluation, small benchmark sizes, or single-run results."
     },
     {
       "flag": "Overclaiming scope",
       "detail": "Title claims 'Agentic Healthcare Systems' broadly, but all experiments are limited to chest X-ray tasks only. Generalization to other imaging modalities, clinical domains, or real-world deployment settings is untested and not bounded."
     },
     {
       "flag": "No compute cost analysis for 55-model system",
       "detail": "The system requires maintaining and running predictions from 55 specialist models plus a selector, but no inference cost, latency, or computational budget is reported. Clinical deployment feasibility is a significant unaddressed concern."
     },
     {
       "flag": "Every result shows ToolSelect winning",
       "detail": "ToolSelect outperforms every baseline on every primary metric across every task. No configurations, tasks, or settings where the method fails or underperforms are reported. This uniformly positive narrative warrants skepticism."
     }
   ],
   "cited_papers": [
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Y. Qin", "S. Liang", "Y. Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Major benchmark for LLM tool use capabilities across real-world APIs."
     },
     {
       "title": "RouteLLM: Learning to route LLMs with preference data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Proposes LLM routing methods for cost-quality tradeoffs, used as baseline in this paper."
     },
     {
       "title": "RouterDC: Query-based router by dual contrastive learning for assembling large language models",
       "authors": ["S. Chen", "W. Jiang", "B. Lin"],
       "year": 2024,
       "relevance": "NeurIPS 2024 paper on contrastive learning for LLM routing, used as baseline."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["T. Schick", "J. Dwivedi-Yu"],
       "year": 2023,
       "relevance": "Foundational work on self-supervised tool use acquisition by LLMs."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2023,
       "relevance": "Core agentic paradigm (thought-action prompting) used in this paper's framework."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework relevant to agentic system design."
     },
     {
       "title": "MDAgents: An adaptive collaboration of LLMs for medical decision-making",
       "authors": ["Y. Kim", "C. Park", "H. Jeong"],
       "year": 2024,
       "relevance": "Multi-agent LLM collaboration for medical tasks, directly relevant to healthcare agent methodology."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-efficient LLM cascading approach relevant to model routing and selection."
     },
     {
       "title": "GraphRouter: A graph-based router for LLM selections",
       "authors": ["T. Feng", "Y. Shen", "J. You"],
       "year": 2024,
       "arxiv_id": "2410.03834",
       "relevance": "GNN-based LLM routing method, used as baseline in this paper."
     },
     {
       "title": "ToolACE: Winning the points of LLM function calling",
       "authors": ["W. Liu", "X. Huang", "X. Zeng"],
       "year": 2025,
       "relevance": "Recent work on LLM tool use and function calling capabilities."
     },
     {
       "title": "MMedAgent: Learning to use medical tools with multi-modal agent",
       "authors": ["B. Li", "T. Yan", "Y. Pan"],
       "year": 2024,
       "relevance": "Multi-modal medical agent that selects and composes imaging tools across modalities."
     },
     {
       "title": "Large language model based multi-agents: A survey of progress and challenges",
       "authors": ["T. Guo", "X. Chen", "Y. Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of multi-agent LLM systems covering planning, reasoning, and tool interaction."
     }
   ]
 }