ai-research-survey

scan.json (24030B)

---

 {
   "paper": {
     "title": "From Single to Multi-Agent Reasoning: Advancing GeneGPT for Genomics QA",
     "authors": ["Kimia Abedini", "Farzad Shami", "Gianmaria Silvello"],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.10581",
     "doi": "10.48550/arXiv.2601.10581"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GenomAgent, a multi-agent framework for genomic QA, outperforms the replicated GeneGPT system by 12% on average (0.93 vs 0.83) on the GeneTuring benchmark while reducing computational cost by 79% ($2.11 vs $10.06). The largest gains come in sequence alignment tasks (28.8% improvement). The paper also provides a reproducibility study of GeneGPT, identifying stop-token parsing failures, context loss, and incomplete data coverage as key error types.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A project page URL is provided in the abstract: https://kimia-abedini.github.io/Genom-Agent/. This is a public URL for the project."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available GeneTuring benchmark (Hou & Ji, 2023), comprising 12 tasks with 50 QA pairs each. The benchmark is a standard public dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The project page is referenced but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1 and 2 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GenomAgent 'outperforms' GeneGPT based solely on comparing point scores (0.93 vs 0.83) with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout: '12% increase in average performance (0.93 vs. 0.83)', '79% reduction in computational cost ($2.11 vs. $10.06)', and per-category improvements in Table 2."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each task has 50 QA pairs from the GeneTuring benchmark, but no justification is given for why this sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run statistics are reported. Results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple GeneGPT configurations (Full, Slim, Turbo, Lang) serve as baselines, and the original paper's baselines (Bing Chat, BioMedLM, GPT-3) are also referenced."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The main baseline GeneGPT was published in 2024 and uses deprecated models (code-davinci-002, GPT-3.5-turbo-16k). The paper acknowledges a concurrent work (Chen et al., 2025, ref [5]) but only mentions it as future comparison rather than including it."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly acknowledges in Future Work (Section 6): 'the 12% average improvement cannot be cleanly attributed to specific architectural choices without systematic ablation analysis.' No ablation is performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple task-specific metrics are used: exact match accuracy for nomenclature, recall for association tasks, vocabulary-mapped matching for cross-species alignment, and partial scoring for human genome alignment."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually reviewed and categorized all mistakes made by the reproduced GeneGPT system into three error types (E1, E2, E3) in Section 3."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The GeneTuring benchmark is a fixed test set of 50 QA pairs per task, not used for any tuning or development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide detailed per-task breakdowns across all 9 tasks grouped into 4 categories (nomenclature, genomic location, functional analysis, sequence alignment)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 categorizes reproduced GeneGPT errors into E1 (incomplete data coverage), E2 (stop-token parsing failures), and E3 (context loss). However, GenomAgent's failures are not discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The reproducibility study shows degraded performance in turbo settings (Table 1), including an 83.33% drop on DNA-to-Human alignment. The lang configuration showed massive variability."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 12% average improvement and these numbers match Table 2 results (0.93 vs 0.83). The 79% cost reduction claim is also supported ($2.11 vs $10.06)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GenomAgent's multi-agent architecture 'addresses efficiency bottlenecks' and 'delivers superior accuracy,' but the authors themselves acknowledge the improvement 'cannot be cleanly attributed to specific architectural choices without systematic ablation analysis.' Multiple variables change simultaneously: model (GPT-4o-mini vs deprecated models), architecture (multi-agent vs single-agent), APIs (multi-source vs single-source), and evaluation protocol adjustments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the 'flexible architecture extends beyond genomics to various scientific domains needing expert knowledge extraction' but testing is only on GeneTuring. The title says 'Genomics QA' but the generalization claim is unsupported. The paper acknowledges in limitations that evaluation is 'limited to the GeneTuring benchmark.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The improvement could be due to the model upgrade (GPT-4o-mini vs deprecated Codex/GPT-3.5), multi-source database access, or evaluation protocol changes (expanded vocabulary, partial scoring), rather than the multi-agent architecture itself. These confounds are not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific task-level metrics (exact match, recall, partial scoring) and reports them as task-level performance. The macro-average is explicitly acknowledged as 'a simplified view of the system's diverse capabilities across heterogeneous genomics QA tasks' (Section 2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-4o-mini' as the replacement model without any version date or API snapshot identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompting strategies and agent roles in natural language but does not provide actual prompt text. Agent-specific instructions are described functionally but not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any model configuration."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Figure 1 describe the multi-agent architecture in detail: Task Detection Agent, MCP Agent, Response Handler Agent, Feature Extractor Agent, Code Writer Agent, Code Executor Agent, and Final Decision Agent, with their roles and interactions. Built on Google Agent Development Kit."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents changes from the original GeneGPT: removal of context truncation, explicit prompt format enforcement, expanded vocabulary mappings, and enhanced partial scoring mechanisms (Section 5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Final Remarks and Future Work) contains substantive discussion of limitations including lack of ablation, benchmark scope limitation, and need for hybrid approaches."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific limitations: '12% average improvement cannot be cleanly attributed to specific architectural choices without systematic ablation analysis' and 'evaluation is limited to the GeneTuring benchmark. This restricted scope prevents us from fully validating GenomAgent's generalizability.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: evaluation limited to GeneTuring benchmark, no ablation analysis performed, no comparison with emerging frameworks like Chen et al. (2025). These are specific things not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw output data (model responses, API call logs) is made available. Only aggregated scores are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The GeneTuring benchmark is well-described: 12 tasks, 50 QA pairs each, 9 tasks selected matching the original GeneGPT paper. The benchmark source (Hou & Ji, 2023) is cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The processing pipeline is documented: query → Task Detection Agent → MCP Agent (parallel API calls) → Response Handler → Final Decision Agent. Evaluation protocol including metric choices is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'partially supported by the HEREDITARY Project, as part of the European Union's Horizon Europe research and innovation programme under grant agreement No GA 101137074.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Padua (Italy) and Aalto University (Finland). No commercial affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The EU Horizon Europe HEREDITARY project is a research grant with no commercial stake in GenomAgent's performance vs GeneGPT."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "Disclosure of Interests section states: 'The authors have no competing interests to declare that are relevant to the content of this article.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o-mini is used but its training data cutoff is not stated. The GeneTuring benchmark (2023) could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o-mini's training data includes GeneTuring benchmark questions or answers."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GeneTuring was published in 2023 (BioRxiv). GPT-4o-mini was trained after this date. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports per-category and total costs for both GenomAgent ($2.11 total) and all GeneGPT configurations ($10.06-$16.76), computed from token counts and real model pricing."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While API costs are reported, no total compute budget, GPU hours, or wall-clock time is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each configuration was run once."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described or budgeted despite agent configurations and prompt designs being choices."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "GenomAgent appears to be a single configuration. No description of how design choices were selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both GenomAgent and the reproduced GeneGPT baseline. No acknowledgment of author-evaluation bias in baseline reimplementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 and Figure 2 explicitly compare performance against cost for all systems, showing the performance-cost tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The GeneTuring benchmark is used without discussion of whether its 50-question QA format adequately measures genomic QA capability or whether it represents real-world genomic research needs."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GenomAgent uses a completely different architecture (multi-agent on Google ADK with GPT-4o-mini) compared to GeneGPT (single-agent with deprecated models). The improvement is attributed to the multi-agent architecture, but the model, framework, and APIs all differ simultaneously. The authors acknowledge this in limitations but do not address it experimentally."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GeneTuring (2023) predates GPT-4o-mini's training. No discussion of whether the model has seen the benchmark questions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (API access to NCBI, HGNC, UCSC) provides information that constitutes feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between any in-context learning examples and test questions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "GenomAgent achieves an average performance score of 0.93, a 12% improvement over GeneGPT's best score of 0.83.",
       "evidence": "Table 2 shows per-task scores across 9 GeneTuring tasks. GenomAgent scores 0.98 on nomenclature, 0.98 on genomic location, 0.89 on functional analysis, and 0.85 on sequence alignment.",
       "supported": "moderate"
     },
     {
       "claim": "GenomAgent reduces computational cost by 79% ($2.11 vs $10.06).",
       "evidence": "Table 2 reports per-category costs computed from token counts and model pricing. GenomAgent total is $2.11 vs GeneGPT Slim's $10.06.",
       "supported": "moderate"
     },
     {
       "claim": "Sequence alignment tasks see the largest improvement (28.8%).",
       "evidence": "Table 2 shows GenomAgent scores 0.85 on sequence alignment vs GeneGPT Slim's 0.66, a 28.8% relative improvement.",
       "supported": "moderate"
     },
     {
       "claim": "GeneGPT's turbo configuration shows degraded performance when reproduced with GPT-4o-mini due to stop-token incompatibility.",
       "evidence": "Table 1 shows relative differences ranging from -83.33% (DNA to Human) to +6.25% (Gene Alias). Error analysis identifies E2 (stop-token parsing failures) as the dominant error type.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Confounded comparison",
       "detail": "GenomAgent changes the model (GPT-4o-mini vs deprecated Codex/GPT-3.5), architecture (multi-agent vs single-agent), data sources (multi-database vs single), and framework (Google ADK vs custom) simultaneously. The 12% improvement cannot be attributed to any specific change, as the authors themselves acknowledge."
     },
     {
       "flag": "No error bars or variance on single-run results",
       "detail": "All results appear to be from single runs with 50 questions per task. LLM outputs are stochastic, and with N=50 per task, point estimates could vary substantially across runs. No variance or confidence intervals are reported."
     },
     {
       "flag": "Evaluation protocol changes",
       "detail": "GenomAgent uses 'expanded vocabulary mappings' and 'enhanced partial scoring' compared to GeneGPT's original protocol. While the paper states these were applied to both systems, these changes could systematically favor the multi-source system that returns more diverse nomenclature."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "GeneTuring was published in 2023 on BioRxiv. GPT-4o-mini was trained after this date and may have seen the benchmark questions during training. No contamination analysis is performed."
     }
   ],
   "cited_papers": [
     {
       "title": "GeneGPT: Augmenting large language models with domain tools for improved access to biomedical information",
       "authors": ["Qiao Jin", "Yifan Yang", "Qingyu Chen", "Zhiyong Lu"],
       "year": 2024,
       "relevance": "State-of-the-art system for tool-augmented LLM genomic QA that this paper replicates and extends."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes of multi-agent LLM systems, directly relevant to agentic AI evaluation methodology."
     },
     {
       "title": "Beyond GeneGPT: A multi-agent architecture with open-source LLMs for enhanced genomic question answering",
       "authors": ["H. Chen", "G. Zuccon", "T. Leelanupab"],
       "year": 2025,
       "relevance": "Concurrent work on multi-agent genomic QA with open-source LLMs, relevant as a contemporary baseline."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper used as the base model in original GeneGPT; foundational to code-generating LLM evaluation."
     },
     {
       "title": "The landscape of emerging AI agent architectures for reasoning, planning, and tool calling: A survey",
       "authors": ["T. Masterman"],
       "year": 2024,
       "arxiv_id": "2404.11584",
       "relevance": "Survey of AI agent architectures relevant to understanding single vs multi-agent system design."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "ReAct framework used in GeneGPT's lang configuration; foundational agentic reasoning method."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing in-context learning, the foundational technique used by GeneGPT."
     },
     {
       "title": "LLM with tools: A survey",
       "authors": ["Zhiheng Shen"],
       "year": 2024,
       "arxiv_id": "2409.18807",
       "relevance": "Survey of tool-augmented LLMs directly relevant to the tool-integration approach evaluated here."
     }
   ]
 }