ai-research-survey

scan.json (20777B)

---

 {
   "paper": {
     "title": "BioTrouble: A Multi-Agent Workflow for Troubleshooting Molecular Biology Techniques",
     "authors": ["Mehrdad Ameri", "Hannie Yousefabadi", "Amin Ramezani"],
     "year": 2026,
     "venue": "bioRxiv",
     "doi": "10.64898/2025.12.30.697016"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/mehrdadameri/BioTrouble. The paper states 'All code and data related to this project are available at GitHub.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and data related to this project are available at GitHub' with a link to the repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is mentioned in the paper. The paper mentions LangGraph framework and OpenRouter API but does not specify library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README instructions or commands to replicate experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as bar charts (Figures 4, 5, 6) but no confidence intervals, error bars, or ± notation are mentioned in the text."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'the multi-agent achieved significantly better scores than the baseline' but no statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports '40% cost reduction' with specific dollar amounts ($0.0036 vs $0.006 per request). Performance comparisons across metrics are shown in figures with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "200 examples used for evaluation and 20 questions for learning system evaluation. No justification for why these sizes were chosen. The paper acknowledges '200 questions... represents a small fraction' but provides no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures reported. Results appear to be single-run evaluations. The learning system evaluation uses 5 iterations but no variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares BioTrouble against baseline models: 'We used a mirror of the model assigned to this generator agent as a baseline to compare its results with the full BioTrouble pipeline.' GPT-4o baseline is also included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o (state-of-the-art LLM) and the same SLMs (Qwen3 8B, Llama 3.1 8B, Gemma 2 9B) used standalone. These are contemporary models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The system has multiple components (RAG, model routing, learning system, 8 agents) but none are individually removed to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used including Wrong Rate and other metrics shown in Figures 4 and 5. The paper references 'all metrics we considered for this evaluation.'"
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation uses LLM-as-a-Judge only: 'In an LLM-as-a-Judge evaluation using 200 examples.' No human expert evaluation of the troubleshooting output quality is performed. The learning system uses human feedback but that is part of the system, not an independent evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/test split or held-out test set. The 200 evaluation examples are not described as being separated from any development data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides categorical comparison by difficulty level: 'in a categorical comparison (difficulty level of questions), the multi-agent achieved significantly better scores than the baseline across all four model evaluation runs.'"
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The learning system evaluation specifically selects 'the top 20 challenging questions that the agent struggled to answer during the main evaluation,' indicating failure cases were identified and addressed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement. No configurations that failed or approaches that were tried and abandoned are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that BioTrouble 'generated comparable troubleshooting recommendations using small language models' compared to SOTA LLM are supported by the evaluation in Figures 4-5 showing SLMs achieving comparable scores to GPT-4o."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'BioTrouble can generate a better troubleshooting plan based on user feedback' and that the multi-agent workflow causes improvement over baselines. However, the comparison is full pipeline vs standalone model — many variables change simultaneously (RAG, routing, multi-agent coordination), making it impossible to attribute causation to any specific component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Molecular Biology Techniques' broadly, but the knowledge base covers a limited set of experiments (Table 2). The paper acknowledges 'we didn't cover all existing molecular biology experiments' in the discussion but the title and abstract do not bound this."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The improvement could be due to RAG providing relevant context rather than the multi-agent architecture, but this is not explored. No threats-to-validity section."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed by name (e.g., 'GPT-4o', 'gpt-4o-mini', 'gpt-4.1-mini', 'qwen3-8b', 'llama-3.1-8b', 'gemma-2-9b', 'granite-4.0-h-micro') but no snapshot dates or API versions are provided for the OpenAI models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes agent roles in natural language but does not include the system prompts or instructions given to any of the 8 agents."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the models used. Token thresholds for model routing are mentioned but specific values are not given."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail: 8 specialized agents, LangGraph framework, state-based architecture, model routing system, RAG retrieval, learning system with Reflector and Curator agents. Workflow diagrams referenced in Figures 1-3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The 200 evaluation questions are not described in terms of how they were created or selected. The knowledge base construction process is briefly mentioned ('carefully gathered protocols and troubleshooting guidelines from sources licensed under CC BY') but details of curation are missing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are discussed in the Discussion section: 'A primary limitation of this project is the availability of free high-quality data' and 'Another limitation of this project is the small evaluation dataset.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations mentioned: limited CC BY data availability affecting knowledge base quality, small evaluation dataset of 200 questions, incomplete coverage of molecular biology experiments. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While limitations are mentioned, the paper does not explicitly state what the results do NOT show. No explicit bounding of claims to the tested molecular biology experiments or specific models evaluated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and data related to this project are available at GitHub: https://github.com/mehrdadameri/BioTrouble.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The 200 evaluation questions' creation/collection process is not described. The knowledge base sources are vaguely described as 'protocols and troubleshooting guidelines from sources licensed under Creative Commons Attribution (CC BY)' without specifics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. The LLM-as-a-Judge evaluation and learning system evaluation do not involve recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from knowledge base construction to evaluation questions to final results is not documented with filtering criteria or counts of examples at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or statement about funding sources appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Shiraz University of Medical Sciences, Shiraz Institute for Cancer Research, and University of Tehran. No product being evaluated is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. The absence of a funding statement does not confirm unfunded status."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a 'Competing interests' section stating 'All authors declare that they have no competing interests.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on a benchmark. It evaluates a multi-agent system's troubleshooting output quality using LLM-as-a-Judge, not testing model knowledge on known test sets."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates a RAG-based system pipeline, not a pre-trained model's knowledge on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation uses custom troubleshooting questions, not a public benchmark that models may have seen during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost per request reported: 'BioTrouble averaged $0.0036 per request, while GPT-4o averaged $0.006' based on 100 requests via OpenRouter API."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total computational budget not stated. Per-request cost is given but total API spend, GPU hours for local models, or total evaluation cost is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "BioTrouble multi-agent workflow outperforms baseline standalone models across all evaluation metrics",
       "evidence": "LLM-as-a-Judge evaluation on 200 examples, results in Figures 4-5 showing multi-agent scores higher than baselines for all four model configurations.",
       "supported": "moderate"
     },
     {
       "claim": "SLMs in BioTrouble achieve comparable performance to GPT-4o baseline",
       "evidence": "Figure 5 shows all three SLMs (Qwen3 8B, Llama 3.1 8B, Gemma 2 9B) achieving 'scores nearly good as the GPT-4o baseline,' with Llama 3.1 8B outperforming GPT-4o on some metrics.",
       "supported": "moderate"
     },
     {
       "claim": "BioTrouble is 40% cheaper than GPT-4o while maintaining comparable functionality",
       "evidence": "$0.0036 vs $0.006 per request based on 100 requests via OpenRouter API.",
       "supported": "moderate"
     },
     {
       "claim": "BioTrouble's learning system improves response quality over 5 iterations with user feedback",
       "evidence": "Top 20 challenging questions re-evaluated over 5 conversations with human feedback, improvement shown in Figure 6. Only tested with Llama 3.1 8B.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "BioTrouble is a multi-agent RAG-based system using 8 specialized agents for molecular biology troubleshooting. Small language models (8-9B parameters) in the multi-agent pipeline achieve comparable performance to standalone GPT-4o at 40% lower cost ($0.0036 vs $0.006 per request). The system includes a feedback-driven learning mechanism that improves responses over iterations. Evaluation uses LLM-as-a-Judge on 200 questions without statistical rigor (no error bars, significance tests, or ablations).",
   "red_flags": [
     {
       "flag": "LLM-as-a-Judge without human validation",
       "detail": "All evaluation relies on LLM-as-a-Judge with no human expert evaluation of troubleshooting quality. For a domain-specific tool in molecular biology, expert validation is critical — an LLM judge may not catch scientifically incorrect advice."
     },
     {
       "flag": "No ablation study",
       "detail": "The system has 8 agents, RAG, model routing, and a learning system, but no ablation shows which components contribute to performance. The improvement could be entirely due to RAG rather than the multi-agent architecture."
     },
     {
       "flag": "No statistical rigor",
       "detail": "No error bars, confidence intervals, significance tests, or variance across runs reported despite claims of 'significantly better scores.' Results appear to be single-run evaluations on 200 examples."
     },
     {
       "flag": "Small evaluation dataset",
       "detail": "Only 200 questions for main evaluation and 20 for learning system evaluation. The paper acknowledges this is 'a small fraction of the troubleshooting scenarios' but draws broad conclusions."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "The baseline is a standalone model without RAG or any context, while BioTrouble provides curated domain knowledge via RAG. This comparison tests whether RAG helps, not whether multi-agent architecture helps."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["L. Wang", "C. Ma", "X. Feng"],
       "year": 2024,
       "relevance": "Foundational survey on LLM-based autonomous agents relevant to understanding agentic AI architectures."
     },
     {
       "title": "Small Language Models are the Future of Agentic AI",
       "authors": ["P. Belcak", "G. Heinrich"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Directly relevant to the survey's scope on small language models for agentic systems."
     },
     {
       "title": "Small Language Models for Agentic Systems: A Survey of Architectures, Capabilities, and Deployment Trade offs",
       "authors": ["R. Sharma", "M. Mehta"],
       "year": 2025,
       "arxiv_id": "2510.03847",
       "relevance": "Survey of SLM architectures for agentic systems, relevant to understanding the landscape of small model agents."
     },
     {
       "title": "Agentic context engineering: Evolving contexts for self-improving language models",
       "authors": ["Q. Zhang", "C. Hu"],
       "year": 2025,
       "arxiv_id": "2510.04618",
       "relevance": "ACE framework that BioTrouble's learning system is inspired by; relevant to context engineering for AI agents."
     },
     {
       "title": "Judging llm-as-a-judge with mt-bench and chatbot arena",
       "authors": ["L. Zheng", "W.-L. Chiang"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-Judge evaluation methodology used by BioTrouble."
     },
     {
       "title": "A survey on rag meeting llms: Towards retrieval-augmented large language models",
       "authors": ["W. Fan", "Y. Ding"],
       "year": 2024,
       "relevance": "Survey on RAG techniques relevant to understanding retrieval-augmented generation in agentic systems."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Y. Gao", "Y. Xiong"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Comprehensive RAG survey relevant to the survey's coverage of retrieval-augmented LLM systems."
     },
     {
       "title": "The rise and potential of large language model based agents: a survey",
       "authors": ["Z. Xi", "W. Chen"],
       "year": 2025,
       "relevance": "Major survey on LLM-based agents relevant to the survey's scope on agentic AI systems."
     }
   ]
 }