ai-research-survey

scan.json (25264B)

---

 {
   "paper": {
     "title": "RAG-MCP: Mitigating Prompt Bloat in LLM Tool Selection via Retrieval-Augmented Generation",
     "authors": ["Tiantian Gan", "Qiyao Sun"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.03275",
     "doi": "10.48550/arXiv.2505.03275"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RAG-MCP applies semantic retrieval to filter relevant MCP tool descriptions before presenting them to an LLM, improving tool selection accuracy from 13.62% (naive all-tools prompting) to 43.13% on MCPBench web search tasks while reducing prompt tokens by roughly 49%. A stress test varying the MCP pool from 1 to 11,100 shows performance degrades sharply beyond ~100 candidate tools, with reliable selection only in small pools.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available web search subset of MCPBench [8] and draws distractor MCPs from the public mcp.so registry [14]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency lists are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (43.13%, 18.20%, 13.62%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RAG-MCP 'significantly outperforms' baselines but provides no statistical significance tests (p-values, t-tests, etc.). Comparisons rely solely on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports accuracy percentages for all three methods (43.13%, 18.20%, 13.62%) along with token counts, providing baseline context for the magnitude of improvement. The abstract notes the tripling (43.13% vs 13.62%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 20 trials per baseline and 20 web search tasks but provides no justification for why these sample sizes were chosen or whether they provide adequate statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across the 20 trials. Only aggregate point estimates appear in Table 1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: 'Blank Conditioning' (all MCPs in prompt) and 'Actual Match' (keyword-filtered MCPs). Section 4.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are ad-hoc strategies (naive prompting and keyword matching) rather than published contemporary tool-selection methods. No comparison against prior retrieval-based tool selection approaches (e.g., Gorilla's retrieval-augmented approach, which is discussed in the related work)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "RAG-MCP has three components (retrieval, validation, invocation) but no ablation study removing individual components. The validation step's contribution is not isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports accuracy, average prompt tokens, and average completion tokens."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using a 'Llama-based verifier' and DeepSeek-v3 as evaluator. No human evaluation of outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the web search subset of MCPBench was used as 'our heldout testbed' (Section 4.2), an external benchmark not used for development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate accuracy across all 20 web search tasks is reported. No per-task or per-category breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The stress test analysis (Section 5) discusses general patterns of degradation but no specific failure examples are shown or analyzed qualitatively."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The stress test and analysis explicitly acknowledge performance degradation: 'retrieval precision and overall throughput degrade as the tool registry scales to thousands of MCPs' (Section 4.1). Figure 3 shows widespread failure at scale."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims token reduction 'by over 50%' but Table 1 shows 1084 vs 2133.84 = 49.2% reduction, technically under 50%. The abstract claims 'scalable and accurate tool integration' but 43.13% accuracy is not accurate in absolute terms, and the stress test shows poor scaling beyond ~100 MCPs."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims retrieval 'causes' improvement but the study design does not disentangle confounds: RAG-MCP uses an additional embedding model (Qwen) that baselines do not, and the effect of prompt reduction vs retrieval quality is not separated. No statistical tests confirm the observed differences are not due to chance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims general 'LLM Tool Selection' and conclusions reference 'hundreds or thousands of tools,' but all experiments use only web search tasks with a single LLM (qwen-max-0125). No other task types, domains, or models are tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. Could the improvement come from the additional Qwen model rather than the retrieval approach? Could the keyword baseline be poorly implemented? Could web search tasks be unusually easy for semantic retrieval? None of these are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures tool selection accuracy and frames results in terms of tool selection accuracy. The measurements match the granularity of the claims — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The base LLM is specified as 'qwen-max-0125' (Section 4.2), but the retriever is only described as 'a lightweight LLM-based retriever (e.g., Qwen)' without a version, and the judge is only 'Llama-based verifier' with no model name or version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompts are provided. The paper describes what the system does in natural language but never shows the actual prompt text sent to any model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, top-k for retrieval, embedding dimensions, similarity thresholds, or other settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The three-step pipeline (retrieval → validation → invocation) is described at a high level with Figure 2, but the validation step is vaguely described ('generate a few-shot example query and test its response') with no implementation detail. How the retriever index is built, what similarity metric is used, and how top-k is determined are not specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "How MCP descriptions were collected from mcp.so, preprocessed, and indexed into the vector store is not documented. The selection of the 20 web search tasks from MCPBench is not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion briefly mentions future work on 'retrieval at extreme scale' but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to web search tasks, the single LLM tested, or the specific MCP registry used."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, logs, or per-trial results are released. Only aggregate statistics in Table 1 and the heatmap in Figure 3 are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the stress test data collection: ground-truth MCPs and N-1 distractors from 4,400+ public MCP servers on mcp.so, with N varied from 1 to 11,100 in 26 intervals. Section 4.2 describes using the MCPBench web search subset with 20 trials."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public benchmark (MCPBench) and public MCP registry (mcp.so)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw MCP descriptions to vector index to retrieval results is not documented. How MCPs were embedded, how the index was constructed, and how distractor selection worked in detail are not specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is mentioned. The acknowledgements section thanks the MCPBench authors but does not disclose any funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Beijing University of Post and Communications and Queen Mary University of London. No product being evaluated is affiliated with their institutions."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Appears to be unfunded student research (university affiliations with student email addresses, no funding mentioned)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates qwen-max-0125 on MCPBench tasks but does not state the model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MCPBench web search tasks or MCP descriptions appeared in Qwen's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MCPBench [8] was published in April 2025. The model qwen-max-0125 likely predates this, reducing contamination risk, but this temporal relationship is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports average prompt tokens and average completion tokens for each method (e.g., RAG-MCP: 1084 prompt, 78.14 completion tokens), providing a measure of per-query inference cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no total API spend, wall-clock time for experiments, or hardware specifications."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Distractor MCPs are randomly selected but no seed sensitivity analysis is performed and no results across multiple seeds are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states '20 independent trials' for each baseline method."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Top-k for retrieval, embedding model choice, and similarity threshold appear to have been set without documented search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final configuration (top-k=1, specific embedding model, etc.) was selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Three methods are compared but no statistical tests are performed at all, making multiple comparison correction impossible to apply."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all three baselines themselves (including the keyword-matching 'Actual Match') but do not acknowledge potential bias in their implementation of competitors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RAG-MCP adds a separate embedding and retrieval step using a Qwen model that baselines do not incur. This additional compute cost is not discussed relative to the performance gain."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MCPBench web search tasks represent real-world tool selection scenarios or whether success on web search generalizes to other tool types."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RAG-MCP adds a retriever model (Qwen) not present in baselines. The improvement could stem from the additional model's capabilities rather than the retrieval approach itself. This confound is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether qwen-max-0125's training data includes MCPBench tasks or MCP server descriptions from mcp.so."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, MCP server names in the prompt may contain hints about functionality."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 20 web search tasks likely share structural similarities (all being web search), but independence between test examples is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG-MCP cuts prompt tokens by over 50% compared to feeding all tools at once.",
       "evidence": "Table 1 shows RAG-MCP uses 1084 avg prompt tokens vs Blank Conditioning's 2133.84, a reduction of ~49.2%. The '50%' threshold is not quite met.",
       "supported": "weak"
     },
     {
       "claim": "RAG-MCP more than triples tool selection accuracy (43.13% vs 13.62% baseline).",
       "evidence": "Table 1 shows 43.13% accuracy for RAG-MCP vs 13.62% for Blank Conditioning, a 3.17x improvement. However, only 20 trials with no statistical tests or error bars.",
       "supported": "moderate"
     },
     {
       "claim": "RAG-MCP enables scalable and accurate tool integration for LLMs.",
       "evidence": "The stress test (Figure 3, Section 5) shows performance degrades sharply beyond ~100 MCPs, with purple (failure) dominating above that threshold. 43.13% accuracy is below chance in absolute terms.",
       "supported": "weak"
     },
     {
       "claim": "LLM tool selection performance degrades as the number of candidate MCPs increases.",
       "evidence": "Figure 3 visualizes per-trial success across MCP pool sizes 1-11,100, showing high success below 30, variability at 31-70, and dominant failure beyond ~100.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very low absolute accuracy",
       "detail": "The best method (RAG-MCP) achieves only 43.13% tool selection accuracy — less than half of selections are correct. The paper frames this as a major success and 'compelling solution' for scalable tool integration."
     },
     {
       "flag": "Overclaimed token reduction",
       "detail": "The abstract claims 'over 50%' token reduction, but Table 1 shows 1084 vs 2133.84 tokens = 49.2% reduction, which is under 50%."
     },
     {
       "flag": "Tiny sample size with no uncertainty quantification",
       "detail": "All conclusions rest on 20 trials per method with no error bars, confidence intervals, or statistical tests. The word 'significantly' is used without any statistical test."
     },
     {
       "flag": "Unspecified evaluation judge",
       "detail": "Answer correctness is judged by a 'Llama-based verifier' whose specific model, version, and accuracy are never stated. DeepSeek-v3 is also used as an evaluator. Neither judge is validated."
     },
     {
       "flag": "Missing critical implementation details",
       "detail": "The retriever model version, embedding dimensions, similarity metric, top-k value, and all hyperparameters are unspecified, making reproduction impossible even with code."
     },
     {
       "flag": "Scalability claim contradicted by own results",
       "detail": "The paper claims to 'enable scalable tool integration' but the stress test shows performance collapses beyond ~100 MCPs, well short of the 4,400+ registry or the claimed 'thousands of tools' scenario."
     }
   ],
   "cited_papers": [
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez"],
       "year": 2024,
       "relevance": "Demonstrates retrieval-augmented API documentation to improve LLM tool use accuracy, directly relevant to tool selection for LLM agents."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessi", "Roberta Raileanu"],
       "year": 2023,
       "relevance": "Foundational work on self-supervised tool-use learning in LLMs, showing models can learn when and how to call APIs."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du"],
       "year": 2023,
       "relevance": "Key agentic workflow paper interleaving reasoning and tool actions, relevant to LLM tool-use evaluation."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper combining parametric LLMs with non-parametric retrieval, the basis for RAG-MCP's approach."
     },
     {
       "title": "WebGPT: Browser-Assisted Question-Answering with Human Feedback",
       "authors": ["Reiichiro Nakano", "Jacob Hilton", "Saurav Balaji"],
       "year": 2022,
       "arxiv_id": "2112.09332",
       "relevance": "Trains GPT-3 to navigate and search the web, relevant to LLM tool use and grounded retrieval to reduce hallucinations."
     },
     {
       "title": "Evaluation Report on MCP Servers",
       "authors": ["Zhiling Luo", "Xiaorong Shi", "Xuanrui Lin", "Jinyang Gao"],
       "year": 2025,
       "arxiv_id": "2504.11094",
       "relevance": "Provides the MCPBench evaluation framework and WebSearch dataset used as the testbed in this paper."
     },
     {
       "title": "Enhancing Function-Calling Capabilities in LLMs: Strategies for Prompt Formats, Data Integration, and Multilingual Translation",
       "authors": ["Yi-Cheng Chen", "Pei-Chi Hsu", "Cheng-Jui Hsu"],
       "year": 2024,
       "arxiv_id": "2412.01130",
       "relevance": "Explores prompt format strategies for LLM function calling, directly relevant to tool-use methodology."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["Aixin Liu", "Bei Feng", "Bing Xue"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "Used as the automated evaluator for answer correctness in the experiments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Using RAG to filter MCP tools before prompting is a practically useful idea for developers building multi-tool LLM agents, though the low absolute accuracy limits immediate applicability."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The finding that filtering irrelevant tools from the prompt improves performance confirms intuitive expectations rather than challenging them."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "References MCP (Anthropic) and uses Qwen/DeepSeek models, but authors are from relatively lesser-known university programs."
     }
   }
 }