ai-research-survey

scan.json (27353B)

---

 {
   "paper": {
     "title": "Stop Wasting Your Tokens: Towards Efficient Runtime Multi-Agent Systems",
     "authors": ["Fulin Lin", "Shaowen Chen", "Ruishan Fang", "Hongwei Wang", "Tao Lin"],
     "year": 2025,
     "venue": "arXiv (under review)",
     "arxiv_id": "2510.26585",
     "doi": "10.48550/arXiv.2510.26585"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SupervisorAgent, a lightweight meta-agent framework for runtime MAS supervision, reduces token consumption by ~29.7% on GAIA without compromising success rates. The framework's three strategies (error correction, inefficiency guidance, observation purification) are validated across 6 benchmarks and 3 foundation models. Ablation shows purification drives token savings while correction/guidance maintain accuracy. The approach generalizes across MAS frameworks (Smolagent, AWorld, OAgents).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in abstract: https://github.com/LINs-lab/SupervisorAgent."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks are publicly available standard datasets (GAIA, GSM8k-Hard, AIME 2024, HumanEval, MBPP, DROP). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided in the paper. Only model names are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Implementation details are described in Appendix A.3 and prompts in A.5, but no step-by-step reproduction instructions or runnable scripts are provided in the paper itself. The GitHub repo may contain these but the paper does not."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported in any results table. Tables 1-4 report point estimates only. Figure 3 shows violin plots of distributions but no formal CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims like '29.68% token reduction' and accuracy comparisons are based solely on comparing point estimates without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are consistently reported with baseline context, e.g., '29.68% token reduction' (Table 1), '23.74% token reduction on HumanEval' (Table 2), with absolute before/after numbers provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. GSM8k-Hard uses 600 samples and DROP uses 800 samples, but no rationale for these specific numbers beyond 'following the sampling strategy of prior work' for DROP."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows violin plots of token cost distributions and variance reduction, but no standard deviations, IQR, or spread measures appear in any results table. Main results (Tables 1-4) are single-point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: Vanilla LLM, CoT-SC, CodeAgent, Smolagent, OWL, OAgents, MetaAgent, AWorld (Tables 1 and 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are very recent (2025): AWorld, OAgents, OWL, MetaAgent, Smolagent. All are current state-of-the-art MAS frameworks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents ablation removing each of the three core strategies (w/o Correction, w/o Guidance, w/o Purification), isolating their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics: accuracy/F1 and token consumption. Also reports pass@k for k=1,2,3 on GAIA, and variance reduction (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs. All evaluation is automated (pass/fail on benchmarks, token counts). Given claims about 'robustness' and 'guidance quality', human evaluation of intervention quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GAIA uses the test set with 164 tasks (Appendix A.2.1 states 'we utilize the test set'). HumanEval and MBPP are standard test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by GAIA difficulty level (L1, L2, L3) in Tables 1, 3, 4. Table 2 breaks down across different benchmark domains."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses overcompression causing 'minor accuracy or F1 drops' and the 'noise-as-signal' problem where purification can paradoxically harm performance. Case study in Appendix A.4 shows a baseline failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings reported: overcompression can harm performance (Section 6), MBPP accuracy drops slightly with SMAS (84.43 vs 85.68 in Table 2), DROP F1 drops (79.80 vs 81.08), and the noise-as-signal discovery."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '29.45% token reduction' — Table 1 shows 29.68% at pass@1 (minor discrepancy between abstract and table). 'Without compromising success rate' — Table 1 pass@1 shows same 50.91% accuracy. Claims about 5 additional benchmarks supported by Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('SupervisorAgent reduces token consumption'). The ablation study (Table 3) provides controlled single-variable manipulation by removing each component individually, which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are reasonably bounded to tested settings. The paper tests across 6 benchmarks, 3 foundation models, and 3 MAS frameworks, and states specific results for each configuration rather than making unbounded generalizations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for observed improvements. The paper does not consider whether gains come from, e.g., prompt engineering effects, specific model sensitivities, or benchmark-specific characteristics rather than the supervision framework itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures token consumption and task accuracy directly, which are the actual metrics claimed. No proxy gap exists — they don't frame token counts as 'efficiency' in a broader sense beyond what they measure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions given: 'GPT-4.1 (OpenAI, 2025)', 'Gemini-2.5-pro-0605', 'Qwen3-235B-2507', 'Qwen3-32B'. Gemini and Qwen include date/version suffixes. GPT-4.1 is a specific API model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided in Appendix A.5 for all supervision types: base prompt, error_occurrence, inefficiency_behavior, excessive observation length, and result_synthesis."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The observation length threshold (3,000 characters) is stated in Appendix A.3.2, but no LLM inference hyperparameters (temperature, top-p, max tokens) are reported for any model."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Extensive scaffolding description in Section 4 (methodology) and Appendix A.3 (implementation details), including the adaptive filter, context window construction, intervention pipeline, and action execution."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset sampling documented: 600 random samples from GSM8k-Hard, 800 from DROP 'following the sampling strategy of prior work', full HumanEval/MBPP, GAIA test set (164 tasks). Ablation subset defined as top-10 most token-intensive tasks per level."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) includes substantive discussion of limitations: overcompression risks, noise-as-signal problem, need for better purification techniques, and the limitation of token cost as sole efficiency metric."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: 'seemingly noisy information, such as HTML structure and truncation cues, serves as a vital signal for ReAct-style agents' and 'overly aggressive purification can paradoxically harm performance.' These are specific to this system."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do NOT show. The paper does not specify which MAS types, task domains, or model families are excluded from their claims. Future directions mention 'self-evolving version' but don't bound current claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-task results, per-run outputs) is made available. Only aggregated results in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is straightforward as all datasets are standard public benchmarks. Appendix A.2.1 describes each dataset's characteristics and the specific splits/subsets used."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from benchmark input to final reported numbers is not fully documented. No description of how outputs were evaluated (automated scripts? manual checking?), how pass@k was computed from runs, or how token counts were aggregated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Zhejiang University and Westlake University. The work was done during Fulin's visit to Westlake."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any model used (GPT-4.1, Gemini-2.5-pro, Qwen3). This is relevant since HumanEval (2021), MBPP (2021), GSM8k, and DROP are all public benchmarks likely in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap despite using well-known benchmarks (HumanEval, MBPP, GSM8k) that have been public for years before the models' training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) have been publicly available for years and are almost certainly in the training data of GPT-4.1 and Qwen3. No contamination analysis is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption is a primary metric reported for all experiments. Tables 1-4 report average tokens consumed per task for every method and configuration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated — no GPU hours, total API spend, or wall-clock time for running experiments. Only per-task average token counts are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. pass@k results imply multiple runs but no reporting of variance across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports pass@1, pass@2, pass@3 on GAIA implying multiple runs, but does not explicitly state the total number of runs per task. For other benchmarks (Table 2), single results are reported with no run count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The observation length threshold (3000 chars) and other design choices appear fixed without justifying how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 3000-character threshold and other filter heuristics are presented without justification for how these values were selected or whether alternatives were explored."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons made across 6 benchmarks, multiple models, and ablation variants. No correction for multiple comparisons applied, and no significance tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors implement their own system and compare against existing frameworks without acknowledging potential bias from their implementation choices or familiarity with their method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Token consumption (a compute proxy) is explicitly reported alongside accuracy for every experiment, enabling direct efficiency-performance comparison. Tables consistently pair accuracy with token counts."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the chosen benchmarks actually measure the claimed capabilities. The paper uses GAIA for 'general problem-solving' and HumanEval for 'code generation' without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this by testing SupervisorAgent across three different MAS frameworks (Smolagent, AWorld, OAgents) in Table 4, and discusses the choice of Smolagent as primary testbed due to its reliance on internal reasoning rather than external tools (Section 6)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. HumanEval (2021) and MBPP (2021) solutions have been publicly available for years and are likely in the training data of 2025 models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The SupervisorAgent itself has access to the global task description, which could provide hints not available in unmonitored settings."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities or whether training data contains related problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied despite using multiple well-known benchmarks with models likely trained on their solutions."
       }
     }
   },
   "claims": [
     {
       "claim": "SupervisorAgent reduces token consumption of Smolagent by 29.68% on GAIA at pass@1 while maintaining equivalent success rates.",
       "evidence": "Table 1 shows 50.91% accuracy for both Smolagent and SMAS at pass@1, with average tokens dropping from 527.76K to 371.12K (29.68% reduction).",
       "supported": "strong"
     },
     {
       "claim": "Efficiency gains are more pronounced on harder tasks, with 32.39% savings on Level 2 and 30.10% on Level 3 at pass@3.",
       "evidence": "Table 1 pass@3 row shows L2 tokens: 605.05K→409.05K (32.39%) and L3: 611.87K→427.72K (30.10%).",
       "supported": "strong"
     },
     {
       "claim": "SupervisorAgent generalizes across diverse benchmarks, achieving 23.74% token reduction on HumanEval alongside an accuracy improvement.",
       "evidence": "Table 2 shows HumanEval tokens drop from 40.91K to 31.19K (23.74%) and accuracy rises from 92.07% to 92.68%.",
       "supported": "moderate"
     },
     {
       "claim": "The framework is model-agnostic, consistently delivering token savings across GPT-4.1, Gemini-2.5-pro, and Qwen3-235B.",
       "evidence": "Figure 4b shows consistent token reduction across all three models on GAIA, though exact numbers are only in the figure.",
       "supported": "moderate"
     },
     {
       "claim": "SupervisorAgent significantly reduces variance in token consumption, improving performance consistency.",
       "evidence": "Figure 3 shows violin plots with visibly tighter distributions for SMAS and bar charts showing reduced variance, especially for L2 (63% reduction mentioned in Figure 1 caption).",
       "supported": "moderate"
     },
     {
       "claim": "Purification is the primary driver of token reduction, while Correction and Guidance are critical for maintaining accuracy.",
       "evidence": "Table 3 ablation: w/o Purification shows highest remaining token cost (851K vs 721K full). Removing Correction or Guidance maintains token savings but accuracy drops from 46.67% to 40.00%.",
       "supported": "strong"
     },
     {
       "claim": "SupervisorAgent generalizes across MAS frameworks, reducing tokens by 36.54% on AWorld and 39.36% on OAgents.",
       "evidence": "Table 4 shows cross-framework results on GAIA subset with consistent token reductions.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All claims of improvement are based on comparing point estimates without any significance tests. Given the stochastic nature of LLM outputs, observed differences could be due to sampling variance."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval and MBPP were published in 2021 and are almost certainly in the training data of 2025 models. This doesn't invalidate the relative comparison (both baseline and SMAS use the same models) but is a methodological gap."
     },
     {
       "flag": "Ablation on non-representative subset",
       "detail": "The ablation study (Table 3) and cross-framework evaluation (Table 4) are conducted on only the top-10 most token-intensive tasks per GAIA level, not the full benchmark. This biases results toward cases where supervision is most impactful."
     },
     {
       "flag": "Minor abstract-table discrepancy",
       "detail": "Abstract claims '29.45%' token reduction while Table 1 shows 29.68%. Small but suggests the abstract may have been written from a different run."
     },
     {
       "flag": "No LLM hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not stated for any model, yet these significantly affect output quality and token usage."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes in multi-agent LLM systems, directly motivating this work's supervision approach."
     },
     {
       "title": "Which agent causes task failures and when? On automated failure attribution of LLM multi-agent systems",
       "authors": ["Shaokun Zhang", "Ming Yin"],
       "year": 2025,
       "arxiv_id": "2505.00212",
       "relevance": "Addresses failure attribution in MAS, the reactive approach that SupervisorAgent aims to surpass with proactive intervention."
     },
     {
       "title": "AgentDropout: Dynamic agent elimination for token-efficient and high-performance LLM-based multi-agent collaboration",
       "authors": ["Zhexuan Wang", "Yutong Wang"],
       "year": 2025,
       "arxiv_id": "2503.18891",
       "relevance": "Design-time MAS efficiency optimization through agent pruning, complementary to SupervisorAgent's runtime approach."
     },
     {
       "title": "Aegis: Taxonomy and optimizations for overcoming agent-environment failures in LLM agents",
       "authors": ["Kevin Song", "Anand Jayarajan"],
       "year": 2025,
       "arxiv_id": "2508.19504",
       "relevance": "Proposes failure taxonomies for LLM agents, relevant to understanding and categorizing agent failure modes."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "Primary evaluation benchmark for multi-agent general problem-solving capabilities."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark for code generation, used as one of the evaluation benchmarks."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent architecture pattern (ReAct) that SupervisorAgent is designed to supervise."
     },
     {
       "title": "OWL: Optimized Workforce Learning for general multi-agent assistance in real-world task automation",
       "authors": ["Mengkang Hu", "Yuhang Zhou"],
       "year": 2025,
       "arxiv_id": "2505.23885",
       "relevance": "Multi-agent framework used as a baseline for comparison."
     },
     {
       "title": "Efficient agents: Building effective agents while reducing cost",
       "authors": ["Ningning Wang", "Xavier Hu"],
       "year": 2025,
       "arxiv_id": "2508.02694",
       "relevance": "Directly addresses agent efficiency and cost reduction, a key concern shared with this paper."
     },
     {
       "title": "MetaAgent: Automatically constructing multi-agent systems based on finite state machines",
       "authors": ["Yaolun Zhang", "Xiaogeng Liu"],
       "year": 2025,
       "arxiv_id": "2507.22606",
       "relevance": "Automatic MAS construction approach used as a baseline, representing design-time optimization."
     },
     {
       "title": "SMART: Self-aware agent for tool overuse mitigation",
       "authors": ["Cheng Qian", "Emre Can Acikgoz"],
       "year": 2025,
       "arxiv_id": "2502.11435",
       "relevance": "Addresses tool overuse in agents, related to the efficiency and intervention themes of SupervisorAgent."
     },
     {
       "title": "AgentPoison: Red-teaming LLM agents via poisoning memory or knowledge bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang"],
       "year": 2024,
       "arxiv_id": "2407.12784",
       "relevance": "Demonstrates memory poisoning vulnerabilities in LLM agents, motivating SupervisorAgent's memory interaction monitoring."
     }
   ]
 }