ai-research-survey

scan.json (30202B)

---

 {
   "paper": {
     "title": "AGENTASK: Multi-Agent Systems Need to Ask",
     "authors": [
       "Bohan Lin",
       "Kuo Yang",
       "Zelin Tan",
       "Yingchuan Lai",
       "Chen Zhang",
       "Guibin Zhang",
       "Xinlei Yu",
       "Miao Yu",
       "Xu Wang",
       "Yudong Zhang",
       "Yang Wang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.07593",
     "doi": "10.48550/arXiv.2510.07593"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract mentions a 'Code' link but no working URL or GitHub repository is provided anywhere in the paper text. The word 'Code' appears at the top of the abstract without an associated URL, which does not constitute a verifiable release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 824-agent execution logs used for taxonomy annotation are not released. The benchmarks used for evaluation (GSM8K, MATH, MMLU, HumanEval, MBPP) are publicly available, but the annotated corpus from which the training data for AgentAsk was derived is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification file is referenced or provided. Model names are given (Qwen-3-4B, Llama-3.2-3B, GPT-4o-mini-0718) but no dependency list or software environment details are included."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing experiments are provided. Algorithm 1 provides pseudocode for training but this is not sufficient for a researcher to reproduce experimental results without the data, code, and environment."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviations (e.g., '94.52 ± 1.27') for the +GPT-5 and +AgentAsk conditions across five benchmarks, though the origin baselines are reported as single-run point estimates without uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '+AgentAsk improves framework-level averages by about +3.29 to +3.45 points') but does not report any statistical significance tests (no p-values, t-tests, or equivalents). Comparisons are based solely on numeric differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Improvements are reported in percentage-point differences (e.g., +3.29 to +3.45 points) relative to baselines, but no standardized effect sizes (Cohen's d, odds ratios) are reported. The improvements look modest (3-5 pp) but without reference to variance or baseline context this is hard to interpret."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper annotates 824 execution logs for the taxonomy but provides no justification for why 824 was sufficient for the claims about error distribution. The evaluation uses standard benchmarks with no sample-size justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Table 1 for +GPT-5 and +AgentAsk conditions (e.g., '94.52 ± 1.27'), providing spread information for those results. However, the origin baselines report single point estimates without variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: single-model prompting (IO, CoT, Self-Refine) and four multi-agent frameworks (GPTSwarm, AFlow, MaAS, MasRouter). AgentAsk is compared against both the unmodified frameworks and a +GPT-5 upper bound."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent baselines (GPTSwarm, AFlow, MaAS, MasRouter) are all recent papers from 2024-2025. The paper also compares against a +GPT-5 upper bound clarifier. These are competitive contemporary baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 3 present ablation experiments removing individual reward components (rpar, reff, rfmt) and comparing to R-only training. Table 9 compares E-GRPO against PPO and GRPO across four frameworks and five benchmarks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses accuracy (for GSM8K, MATH, MMLU) and Pass@1 (for HumanEval, MBPP), plus latency and extra cost as efficiency metrics. Multiple dimensions are assessed simultaneously."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of model outputs is not relevant here; the paper evaluates on automated benchmarks with objective metrics (accuracy, Pass@1). Human annotators are used to build the taxonomy, not to evaluate AgentAsk's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not clarify whether the benchmark splits used for evaluation are held-out test sets or whether any tuning decisions were made on the same splits. The training procedure uses multi-agent execution logs, but the relationship between training data and evaluation splits is not made explicit."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (GSM8K, MATH, HumanEval, MMLU, MBPP) and per framework (GPTSwarm, AFlow, MaAS, MasRouter). Section 5.5 additionally breaks down performance by error type (Data Gap, Signal Corruption, Referential Drift, Capability Gap)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses error types that AgentAsk handles less effectively: 'For tasks involving more Referential Drift and Capability Gap...the improvements are more limited.' The Limitations section also discusses AgentAsk's inability to handle errors arising from internal model limitations like hallucinations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows that AgentAsk with Qwen-3-4B SFT on MaAS slightly decreases GSM8K accuracy (-0.20 pp) and Table 6 shows a -0.08 pp MMLU decrease, which are reported transparently. Resolved@Edge rates also reveal limitations for Referential Drift (58.3%) and Capability Gap (49.5%) error types."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'improving accuracy by up to 4.69%' which is confirmed in Table 1 (MaAS +GPT-5 achieves 84.71, +4.69 over 80.02). The claim of 'latency and extra costs below 10%' is supported by efficiency data in Tables 2 and 5-7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies in Section 5.4 provide controlled single-variable manipulation to support causal claims about each reward component's contribution. Removing individual components (rpar, reff, rfmt) consistently degrades performance, supporting causal interpretation. The plug-and-play framework design holds orchestration fixed, isolating the effect of the clarifier."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses language like 'edge-level issues are a common source of failure' and 'improving the reliability of multi-agent systems' broadly, but evaluation is limited to five benchmarks (math reasoning, QA, code generation) with a specific executor (GPT-4o-mini-0718) and four specific frameworks. Claims about general applicability to arbitrary MAS settings are not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes gains specifically to edge-level clarification but does not substantively discuss alternative explanations, such as whether gains might partly be due to simply calling the LLM an additional time (not necessarily for clarification), increased token budget, or the effect of adding any additional processing step. The +GPT-4o-mini baseline partially addresses this but is not framed as an alternative explanation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-4o-mini-0718' (with version date) and 'Qwen-3-4B' and 'Llama-3.2-3B' as backbone models. GPT-5 is referenced by an OpenAI URL accessed 2025-08-07. These are sufficiently specific."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F (Figure 7) provides the full edge-level clarifier prompt with the complete instruction text including the four error types (DG, SC, RD, CG), when to ask, what to ask, who to ask, and the NONE condition. This is the actual prompt text sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions temperature=0.3 in the robustness experiments (Appendix C.1) but does not provide a complete hyperparameter table. Key RL training hyperparameters such as learning rate, number of training steps, lambda_ask, lambda_sw, alpha_eff, alpha_fmt, epsilon, and beta are referenced in equations but their actual values are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "AgentAsk's scaffolding is described in detail: Sections 4.1-4.3 describe the edge-local state, action space, policy structure, two-stage training (SFT then RL), and reward design. Algorithm 1 provides complete pseudocode for the training procedure. The interaction graph structure is also described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The training corpus is described as being built 'from a set of logged multi-agent executions' with labels assigned by 'a teacher model,' but the paper does not document how execution logs were collected, filtered, or preprocessed. The number of training examples N is referenced but not stated. The criteria for including or excluding execution logs are not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the Conclusion that discusses that AgentAsk cannot eliminate errors arising from internal model limitations like hallucinations and inconsistencies."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is largely generic: it states AgentAsk 'cannot fully eliminate all errors' arising from 'well-known limitations in large models, such as hallucinations.' This does not constitute specific threats to validity for the experimental results, such as potential contamination of benchmarks, selection of favorable frameworks, or confounds in the efficiency measurements."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no explicit discussion of settings where AgentAsk would not work, tasks or domains not covered, or limitations of the benchmark choice. The limitation paragraph discusses general model limitations rather than scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 824 annotated execution logs that form the basis of the taxonomy and the training corpus are not made available. Without access to these logs, the taxonomy distribution claims (Data Gap 29.1%, Referential Drift 27.3%, Signal Corruption 36.8%, Capability Gap 6.8%) cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes auditing '824 execution logs' but does not explain how these logs were collected: from which frameworks, which tasks, over what time period, or with what inclusion/exclusion criteria. Appendix A.2 states logs come 'from a set of logged multi-agent executions' without further detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper involves annotators for the taxonomy but not participant recruitment in the human-subjects sense. The annotators are described as 'multiple professional annotators with expertise in MAS' but this is annotation quality control, not participant recruitment for a study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from execution logs to training data is not fully documented. We know logs are annotated by a teacher model into (type, agent, question) triples, but the filtering criteria, the size of the training dataset, and any preprocessing steps between raw logs and the SFT dataset are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section is present in the paper, and no funding sources are disclosed. The author affiliations (USTC, Shanghai AI Lab, Xi'an Jiaotong, NUS) are listed but no grants or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed on the title page: University of Science and Technology of China, Shanghai AI Laboratory, Xi'an Jiaotong University, and National University of Singapore. No author is affiliated with OpenAI or DeepSeek, whose models are used."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed despite authors being affiliated with major research institutions (USTC, Shanghai AI Lab, NUS) where unfunded research is implausible. Since funding is not disclosed, funder independence cannot be assessed. The schema says 'NA if unfunded' but the absence of a funding disclosure from well-resourced institutions is not the same as confirmed unfunded status."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement anywhere in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4o-mini-0718 and GPT-5 on benchmarks including HumanEval and MMLU but does not state the training cutoff dates for these models. GPT-5 is referenced with an OpenAI URL accessed 2025-08-07 but no training cutoff is provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses well-known public benchmarks (HumanEval published 2021, MMLU published 2021, GSM8K 2021) with models that were almost certainly trained on data that includes these benchmarks. No discussion of potential train/test overlap is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MMLU were published in 2021, well before any of the evaluated models were trained. The paper does not discuss whether these benchmarks may be in the models' training data, which is a significant concern for evaluating true reasoning ability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "While annotators are used for the taxonomy classification, this is not a human subjects study. The annotators classify agent execution logs, not human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human participants in a research sense. Professional annotators classify execution logs but this is not a human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the research sense. The 'professional annotators with expertise in MAS' are not research participants whose demographics need to be reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants whose inclusion/exclusion criteria would need to be specified."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment with conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants who could drop out of a study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 and subsequent tables report 'Extra Cost' as a relative normalized metric (with origin=0 as baseline). The paper reports that AgentAsk achieves extra costs of 4.9-7.7 on the normalized scale while +GPT-5 costs 24-44 units. However, these are relative rather than absolute API costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, total API spend, or training compute are reported. The paper describes that AgentAsk is trained with SFT followed by RL (E-GRPO) but provides no quantification of the compute required for this training."
       }
     }
   },
   "claims": [
     {
       "claim": "AgentAsk consistently improves accuracy by up to 4.69% across five benchmarks and four multi-agent frameworks while keeping latency and extra costs below 10% compared to baseline MAS.",
       "evidence": "Table 1 shows framework-level average improvements of +3.29 to +3.45 percentage points for +AgentAsk across four frameworks. The maximum individual gain is +4.69 (MaAS+GPT-5 on average). Table 2 shows extra cost of 4.9 for AgentAsk vs. 34.0 for +GPT-5.",
       "supported": "strong"
     },
     {
       "claim": "An audit of 824 execution logs reveals four dominant error types: Data Gap (29.1%), Signal Corruption (36.8%), Referential Drift (27.3%), and Capability Gap (6.8%).",
       "evidence": "Section 3.5 states these percentages from the annotated corpus (N=824). Fleiss' Kappa of 0.84 is reported for inter-rater reliability. Figure 2 shows the empirical distribution.",
       "supported": "moderate"
     },
     {
       "claim": "AgentAsk achieves near upper-bound (+GPT-5) accuracy at substantially lower cost, with only a 0.24 point accuracy gap despite 21-point reduction in latency and 29.1-point reduction in extra cost.",
       "evidence": "Table 2 (MasRouter@GSM8K): +GPT-5 achieves 95.10 accuracy at latency 129 and extra cost 34.0; +AgentAsk achieves 94.86 at latency 108 and extra cost 4.9.",
       "supported": "strong"
     },
     {
       "claim": "Each reward component (rpar, reff, rfmt) is necessary for optimal performance in AgentAsk.",
       "evidence": "Table 3 shows that removing each component individually leads to accuracy decreases and latency/cost increases. E.g., removing reff drops accuracy from 83.55 to 80.05 (-3.50 pp) on MBPP.",
       "supported": "strong"
     },
     {
       "claim": "AgentAsk is architecture-agnostic and can be transferred across frameworks: trained on one framework and then integrated into others.",
       "evidence": "Section 5.1 states 'we trained AgentAsk on one framework and then gradually integrated it into other frameworks.' Tables 1 and 4 show consistent performance across all four evaluated frameworks.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "AgentAsk proposes a lightweight edge-level clarification module for multi-agent LLM systems that intervenes at message handoffs to prevent error propagation. Based on an annotation of 824 execution logs, the authors identify four error types (Data Gap, Signal Corruption, Referential Drift, Capability Gap) and train AgentAsk with SFT followed by a novel E-GRPO reinforcement learning objective to balance clarification quality against cost. Evaluated across five benchmarks and four MAS frameworks, AgentAsk improves average accuracy by 3.29-3.45 percentage points while keeping extra cost below 10% of baseline, recovering most of the gain achievable with the much more expensive GPT-5 clarifier.",
   "red_flags": [
     {
       "flag": "No training hyperparameters reported",
       "detail": "The paper uses SFT followed by E-GRPO but does not report key hyperparameter values (learning rate, lambda_ask, alpha_eff, alpha_fmt, epsilon, beta, number of training steps/epochs). Without these, the method cannot be reproduced."
     },
     {
       "flag": "Training data not released or fully described",
       "detail": "The 824-execution-log corpus used for taxonomy annotation and the SFT training dataset are neither released nor fully described. The number of training examples N is referenced but never stated. Without this data, training AgentAsk from scratch is not possible."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "The paper evaluates on HumanEval (published 2021), MMLU (2021), and GSM8K (2021) using GPT-4o-mini and GPT-5, which were almost certainly trained on data containing these benchmarks. The paper does not discuss this contamination risk at all."
     },
     {
       "flag": "Relative cost units lack absolute grounding",
       "detail": "Latency and extra cost are reported as relative normalized values (origin=100), making it impossible to determine actual API costs or wall-clock times. This obscures whether AgentAsk is practically affordable at scale."
     },
     {
       "flag": "Taxonomy annotation corpus opacity",
       "detail": "The 824 execution logs are described as coming from unspecified frameworks and tasks. The collection procedure, selection criteria, and representativeness of this corpus are not described, making it impossible to assess whether the reported error type distributions generalize."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper has no acknowledgments section and discloses no funding sources, making it impossible to assess potential conflicts of interest."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": [
         "Mert Cemri",
         "Melissa Z Pan",
         "Shuyi Yang",
         "Lakshya A Agrawal",
         "Kurt Keutzer",
         "Matei Zaharia",
         "Joseph E. Gonzalez",
         "Ion Stoica"
       ],
       "year": 2025,
       "relevance": "Large-scale audit of multi-agent LLM system failures, directly motivating the problem studied in AgentAsk and providing empirical evidence that MAS often fail to outperform strong single-agent baselines."
     },
     {
       "title": "GPTSwarm: Language agents as optimizable graphs",
       "authors": [
         "Mingchen Zhuge",
         "Wenyi Wang",
         "Louis Kirsch",
         "Francesco Faccio",
         "Dmitrii Khizbullin",
         "Jürgen Schmidhuber"
       ],
       "year": 2024,
       "relevance": "One of four multi-agent frameworks used as baselines in AgentAsk evaluation, treating agents and communication links as an optimizable graph."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": [
         "Jiayi Zhang",
         "Jinyu Xiang",
         "Zhaoyang Yu",
         "Fengwei Teng",
         "Jiaqi Chen",
         "Mingchen Zhuge",
         "Bang Liu",
         "Chenglin Wu"
       ],
       "year": 2025,
       "arxiv_id": "2501.08944",
       "relevance": "One of four multi-agent frameworks used as baselines, searching over code-represented workflows via Monte Carlo tree search."
     },
     {
       "title": "Multi-agent architecture search via agentic supernet",
       "authors": [
         "Guibin Zhang",
         "Luyang Niu",
         "Junfeng Fang",
         "Kun Wang",
         "Lei Bai",
         "Xiang Wang"
       ],
       "year": 2025,
       "relevance": "One of four multi-agent frameworks used as baselines (MaAS), sampling query-dependent sub-architectures from an agentic supernet."
     },
     {
       "title": "MasRouter: Learning to route LLMs for multi-agent systems",
       "authors": [
         "Yanwei Yue",
         "Guibin Zhang",
         "Boyang Liu",
         "Guancheng Wan",
         "Kun Wang",
         "Dawei Cheng",
         "Yiyan Qi"
       ],
       "year": 2025,
       "relevance": "One of four multi-agent frameworks used as baselines in evaluation, learning a cascaded controller for collaboration mode, role allocation, and LLM selection."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": [
         "Sirui Hong",
         "Mingchen Zhuge",
         "Jonathan Chen",
         "Xiawu Zheng",
         "Yuheng Cheng",
         "Jinlin Wang",
         "Chenglin Wu",
         "Jürgen Schmidhuber"
       ],
       "year": 2024,
       "relevance": "Early influential MAS framework organizing multiple LLMs into role-specialized teams with structured workflows, representing the genre of systems that AgentAsk targets."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": [
         "Chen Qian",
         "Wei Liu",
         "Hongzhang Liu",
         "Nuo Chen",
         "Zhiyuan Liu",
         "Maosong Sun"
       ],
       "year": 2024,
       "relevance": "Multi-agent framework for software development that demonstrates collaborative LLM-based programming, in scope as an example of MAS evaluated in related work."
     },
     {
       "title": "Reflexion: language agents with verbal reinforcement learning",
       "authors": [
         "Noah Shinn",
         "Federico Cassano",
         "Ashwin Gopinath",
         "Karthik Narasimhan",
         "Shunyu Yao"
       ],
       "year": 2023,
       "relevance": "Self-feedback loop approach for improving agent reliability that AgentAsk is compared against conceptually as an alternative method for error correction."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": [
         "Xiao Liu",
         "Hao Yu",
         "Hanchen Zhang",
         "Yifan Xu",
         "Xuanyu Lei",
         "Yu Gu",
         "Tianjun Zhang",
         "Yu Su",
         "Huan Sun"
       ],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLMs as agents, representing the broader benchmark evaluation methodology relevant to this survey."
     },
     {
       "title": "Talk isn't always cheap: Understanding failure modes in multi-agent debate",
       "authors": [
         "Andrea Wynn",
         "Harsh Satija",
         "Gillian Hadfield"
       ],
       "year": 2025,
       "arxiv_id": "2509.05396",
       "relevance": "Analysis of conditions where multi-agent debate can fail or degrade accuracy, directly relevant to understanding MAS reliability."
     },
     {
       "title": "Which agent causes task failures and when? On automated failure attribution of LLM multi-agent systems",
       "authors": [
         "Shaokun Zhang",
         "Ming Yin",
         "Jieyu Zhang",
         "Jiale Liu",
         "Chi Wang",
         "Qingyun Wu"
       ],
       "year": 2025,
       "relevance": "Automated failure attribution methodology for LLM multi-agent systems, complementary to the error taxonomy developed in AgentAsk."
     },
     {
       "title": "MultiAgentBench: Evaluating the collaboration and competition of LLM agents",
       "authors": [
         "Kunlun Zhu",
         "Hongyi Du",
         "Zhaochen Hong",
         "Xiaocheng Yang",
         "Shuyi Guo",
         "Heng Ji",
         "Jiaxuan You"
       ],
       "year": 2025,
       "arxiv_id": "2503.01935",
       "relevance": "Benchmark for evaluating LLM agent collaboration and competition, relevant to the broader multi-agent evaluation literature this paper contributes to."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": [
         "DeepSeek-AI",
         "Daya Guo",
         "Dejian Yang"
       ],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "The GRPO optimization method from DeepSeek-R1 is the foundation for AgentAsk's E-GRPO training algorithm."
     }
   ]
 }