ai-research-survey

scan.json (28743B)

---

 {
   "paper": {
     "title": "Where LLM Agents Fail and How They Can Learn from Failures",
     "authors": [
       "Kunlun Zhu",
       "Zijia Liu",
       "Bingxuan Li",
       "Muxin Tian",
       "Yingxuan Yang",
       "Jiaxun Zhang",
       "Pengrui Han",
       "Qipeng Xie",
       "Fuyang Cui",
       "Weijia Zhang",
       "Xiaoteng Ma",
       "Xiaodong Yu",
       "Gowtham Ramesh",
       "Jialian Wu",
       "Zicheng Liu",
       "Pan Lu",
       "James Zou",
       "Jiaxuan You"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.25370"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'The code and data will be available at https://github.com/ulab-uiuc/AgentDebug', which is a promise of future release. Per the schema criteria, a promise of future release counts as NO."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises data availability at the same GitHub URL as the code, but this is a future release promise, not a currently accessible dataset. No evidence that the AgentErrorBench dataset is currently downloadable."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification file is mentioned. The paper specifies models used (GPT-4.1, GPT-4o-mini, Qwen3-8B, Qwen3-Next-80B) and temperature (0 for GPT-4.1), but provides no dependency specifications to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Algorithm 1 describes the method at a pseudocode level but not the commands or scripts needed to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and Figures 5-7 report only point estimates (percentage accuracy, success rates) with no confidence intervals or error bars. For example, Table 1 reports 'AgentDebug: 45.0% step accuracy' with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The paper claims AgentDebug 'consistently surpasses baselines' (Section 4.1) based solely on comparing point estimates without any p-values, t-tests, or other statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout: '24.3% vs. 0.3% all-correct' (Table 1), '45.0% vs. 28.0% step accuracy' (Table 1), 'success from 21 to 55' (Section 4.2). Per the schema, 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper consistently provides this form of context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 200 annotated trajectories (100 ALFWorld, 50 WebShop, 50 GAIA), but no power analysis or justification for why this sample size is sufficient for the claims made is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results across runs are reported as single point estimates. Figure 7a shows accumulative success rates across attempts but these are cumulative counts, not variance or standard deviation across repeated experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 includes three baselines for error detection: Direct Prompting, Brute Force, and Binary Search. Section 4.2 includes Self-Refine, Vanilla Debugger, Tree-of-Thought (ToT), and Best-of-N as baselines for downstream task success."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: Self-Refine and Tree-of-Thought are recent methods, and the paper explicitly states it controls for compute budget to ensure fair comparison. The baselines are appropriate for the task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 presents ablation studies analyzing: (1) max number of attempts, (2) different base models for AgentDebug, and (3) different rollout strategies (ReAct, Reflection, Act-only, Memory+ReAct vs Modular)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports three metrics for error detection: Step accuracy (S), Step+Module accuracy (S+M), and All Correct (Step+Module+Error Type). Task success rate is also reported separately in Section 4.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark was annotated by human experts (10 graduate students), but there is no human evaluation of AgentDebug's outputs. Human annotation was used for dataset construction, not for evaluating the system's outputs. All system evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "AgentErrorBench is used as the test set for evaluating error detection (Section 4.1). The downstream task evaluation uses ALFWorld, GAIA, and WebShop benchmarks as separate test environments. The paper does not describe any tuning decisions made using these sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 and Figures 5-6 break down results per benchmark (ALFWorld, WebShop, GAIA) separately. Appendix A.4 provides failure distribution breakdowns per module, error type, and benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses error propagation patterns in detail. Appendix A.3 provides a concrete qualitative failure case comparison. Figure 8 illustrates cascading failure patterns across trajectory steps."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7b (ablation) shows that alternative base models (Llama-3.3-70B, GPT-4o-mini, Qwen3-Next-80B) perform substantially worse as detectors. The paper also reports that Brute Force performs worse than Direct Prompting in some metrics."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 24% higher all-correct accuracy and 17% higher step accuracy, which are confirmed by Table 1 (24.3% vs 0.3% all-correct; 45.0% vs 28.0% step accuracy). The 26% relative improvement in task success is shown in Figures 5-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'targeting root-cause errors...is key to efficient debugging' and 'correcting a single root-cause mistake can often flip an otherwise failing trajectory.' These are tested via ablation but there is no controlled single-variable manipulation — the comparison is between full methods, not isolated components, making causal attribution difficult."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's conclusion states 'AgentDebug establishes debugging as a foundation for agents that can continuously learn and evolve,' which overgeneralizes from three specific benchmarks (ALFWorld, WebShop, GAIA). The title 'Where LLM Agents Fail' suggests broader scope than what is tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section (Appendix A.1) discusses scale and domain diversity but does not consider alternative explanations for the results — for example, whether gains could be due to additional compute (though they control for this) or whether GPT-4.1 specifically enables the gains rather than the framework design."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'We use GPT-4.1 as the base model' and Section 4.2 mentions 'GPT-4o-mini, Qwen3-8B, and Qwen3-Next-80B.' None of these include specific version snapshots, API versions, or dates, which is necessary since model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 provides verbatim prompt content for the Detector Prompt (Figure 14), AgentDebug Prompt (Figure 15), Baseline Prompts (Figure 16), and Environment Rollout Prompts for ALFWorld (Figures 16-17). The actual prompt text with placeholders is provided along with the fill values described in context."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'temperature set to 0 for deterministic outputs' for GPT-4.1, but other hyperparameters (temperature for other models, max tokens, top-p) are not reported. For the backbone agents used in Section 4.2, no sampling hyperparameters are given."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Algorithm 1 describe AgentDebug's three-stage scaffolding in detail: Stage 1 (fine-grained analysis), Stage 2 (critical error detection), Stage 3 (iterative debugging with re-rollouts). The modular rollout design with memory, reflection, planning, and action modules is described in Section 3.1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.2 states 200 representative trajectories were 'curated' from 500+ failed trajectories, but the selection criteria for which 200 were included from the 500+ are not explained. The paper does not describe filtering criteria for how the 200 representative trajectories were selected."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 is titled 'LIMITATION' and provides a substantive discussion of limitations, including scale/domain diversity constraints and the cost of annotation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 identifies specific threats: (1) the benchmark 'remains limited in scale and domain diversity' specifically naming multimodal environments and safety-critical applications, and (2) the annotation cost constraint preventing a trained debugging model. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions scope boundaries only vaguely. The paper does not explicitly state what the results do NOT show — for example, it does not bound claims to the specific models tested or acknowledge that results may not apply to multi-agent settings or different error taxonomies."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises future data availability at GitHub but AgentErrorBench (the 200 annotated trajectories) is not currently available for independent verification. The 500+ failed trajectories used for taxonomy development are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes that 200 representative trajectories were collected: 100 from ALFWorld, 50 from WebShop, 50 from GAIA, and how annotation proceeded at the decision-step level using the taxonomy schema."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes that 'ten expert annotators—graduate students with prior experience in NLP and LLMs agent research' performed the annotations. The qualification criteria (graduate students with relevant experience) are stated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper collected 500+ failed trajectories but only used 200 in the benchmark without explaining the selection criteria. The pipeline from 500+ to 200 is not documented with explicit filtering criteria, which represents an unexplained jump."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant source is mentioned in the paper. The acknowledgment section thanks the OpenManus team for 'discussion and providing some resources' but does not disclose any funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the first page: University of Illinois Urbana-Champaign, Stanford University, AMD, OpenManus, University of Toronto, and Likelihood Lab. The paper does not evaluate any of these institutions' products specifically."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed; this question does not apply since there is no identified funder whose independence can be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or financial interests declaration in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4.1, GPT-4o-mini, Qwen3-8B, and Qwen3-Next-80B on benchmarks including GAIA and WebShop, but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether ALFWorld, GAIA, or WebShop benchmarks were in the training data of the evaluated models is provided. GAIA was published in 2023 and WebShop in 2022, making contamination plausible for models with later training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld (2020), WebShop (2022), and GAIA (2023) are public benchmarks that likely appeared in training data for GPT-4.1 and Qwen3 series models. The paper does not discuss this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This paper involves human annotators for benchmark construction but is not a human subjects study in the experimental sense; it is a benchmark and system evaluation paper. No pre-registration is relevant or expected."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The ethical statement explicitly states 'Our study does not involve human subjects, sensitive personal data, or information that could directly identify individuals.' The graduate student annotators are not study subjects; IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are described as 'graduate students with prior experience in NLP and LLMs agent research,' which is sufficient characterization for their role as annotators rather than study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense; the annotation is a data construction process, not a human study. Human studies items are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental conditions involving human subjects requiring randomization. Human studies items are not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment requiring blinding. Human studies items are not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense. Human studies items are not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "AgentDebug calls LLMs for per-step error analysis plus critical error detection per trajectory. With 200 trajectories and multiple re-rollouts, this involves substantial API costs. The paper does not report API costs, tokens consumed, or cost per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper controls for total token usage between methods (Section 4.2: 'the max number of attempts of all baselines is matched to AgentDebug by total token usage'), but does not state the absolute compute budget in terms of total tokens, GPU hours, or API spend."
       }
     }
   },
   "claims": [
     {
       "claim": "AgentDebug achieves 24% higher all-correct accuracy (24.3% vs 0.3%) and 17% higher step accuracy (45.0% vs 28.0%) compared to the strongest baseline on AgentErrorBench.",
       "evidence": "Table 1 reports average all-correct accuracy of 24.3% for AgentDebug vs 0.3% for Direct Prompting (strongest baseline), and step accuracy of 45.0% vs 28.0%. Section 4.1 presents these results.",
       "supported": "strong"
     },
     {
       "claim": "AgentDebug enables LLM agents to achieve up to 26% relative improvements in task success across ALFWorld, GAIA, and WebShop.",
       "evidence": "Figure 5 shows ALFWorld improvements: GPT-4o-mini from 21 to 55 (162% improvement), Qwen3-8B from 48 to 74, Qwen3-Next-80B from 60 to 84. Figure 6 shows cross-benchmark comparisons. The 26% figure appears to refer to relative improvement on ALFWorld specifically.",
       "supported": "moderate"
     },
     {
       "claim": "Error propagation — early mistakes cascading into later failures — is the primary bottleneck in LLM agent reliability.",
       "evidence": "Section 2.1 states this as a 'Key Insight' derived from analyzing 500+ failed trajectories. Figure 8 illustrates cascading failures. Section 5.2 provides additional analysis showing memory and reflection errors cause most propagation.",
       "supported": "moderate"
     },
     {
       "claim": "Human annotators achieved substantial inter-annotator agreement (Cohen's kappa = 0.55) on the AgentErrorTaxonomy categories.",
       "evidence": "Section 2.2 explicitly reports: 'Inter-annotator agreement, measured using Cohen's κ, reached κ = 0.55 across modules, indicating substantial agreement.'",
       "supported": "strong"
     },
     {
       "claim": "GPT-4.1 substantially outperforms other models as the AgentDebug base model, achieving 42% step accuracy vs 16% for Llama-3.3-70B.",
       "evidence": "Figure 7b (Table) shows GPT-4.1: 42% step, 32% all-correct; Llama-3.3-70B: 16% step, 2% all-correct. Section 5.1 reports this ablation.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "The paper introduces AgentErrorTaxonomy (a five-module failure taxonomy covering memory, reflection, planning, action, and system errors), AgentErrorBench (200 annotated failure trajectories from ALFWorld, GAIA, and WebShop), and AgentDebug (a three-stage debugging framework that localizes root-cause errors and provides corrective feedback). AgentDebug achieves 24.3% all-correct accuracy on error localization versus 0.3% for baselines, and improves task success rates by up to 26% relative gains across three benchmarks. A key finding is that error propagation — where early mistakes cascade into downstream failures — is the central reliability bottleneck, with memory and reflection errors being the most common sources of cascading failure.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons between AgentDebug and baselines are based on point estimates with no confidence intervals, standard deviations, or significance tests. With only 50-100 trajectories per benchmark, differences could be within noise bounds."
     },
     {
       "flag": "Model version not specified",
       "detail": "The paper uses 'GPT-4.1' and 'GPT-4o-mini' without API version numbers or snapshot dates. Since OpenAI updates these models, results may not be reproducible."
     },
     {
       "flag": "Selection bias in benchmark construction",
       "detail": "200 trajectories were 'curated' from 500+ failed trajectories without documenting the selection criteria. If the 200 were selected to make the taxonomy look clean, the benchmark may not represent the true distribution of agent failures."
     },
     {
       "flag": "Contamination not addressed",
       "detail": "ALFWorld (2020), WebShop (2022), and GAIA (2023) are public benchmarks that likely appeared in GPT-4.1 and Qwen3 training data, but contamination risk is not discussed."
     },
     {
       "flag": "Code and data promised but not released",
       "detail": "The abstract promises code and data availability at GitHub, but as a future release. Results cannot be independently verified at time of publication."
     },
     {
       "flag": "Overgeneralized conclusion",
       "detail": "The conclusion frames AgentDebug as 'a foundation for agents that can continuously learn and evolve from their mistakes,' which significantly overstates what was shown: improvement on 3 specific benchmarks with a single model (GPT-4.1) as the debugger."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik Narasimhan",
         "Yuan Cao"
       ],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational paper on the reasoning-acting paradigm for LLM agents, directly relevant to understanding agent failure modes."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": [
         "Shunyu Yao",
         "Dian Yu",
         "Jeffrey Zhao",
         "Izhak Shafran",
         "Thomas L. Griffiths",
         "Yuan Cao",
         "Karthik Narasimhan"
       ],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Test-time scaling method for LLM reasoning used as baseline for agent debugging; relevant to evaluating agent methodology quality."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": [
         "Noah Shinn",
         "Federico Cassano",
         "Edward Berman",
         "Ashwin Gopinath",
         "Karthik Narasimhan",
         "Shunyu Yao"
       ],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Self-reflection method for LLM agents that is a key baseline and related work for agent error recovery."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": [
         "Gregoire Mialon",
         "Clementine Fourrier",
         "Thomas Wolf",
         "Yann LeCun",
         "Thomas Scialom"
       ],
       "year": 2023,
       "relevance": "Benchmark used to evaluate AgentDebug; relevant as an evaluation environment for LLM agent capabilities."
     },
     {
       "title": "WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents",
       "authors": [
         "Shunyu Yao",
         "Howard Chen",
         "John Yang",
         "Karthik Narasimhan"
       ],
       "year": 2022,
       "relevance": "E-commerce agent benchmark used to evaluate AgentDebug; important for evaluating web interaction agent methods."
     },
     {
       "title": "ALFWorld: Aligning Text and Embodied Environments for Interactive Learning",
       "authors": [
         "Mohit Shridhar",
         "Xingdi Yuan",
         "Marc-Alexandre Cote",
         "Yonatan Bisk",
         "Adam Trischler",
         "Matthew Hausknecht"
       ],
       "year": 2020,
       "arxiv_id": "2010.03768",
       "relevance": "Embodied agent benchmark used to evaluate AgentDebug; key testbed for long-horizon task completion."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": [
         "Mert Cemri",
         "Melissa Z. Pan",
         "Shuyi Yang",
         "Lakshya A. Agrawal",
         "Bhavya Chopra",
         "Rishabh Tiwari",
         "Kurt Keutzer",
         "Aditya G. Parameswaran",
         "Dan Klein",
         "Kannan Ramchandran",
         "Matei Zaharia",
         "Joseph E. Gonzalez",
         "Ion Stoica"
       ],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Related work on failure analysis in multi-agent LLM systems; directly relevant to understanding agent error taxonomy scope."
     },
     {
       "title": "Defining and Detecting the Defects of the Large Language Model-Based Autonomous Agents",
       "authors": [
         "Kaiwen Ning",
         "Jiachi Chen",
         "Jingwen Zhang",
         "Wei Li",
         "Zexu Wang",
         "Yuming Feng",
         "Weizhe Zhang",
         "Zibin Zheng"
       ],
       "year": 2024,
       "arxiv_id": "2412.18371",
       "relevance": "Prior work on agent defect taxonomy that this paper builds upon and differentiates from."
     },
     {
       "title": "VeriLA: A Human-Centered Evaluation Framework for Interpretable Verification of LLM Agent Failures",
       "authors": [
         "Yoo Yeon Sung",
         "Hannah Kim",
         "Dan Zhang"
       ],
       "year": 2025,
       "arxiv_id": "2503.12651",
       "relevance": "Related work on human-centered evaluation of LLM agent failures, relevant to the survey's interest in methodology quality."
     },
     {
       "title": "Testing and Understanding Erroneous Planning in LLM Agents through Synthesized User Inputs",
       "authors": [
         "Zhenlan Ji",
         "Daoyuan Wu",
         "Pingchuan Ma",
         "Zongjie Li",
         "Shuai Wang"
       ],
       "year": 2024,
       "arxiv_id": "2404.17833",
       "relevance": "Related work on LLM agent planning failures that informed this paper's taxonomy development."
     },
     {
       "title": "Which Agent Causes Task Failures and When? On Automated Failure Attribution of LLM Multi-Agent Systems",
       "authors": [
         "Shaokun Zhang",
         "Ming Yin",
         "Jieyu Zhang",
         "Jiale Liu",
         "Zhiguang Han",
         "Jingyang Zhang",
         "Beibin Li",
         "Chi Wang",
         "Huazheng Wang",
         "Yiran Chen",
         "Qingyun Wu"
       ],
       "year": 2025,
       "arxiv_id": "2505.00212",
       "relevance": "Related work on failure attribution in multi-agent systems, relevant to agent debugging methodology."
     },
     {
       "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-World APIs",
       "authors": [
         "Yujia Qin",
         "Shihao Liang",
         "Yining Ye",
         "Kunlun Zhu"
       ],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Tool-use LLM agent framework relevant to understanding tool-use errors in the agent error taxonomy."
     }
   ]
 }