ai-research-survey

scan.json (28712B)

---

 {
   "paper": {
     "title": "MAESTRO: Multi-Agent Evaluation Suite for Testing, Reliability, and Observability",
     "authors": [
       "Tie Ma",
       "Yixi Chen",
       "Vaastav Anand",
       "Alessandro Cornacchia",
       "Amândio R. Faustino",
       "Guanheng Liu",
       "Shan Zhang",
       "Hongbin Luo",
       "Suhaib A. Fahmy",
       "Zafar A. Qazi",
       "Marco Canini"
     ],
     "year": 2026,
     "venue": "arXiv pre-print",
     "arxiv_id": "2601.00481"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/sands-lab/maestro (stated in Section 1, end of introduction)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (HotpotQA) and official example repositories from frameworks (ADK, AutoGen, LangGraph). The MAS instances are drawn from public sources. The paper states plans to release figures as a dataset in the appendix."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided in the paper. The paper mentions specific model names and tools (Tavily, Google Search, psutil, OpenTelemetry) but does not provide a reproducible environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the architecture and integration modes are described conceptually (Section 3.1), no step-by-step reproduction instructions (e.g., commands to run, scripts to execute) are provided in the paper itself. The GitHub repository may contain these, but the paper does not."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Boxplots with interquartile ranges are used throughout (Figures 4, 5, 6, 7, 9, 10, 12). Median and IQR values are reported for key metrics (e.g., 'interquartile range 30.6–356.6 s' in Section 4.4)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'CRAG consistently occupies the lower-cost and lower-latency region', 'Architecture dominates resource patterns') but uses no statistical significance tests (no p-values, t-tests, or bootstrap tests). Claims are based on comparing medians and visual inspection of distributions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout. For example: 'CRAG achieves a median cost of $0.0010 per task, which is more than an order of magnitude lower than both Plan-and-Execute (median $0.0126) and LATS (median $0.0101)' (Section 4.4). Also: 'median accuracy improvement of 35.7%' for CRAG with tools (Section 4.7)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'at least 20 independent runs' per MAS instance (Section 4.1) but provides no justification for why 20 runs is sufficient, no power analysis, and no acknowledgment of whether this sample size supports the comparative claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance across runs is reported through boxplots showing full distributions (Figures 4, 5, 9, 10, 12), interquartile ranges (e.g., IQR 30.6–356.6 s for Plan-and-Execute in Section 4.4), and explicit discussion of run-to-run variance (e.g., Section 4.3 on call graph stability)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares multiple MAS architectures against each other (CRAG, Plan-and-Execute, LATS) in the Architecture Suite (Section 4.4-4.7), and compares across 12 MAS instances in the Full Suite. Section 2.2 discusses limitations of existing benchmarks as motivation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The MAS instances use contemporary frameworks (ADK, AutoGen, LangGraph, MCP-Agent) and contemporary models (Gemini-2.5-Flash, GPT-5-mini, GPT-5-nano, GPT-4o-mini). The benchmark references are from 2023-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual factors while controlling others: architecture (Section 4.4), model choice (Section 4.5), and tool usage on/off (Section 4.7). This constitutes ablation-style analysis isolating contributions of each factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy, cost (monetary), latency/task duration, CPU usage, memory usage, communication volume, Jaccard similarity, LCS similarity, and failure categorization (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of MAS outputs uses automated LLM-as-judge (gpt-4o-mini) for correctness. Section 4.6 explicitly discusses limitations of LLM-as-judge failure attribution, noting divergence across judge models, but no human evaluation of MAS outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a held-out test set. Inputs are drawn from official example repositories, public datasets (HotpotQA), and synthetic inputs (Section 4.1), but there is no separation of development and test splits. The same configurations appear to be used for both tuning and reporting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by architecture (CRAG vs. Plan-and-Execute vs. LATS), by model (six different LLMs), and by tool configuration (with/without tools). Figures 9-12 provide per-architecture and per-model breakdowns. Table 3 provides per-category failure breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 is devoted to failure mode analysis. Table 3 categorizes failures (missing/underspecified output: 47.61%, wrong fact/entity: 27.66%, empty prediction: 15.96%, exception: 6.38%, timeout: 1.86%). Model-specific failure patterns are discussed in detail."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are reported: 'Accuracy degrades with increasing architectural complexity' (Section 4.4), 'Upgrading the base LLM does not reliably reduce cost or improve accuracy' (Finding 6), Plan-and-Execute with tools loses median accuracy (Section 4.7), LATS shows marginal and unstable gains with tools."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'MAS architecture is the dominant driver of resource profiles, reproducibility, and cost–latency–accuracy trade-off' are supported by Sections 4.2-4.7, and that 'MAS executions can be structurally stable yet temporally variable' is supported by Section 4.3 (Jaccard avg 0.86 vs LCS avg 0.65)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through controlled experiments that vary one factor at a time (architecture in Section 4.4, model in Section 4.5, tools in Section 4.7). The ablation design is adequate — single-variable manipulation with controlled baselines. Claims like 'tool integration mitigates speculative generation' are supported by trace-level evidence (Section 4.7)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states: 'the insights derived from our evaluation are necessarily grounded in the specific instances and configurations studied. As a result, some findings may not directly transfer to future MAS designs or to application domains not represented in our benchmark.' Findings consistently use qualifying language like 'in our setup', 'in our evaluation'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for key findings. For example, Section 4.5 explores why stronger models don't improve accuracy (execution dynamics and variance amplification rather than model capacity). Section 4.7 provides trace-level analysis explaining when tools reduce cost (reduced speculative generation). Section 4.6 discusses LLM-as-judge disagreement as a confound."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names: 'Gemini-2.0-Flash-Lite', 'Gemini-2.5-Flash-Lite', 'Gemini-2.5-Flash', 'GPT-4o-mini', 'GPT-5-mini', 'GPT-5-nano', 'gpt-oss-120b'. No snapshot dates, API versions, or exact version identifiers are provided. Per the schema, marketing names without snapshot dates do NOT count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using 'naive artifacts', 'public datasets', and 'synthetic inputs' (Section 4.1) but does not provide the actual prompts used for the MAS instances, the LLM-as-user simulation prompts, or the LLM-as-judge prompts. For human-in-the-loop simulation: 'a designated LLM generates replies conditioned on the MAS outputs' but the conditioning prompt is not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or sampling settings are reported for any of the LLMs used (backbone models, LLM-as-user, or LLM-as-judge). The paper mentions a 10-minute timeout cap and 8,192 token limit for the architecture suite, but these are execution constraints, not LLM hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the scaffolding architecture of each MAS instance: agent counts, interaction patterns (planning, coordination, debate, correction), tool configurations, and workflow structures. Figure 3 shows detailed architecture diagrams for the three Architecture Suite instances (CRAG, Plan-and-Execute, LATS). Table 2 summarizes all 12 instances."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 describes three input types (naive artifacts, public datasets, synthetic inputs) but does not document how inputs were selected from these sources, what filtering was applied, or how synthetic inputs were generated (beyond 'LLM-generated prompts'). The number of inputs per MAS instance is not clearly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 is titled 'Limitation' and discusses generalizability and telemetry overhead as specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses specific threats: 'the insights derived from our evaluation are necessarily grounded in the specific instances and configurations studied', 'advances in agent orchestration or model capabilities may invalidate certain observations over time', and telemetry overhead 'may degrade system performance'. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: 'Currently, MAESTRO only supports the adjustment of a few parameters, such as model choice and tool usage' (footnote 1), 'MAESTRO supports only pre-defined MAS instances' (footnote 2). Section 5.1 states findings 'may not directly transfer to future MAS designs or to application domains not represented in our benchmark'. Section 5.2 lists several out-of-scope areas (distributed MAS, framework overhead, parallelism effects)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper states plans to 'release them as a dataset' (Appendix A.1) for figures, but raw execution traces and telemetry data are not currently available. No data download link is provided beyond the code repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: inputs from official example repositories, public datasets (HotpotQA), and synthetic LLM-generated inputs. At least 20 independent runs per MAS instance with specified model configurations (six models listed). The observation component (Section 3.1.1) describes what data is collected during execution."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates MAS systems using automated execution and LLM-as-judge evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Section 3.1.1: MAS instances preparation -> Configuration -> Runtime execution -> Observation (telemetry collection via OpenTelemetry and psutil) -> Post-processing (aggregation and analysis). Appendix A.1 details the post-processing outputs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Author affiliations are listed (KAUST, Beihang University, MPI-SWS, LUMS) but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Beihang University, KAUST, MPI-SWS, LUMS. The footnote notes 'Work done while Tie Ma was interning at KAUST'."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Without knowing the funder, independence cannot be assessed. The paper evaluates products from Google (Gemini, ADK), OpenAI (GPT models), and Microsoft (AutoGen) — if any of these funded the work, there would be a conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLM performance on tasks (including HotpotQA for question answering accuracy), but no training data cutoff dates are stated for any of the six models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (published 2018) is used as a benchmark for QA accuracy evaluation. The models tested were trained after 2018 and may have seen HotpotQA examples in training. This potential overlap is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA has been publicly available since 2018, well before the training cutoff of all models used. The accuracy measurements (e.g., CRAG 70.6%, Plan-and-Execute 48.3%) could be inflated by contamination. This risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated MAS systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Monetary cost per task is reported extensively. For example: CRAG median $0.0010, Plan-and-Execute median $0.0126, LATS median $0.0101 (Section 4.4). Cost is a central metric throughout the evaluation, with per-model and per-architecture breakdowns in Figures 9 and 10."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-task CPU/memory usage is reported (Section 4.2), the total computational budget for the full evaluation campaign is not stated. The paper does not report total API spend, total GPU hours, or total number of runs conducted across all experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "MAS executions are structurally stable (high Jaccard similarity avg 0.86) but temporally variable (moderate LCS similarity avg 0.65).",
       "evidence": "Section 4.3, Figure 7: Jaccard similarity averages 0.86 across all examples, while LCS similarity averages 0.65. CRAG and Tree-of-Thoughts show high Jaccard but low LCS scores.",
       "supported": "strong"
     },
     {
       "claim": "MAS architecture is the dominant driver of resource consumption profiles, outweighing model choice.",
       "evidence": "Section 4.2, Figures 4-6: CRAG exhibits 9.7% avg CPU and 405.3MB avg memory, while Plan-and-Execute uses 0.07% CPU. Model choice influences CPU but has negligible impact on memory.",
       "supported": "strong"
     },
     {
       "claim": "Upgrading the base LLM does not reliably reduce cost or improve accuracy in MAS.",
       "evidence": "Section 4.5, Figure 10: gpt-5-mini and gpt-5-nano have comparable mean cost (0.033 vs 0.043) despite size differences. GPT-4o-mini achieves lower median cost ($0.0034) than both 5-series models. Accuracy shows no monotonic relationship with model capability.",
       "supported": "strong"
     },
     {
       "claim": "Specialized architectures (CRAG) simultaneously reduce resource consumption and maintain strong task performance compared to general-purpose architectures.",
       "evidence": "Section 4.4, Figure 9: CRAG achieves 70.6% average accuracy vs 48.3% for Plan-and-Execute, while CRAG's median cost ($0.0010) is over 10x lower than Plan-and-Execute ($0.0126).",
       "supported": "strong"
     },
     {
       "claim": "MAS failures predominantly manifest as silent semantic errors (75.17% of all failures).",
       "evidence": "Section 4.6, Table 3: Silent semantic failures (missing/underspecified output 47.61% + wrong fact/entity 27.66%) constitute 75.17% of failures. Only 24.84% are explicit failures (exceptions + timeouts).",
       "supported": "strong"
     },
     {
       "claim": "External tools improve accuracy only when the agent architecture can integrate them without amplifying execution overhead.",
       "evidence": "Section 4.7, Figure 12: CRAG shows 35.7% median accuracy improvement with tools. Plan-and-Execute loses median accuracy with tools. LATS shows marginal gains (4.2% median improvement) with tools, positive in only half of runs.",
       "supported": "strong"
     },
     {
       "claim": "LLM-as-judge failure attribution exhibits substantial divergence across judge models.",
       "evidence": "Section 4.6, Figure 11b: Different judge models (Gemini-2.5-Flash, GPT-4o, GPT-OSS-120B) disagree on failure categorization for identical inputs. Offline judges cannot identify system-level failures (exceptions, timeouts) without runtime signals.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "MAESTRO is an open-source evaluation suite for LLM-based multi-agent systems that standardizes configuration, execution, and telemetry collection across heterogeneous MAS architectures. Evaluation of 12 MAS instances across 6 backbone models reveals that MAS architecture is the dominant factor determining resource profiles, reproducibility, and the cost-latency-accuracy trade-off, outweighing model and tool configuration choices. The study finds that upgrading backbone models does not reliably improve MAS accuracy or reduce cost, and that 75% of MAS failures are silent semantic errors invisible without runtime telemetry. Task-specialized architectures (e.g., CRAG) achieve both lower cost and higher accuracy than general-purpose architectures.",
   "red_flags": [
     {
       "flag": "No statistical significance tests for comparative claims",
       "detail": "The paper makes numerous comparative claims about architectures, models, and tool configurations being different or dominant, but uses no statistical tests. All comparisons are based on visual inspection of boxplots and comparison of medians. With only ~20 runs per configuration, stochastic variance could explain some observed differences."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HotpotQA (published 2018) is used for accuracy evaluation of models trained well after 2018. Accuracy numbers (e.g., CRAG 70.6%) may be inflated by data contamination. This is particularly relevant since the paper's architecture-focused findings rely heavily on accuracy comparisons."
     },
     {
       "flag": "LLM-as-judge for correctness evaluation without human validation",
       "detail": "GPT-4o-mini is used as the sole evaluator for correctness, but Section 4.6 itself demonstrates that different LLM judges disagree substantially on failure attribution. No human validation of the judge's accuracy assessments is provided, introducing potential systematic bias into all accuracy-dependent findings."
     },
     {
       "flag": "Missing hyperparameter and prompt details",
       "detail": "No temperature, top-p, or sampling parameters are reported for any of the six backbone models, the LLM-as-user simulation, or the LLM-as-judge. Actual prompts are not provided. These omissions make exact reproduction impossible even with the released code."
     },
     {
       "flag": "Potential confound between architecture and task specialization",
       "detail": "The Architecture Suite finding that CRAG outperforms Plan-and-Execute and LATS may be confounded: CRAG is explicitly designed for retrieval-centric workloads and the evaluation uses HotpotQA (a retrieval-centric QA task). The paper acknowledges this (CRAG is 'optimized for retrieval-centric workloads') but still frames findings broadly about 'specialized vs. general architectures'."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentBench: evaluating LLMs as agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Major benchmark for evaluating LLM agent capabilities, directly relevant to evaluating agentic AI systems."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Taxonomy of failure modes in multi-agent LLM systems, directly relevant to reliability evaluation of agentic AI."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Foundational multi-agent framework widely used in agentic AI research."
     },
     {
       "title": "Magentic-One: A generalist multi-agent system for solving complex tasks",
       "authors": ["Adam Fourney", "Gagan Bansal", "Hussein Mozannar"],
       "year": 2024,
       "relevance": "Generalist multi-agent system from Microsoft, evaluated as a MAS instance in MAESTRO."
     },
     {
       "title": "Measuring agents in production",
       "authors": ["Melissa Z Pan", "Negar Arabzadeh", "Riccardo Cogo"],
       "year": 2025,
       "arxiv_id": "2512.04123",
       "relevance": "Survey of production MAS evaluation practices, reports 75% of teams evaluate without benchmarks."
     },
     {
       "title": "Beyond black-box benchmarking: observability, analytics, and optimization of agentic systems",
       "authors": ["Dany Moshkovich", "Hadar Mulian", "Sergey Zeltyn"],
       "year": 2025,
       "arxiv_id": "2503.06745",
       "relevance": "Observability-based benchmarking for agentic systems, directly related to MAESTRO's approach."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Foundational work on agent self-reflection strategies, relevant to agentic AI design patterns."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Tree-of-Thoughts reasoning strategy evaluated as a MAS instance in MAESTRO."
     },
     {
       "title": "Language agent tree search unifies reasoning acting and planning in language models",
       "authors": ["Andy Zhou", "Kai Yan", "Michal Shlapentokh-Rothman"],
       "year": 2023,
       "arxiv_id": "2310.04406",
       "relevance": "LATS architecture evaluated as a key MAS instance in the Architecture Suite of MAESTRO."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F Liu", "Kevin Lin", "John Hewitt"],
       "year": 2024,
       "relevance": "Cited to explain why longer interaction histories degrade MAS accuracy, relevant to understanding LLM limitations."
     },
     {
       "title": "Beyond self-talk: a communication-centric survey of LLM-based multi-agent systems",
       "authors": ["Bingyu Yan", "Zhibo Zhou", "Litian Zhang"],
       "year": 2025,
       "arxiv_id": "2502.14321",
       "relevance": "Survey of communication strategies in multi-agent LLM systems."
     },
     {
       "title": "Large language model based multi-agents: a survey of progress and challenges",
       "authors": ["Taicheng Guo", "Xiuying Chen", "Yaqi Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Comprehensive survey of LLM-based multi-agent systems covering progress and challenges."
     },
     {
       "title": "TRAIL: trace reasoning and agentic issue localization",
       "authors": ["Darshan Deshpande", "Varun Gangal", "Hersh Mehta"],
       "year": 2025,
       "arxiv_id": "2505.08638",
       "relevance": "Trace-based reasoning and issue localization for agentic systems, directly relevant to MAS observability."
     },
     {
       "title": "AgentArch: a comprehensive benchmark to evaluate agent architectures in enterprise",
       "authors": ["Tara Bogavelli", "Roshnee Sharma", "Hari Subramani"],
       "year": 2025,
       "arxiv_id": "2509.10769",
       "relevance": "Benchmark for evaluating agent architectures in enterprise settings."
     }
   ]
 }