ai-research-survey

scan.json (28835B)

---

 {
   "paper": {
     "title": "Towards a Science of Scaling Agent Systems",
     "authors": ["Yubin Kim", "Ken Gu", "Chanwoo Park", "Chunjong Park", "Samuel Schmidgall", "A. Ali Heydari", "Yao Yan", "Zhihan Zhang", "Yuchen Zhuang", "Yun Liu", "Mark Malhotra", "Paul Pu Liang", "Hae Won Park", "Yuzhe Yang", "Xuhai Xu", "Yilun Du", "Shwetak Patel", "Tim Althoff", "Daniel McDuff", "Xin Liu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.08296",
     "doi": "10.48550/arXiv.2512.08296"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Multi-agent system performance is highly task-dependent, ranging from +80.9% improvement on parallelizable financial reasoning tasks to -70% degradation on sequential planning tasks. A predictive mixed-effects model (R²=0.524) using coordination metrics achieves 87% accuracy in selecting optimal architectures for held-out configurations. Three dominant effects identified: tool-coordination trade-off, capability saturation above ~45% single-agent baseline, and architecture-dependent error amplification (17.2× for independent vs 4.4× for centralized). Out-of-sample validation on GPT-5.2 confirms four of five scaling principles generalize.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: Finance-Agent, BrowseComp-Plus, PlanCraft, and WorkBench, all cited with references and publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LiteLLM and LangChain (Appendix E) but does not provide requirements.txt, Dockerfile, or detailed dependency versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Appendix E describes the setup conceptually but not with executable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for regression coefficients (Table 4), bootstrap SEs, and error amplification factors (e.g., '95% CI: [14.3, 20.1]')."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "p-values reported throughout for regression coefficients (Table 4), t-tests for turn count differences (e.g., t(178)=16.8, p<0.001), Shapiro-Wilk and Breusch-Pagan tests for residual diagnostics."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standardized regression coefficients reported (e.g., β̂=-0.267), Cohen's d referenced in Figure 6 caption (d>1.2, d≈0.35), and percentage improvements with baselines throughout (e.g., '+80.8% mean 0.631 vs. SAS 0.349')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=180 configurations is stated but no justification for why this number is sufficient. Sample sizes per benchmark (50-100 instances) are stated in Appendix E.4 as 'balancing computational cost with statistical significance' but no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported for turn counts (e.g., '7.2±2.1'), redundancy rates, cross-validation R² (±0.033 SD), MAE (±0.011), and coefficient of variation across families (CV<0.02)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Single-Agent System (SAS) serves as the baseline throughout, with all MAS variants compared against it. Prior claims ('More agents is all you need') are directly contrasted."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use frontier models (GPT-5, Gemini 2.5 Pro, Claude 4.5 Sonnet) and recent benchmarks (BrowseComp-Plus 2025, Finance-Agent 2025). All are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The five architectures (SAS, Independent, Centralized, Decentralized, Hybrid) form a structured ablation over orchestrator presence and peer communication (Section 4.1). Agent heterogeneity experiments in Figure 4 also serve as ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: task success/accuracy, coordination efficiency (Ec), error amplification (Ae), message density, redundancy, overhead, success per 1K tokens, information gain, and cost per 1% gain."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs. All evaluation is automated via domain-specific validators with reported inter-rater reliability (Cohen's κ), but these are automated, not human."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Five-fold cross-validation with experiment-level holdout is used (Section 4.3). Out-of-sample validation on GPT-5.2 (released after the study) provides a truly held-out test (Appendix B)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by benchmark (Figure 2), by architecture, by LLM family (Figures 3, 4, 6), and by error category (Section 4.4). Table 4 shows per-predictor breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis: PlanCraft degradation (-39% to -70%) with execution trace examples, error taxonomy (Section 4.4) with four error categories, MAST failure taxonomy applied, and architecture-specific failure modes documented."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Major negative results reported: PlanCraft universal degradation, independent MAS underperformance, capability saturation, overall mean MAS improvement of -3.5%. The paper's central thesis is that MAS often hurts performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about R²=0.524, 87% architecture prediction accuracy, error amplification ratios (17.2× vs 4.4×), +80.8% Finance Agent improvement, -39% to -70% PlanCraft degradation, and GPT-5.2 validation (MAE=0.071) are all supported with detailed results in Sections 4.2-4.4 and Appendix B."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly designs controlled experiments to enable causal attribution, controlling for prompts, tools, and computational budgets while varying only coordination structure and model capability (Section 4.1). The ablation-like structure of the five architectures supports causal claims about coordination mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Limitations) explicitly bounds generalization: four benchmarks may not capture full spectrum, scaling limited to 9 agents, all agents share base architectures, prompts not optimized per model. The title says 'Towards' a science, appropriately hedging."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations: whether performance gains come from compute vs coordination (matched budgets address this), whether prompt sensitivity could affect results (Limitation v), and considers model-specific mechanisms for family differences (Section 4.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about measurement: task success metrics are domain-specific and explicitly defined (Section 4.1 Metrics), and the paper distinguishes between coordination metrics (proxies) and actual performance. The R²=0.524 is presented honestly as explaining 'more than half of performance variance.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referenced by marketing names only: 'GPT-5', 'GPT-5-mini', 'GPT-5-nano', 'Gemini 2.5 Pro', 'Claude Sonnet 4.5', etc. No API versions, snapshot dates, or specific model IDs (e.g., gpt-5-0613) are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section E.3 describes a prompt compilation system with YAML templates and variable interpolation, but no actual prompt text is provided. The paper says 'prompts are defined in YAML files' but does not include them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.2 reports architecture parameters: max 10 iterations for SAS, 3 agents for MAS, 5 orchestration rounds, 3 debate rounds, 3 iterations per round. Section E.5 reports temperature τ=0.7 and K=10 for information gain computation. Token budget mean 4,800 stated in Section 4.4."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 formally defines each architecture with communication topology, orchestration policy, and complexity analysis. Table 2 provides detailed complexity metrics. Section E describes tools, API integration, and agent configuration in detail."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D describes each dataset with instance counts. Section E.4 describes sample sizes and selection criteria. Metrics are defined with normalization procedures (Section 4.1). Token overlap analysis methodology described with BERTScore threshold of 0.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Limitations and Future Works' provides a dedicated section with six numbered limitation categories (i-vi)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: (i) scaling limited to 9 agents, (ii) agents share same base architecture, (iii) tool-heavy environments as failure modes, (iv) prompts not optimized per model, (v) four benchmarks may not cover full spectrum, (vi) economic viability concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states boundaries: does not include embodied agents, multi-user interaction, or long-horizon temporal dependencies. Agent scaling explored only up to 9. Only four benchmarks tested. Prompts not tuned per model family."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (execution traces, per-instance results, agent logs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D describes each benchmark dataset and its collection. Section 4.1 describes the experimental setup in detail: 180 configurations, 5 architectures × 9 models × 4 benchmarks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments are automated benchmark evaluations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: benchmarks → agent configurations → API calls via LiteLLM/LangChain → domain-specific validators → coordination metrics computation → mixed-effects modeling. Section E provides implementation details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section mentioning funding sources. The paper is marked '© 2025 Google. All rights reserved' but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Google Research, Google DeepMind, and MIT. The superscripts map each author to their affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are from Google Research/DeepMind, and the paper evaluates Google's Gemini models alongside competitors. Google has a direct financial interest in the relative performance of its models and in the conclusions about multi-agent architectures."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure section. Given Google affiliation and evaluation of Google products, this is a notable omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the nine models evaluated. The paper discusses benchmark contamination conceptually (referencing Kapoor et al. 2025) but does not state cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any benchmark data could appear in the training sets of the evaluated models. Some benchmarks (PlanCraft, Workbench) may predate model training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Despite referencing Kapoor et al. (2025) on benchmark evaluation pitfalls and discussing contamination conceptually for non-agentic tasks, the paper does not address contamination risk for its own four benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost analysis in Section 4.4: dollar costs per trial by model family (OpenAI ~$0.008, Anthropic ~$0.024, Google ~$0.012 per 1% gain), success per 1K tokens (SAS 67.7, Centralized 21.5, Hybrid 13.6), and Figure 3 shows cost-performance trade-offs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Token budgets stated: mean 4,800 tokens per trial matched across architectures (Section 4.4, Table 5). Total of 15,750 instance runs across 180 configurations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Results appear to be single-run per configuration, with variance coming from across-configuration comparisons rather than multiple seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "N=180 configurations stated, with 15,750 total instance runs. Per-benchmark instance counts given in Appendix E.4 (50-100 instances each). K=10 Monte Carlo samples stated for information gain."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Architecture parameters (iterations, rounds, agent counts) appear fixed without justification for specific values chosen."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 180 configurations are reported, not just the best. Table 4 shows all coefficients including non-significant ones. Cross-validation used for model selection (Table 3 compares model specifications)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 reports 20 p-values from the regression model without correction for multiple comparisons. No Bonferroni, Holm, or other family-wise error rate correction mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented all five architectures and evaluate them. No discussion of self-comparison bias or acknowledgment that their implementations of baselines might not be optimal (per Lucic et al. 2018, which they don't cite)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Token budgets explicitly matched across architectures (Section 4.1, 4.4). Figure 3 shows cost vs performance. Efficiency metric Ec normalizes success by computational cost. The paper explicitly discusses how fixed budgets affect per-agent reasoning capacity."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 extensively discusses what makes tasks 'agentic' vs non-agentic, formally defining agentic evaluation criteria. The paper questions whether non-agentic benchmarks give misleading guidance and chooses benchmarks satisfying three necessary conditions for agentic evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central design principle is controlling for scaffold confounds: 'identical task prompts, tools, and computational budgets across all configurations, while systematically varying only coordination structure and model capability' (Section 1, 4.1)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark tasks existed before model training cutoffs. BrowseComp-Plus (2025) and Finance-Agent (2025) are recent, but PlanCraft (2024) and Workbench (2024) could have been in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information. For agentic tasks with tool use, this is less of a concern, but the paper does not address it."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and benchmark instances share structural similarities or overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied despite evaluating pre-trained models on benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "Multi-agent systems show highly heterogeneous performance, ranging from +80.9% improvement (Finance Agent, Centralized) to -70.0% degradation (PlanCraft, Independent).",
       "evidence": "Section 4.2, Figure 2: detailed per-benchmark, per-architecture results with percentage changes vs SAS baseline. N=180 configurations across 4 benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "A predictive mixed-effects model achieves cross-validated R²=0.524 and 87% accuracy in predicting optimal architectures for held-out configurations.",
       "evidence": "Section 4.3, Table 3: five-fold cross-validation with experiment-level holdout. R²_CV=0.524 (±0.033 SD), MAE=0.089 (±0.011). Model comparison shows 20% improvement over categorical architecture labels.",
       "supported": "strong"
     },
     {
       "claim": "There is a capability saturation effect: coordination yields diminishing or negative returns (β̂=-0.404, p<0.001) once single-agent baselines exceed ~45%.",
       "evidence": "Section 4.3, Table 4: P_SA × log(1+n_a) interaction coefficient β̂=-0.404, 95% CI [-0.557, -0.252], p<0.001. Decision boundary derived at raw performance ≈0.45.",
       "supported": "strong"
     },
     {
       "claim": "Independent agents amplify errors 17.2× while centralized coordination contains this to 4.4×.",
       "evidence": "Table 5: error amplification factors across architectures. Section 4.4 provides 95% CIs for independent [14.3, 20.1] and centralized [3.8, 5.0].",
       "supported": "moderate"
     },
     {
       "claim": "Out-of-sample validation on GPT-5.2 achieves MAE=0.071 and confirms four of five scaling principles.",
       "evidence": "Appendix B, Tables 7-9: GPT-5.2 (Intelligence Index 75) tested on BrowseComp-Plus. SAS over-predicted (+49.5%), but MAS predictions well-calibrated (average error 7.4%). Kendall's τ=0.200 is weak for ranking.",
       "supported": "moderate"
     },
     {
       "claim": "Coordination scaling follows model-agnostic principles with maximum inter-family difference Δmax=0.023.",
       "evidence": "Section 4.1: cross-family consistency with CV<0.02 across families. However, Section 4.2 and Figure 4 show notable family-specific differences in architecture preferences.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own products",
       "detail": "Most authors are Google Research/DeepMind employees evaluating Gemini models alongside competitors. While results don't obviously favor Google (Gemini is not always best), the conflict is not disclosed. No competing interests statement."
     },
     {
       "flag": "No seed sensitivity analysis",
       "detail": "For a paper establishing 'scaling principles,' the absence of multi-seed evaluation is notable. Henderson et al. (2018), which the paper cites in its experimental rigor module description, showed RL results can vary 2x across seeds. The 180 configurations provide cross-configuration variance but not within-configuration stability."
     },
     {
       "flag": "Weak out-of-sample ranking validation",
       "detail": "The GPT-5.2 validation (Appendix B) reports Kendall's τ=0.200 for architecture ranking, described as 'weak.' SAS is over-predicted by 49.5%. The '4/5 findings validated' framing obscures that the model fails at the primary task of predicting optimal architecture for this held-out model."
     },
     {
       "flag": "No contamination analysis despite evaluating pre-trained models",
       "detail": "The paper cites Kapoor et al. (2025) on benchmark evaluation pitfalls but does not address contamination risk for its own four benchmarks, especially PlanCraft (2024) and WorkBench (2024) which predate some model training periods."
     },
     {
       "flag": "No code or data release",
       "detail": "For a paper proposing quantitative scaling principles as an engineering tool, the absence of code, prompts, and raw experimental data limits reproducibility and independent verification of the 180-configuration evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "AI agents that matter",
       "authors": ["S. Kapoor", "B. Stroebl", "Z. S. Siegel", "N. Nadgir", "A. Narayanan"],
       "year": 2025,
       "relevance": "Key prior work on agentic evaluation pitfalls, benchmark rigor, and cost metrics for agent systems."
     },
     {
       "title": "More agents is all you need",
       "authors": ["J. Li", "Q. Zhang", "Y. Yu", "Q. Fu", "D. Ye"],
       "year": 2024,
       "relevance": "Claim that scaling agent count improves performance, directly challenged by this paper's findings."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Multi-Agent System Failure Taxonomy (MAST) used in this paper's error analysis."
     },
     {
       "title": "Establishing best practices in building rigorous agentic benchmarks",
       "authors": ["Y. Zhu", "T. Jin", "Y. Pruksachatkun"],
       "year": 2025,
       "relevance": "Agentic Benchmark Checklist (ABC) extended by this paper for evaluation rigor."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "relevance": "Major agentic benchmark for software engineering, referenced as exemplar of agentic evaluation."
     },
     {
       "title": "Scaling large language model-based multi-agent collaboration",
       "authors": ["C. Qian", "Z. Xie", "Y. Wang"],
       "year": 2025,
       "relevance": "Proposes collaborative scaling laws for MAS, which this paper finds show no universal pattern."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["S. Hong", "M. Zhuge", "J. Chen"],
       "year": 2024,
       "relevance": "Meta-programming workflows for MAS that mitigate hallucination cascades."
     },
     {
       "title": "Single-agent or multi-agent systems? Why not both?",
       "authors": ["M. Gao", "Y. Li", "B. Liu"],
       "year": 2025,
       "arxiv_id": "2505.18286",
       "relevance": "Shows benefits of MAS diminish as base models improve, supporting capability saturation finding."
     },
     {
       "title": "Multi-agent architecture search via agentic supernet",
       "authors": ["G. Zhang", "L. Niu", "J. Fang"],
       "year": 2025,
       "relevance": "Achieves comparable MAS performance at 6-45% cost through dynamic architecture search."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["J. Kaplan", "S. McCandlish", "T. Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational neural scaling laws that this paper contrasts with agent coordination scaling."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Y. Du", "S. Li", "A. Torralba"],
       "year": 2023,
       "relevance": "Demonstrates peer-to-peer debate effectiveness depends on task decomposability."
     },
     {
       "title": "Should we be going MAD? A look at multi-agent debate strategies for LLMs",
       "authors": ["A. Smit", "P. Duckworth", "N. Grinsztajn"],
       "year": 2023,
       "arxiv_id": "2311.17371",
       "relevance": "Shows multi-agent debate does not reliably outperform single-agent self-consistency."
     }
   ]
 }