ai-research-survey

scan.json (24159B)

---

 {
   "paper": {
     "title": "AGENTSNET: Coordination and Collaborative Reasoning in Multi-Agent LLMs",
     "authors": [
       "Florian Grötschla",
       "Luis Müller",
       "Jan Tönshoff",
       "Mikhail Galkin",
       "Bryan Perozzi"
     ],
     "year": 2025,
     "venue": "Preprint",
     "arxiv_id": "2507.08616"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository at https://github.com/floriangroetschla/AgentsNet (Section 5.1, Implementation). Open-source code is explicitly stated."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace at https://huggingface.co/datasets/disco-eth/AgentsNet (Section 5.1, Implementation)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LangChain and NetworkX as implementation frameworks but does not provide a requirements.txt, Dockerfile, or detailed version specifications for all dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper itself. The code is available on GitHub but no 'Reproducing Results' section or README instructions are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports standard error of the mean in parentheses for all results (e.g., '0.14 (0.04)'). Figure 1 explicitly notes 'Error bars indicate standard error of the mean.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims ('best performing models are Claude 3.7 Sonnet, Gemini 2.5 Pro, and Gemini 2.5 Flash'; 'Gemini 2.5 Flash is roughly on par with Claude 3.7') but uses no significance tests (p-values, t-tests, bootstrap, etc.) to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw benchmark scores but never explicitly computes or frames relative differences as effect sizes. Comparisons like 'roughly on par' and 'cheaper by a factor of 20' are made informally without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "27 network topologies with at least one repeat per graph are used (Section 5.1). There is no power analysis or explicit justification for why 27 graphs and the number of repeats are sufficient for the precision of results."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports standard error of the mean for all task-model combinations (values in parentheses). The paper references Miller [32] as the statistical methodology basis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple frontier LLM models are compared against each other on the benchmark, including Claude, GPT, Gemini, Llama 4, and o4-mini variants (Table 2). These serve as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are 2024-2025 frontier models (Claude 3.5 Haiku, Claude 3.7 Sonnet, GPT-4.1 mini, Gemini 2.5 Pro, Llama 4, o4-mini), representing the state of the art at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is reported to analyze the contribution of individual components (e.g., number of message-passing rounds, chain-of-thought prompting). The paper evaluates different models on the benchmark but does not ablate benchmark design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses binary (fully correct) evaluation as the main metric and also discusses and reports soft evaluation scores in Appendix B, providing a continuous measure of solution quality."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is clearly irrelevant to this benchmark paper, which evaluates LLM agents on structured mathematical problems with objective ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Graphs are generated procedurally from distributions; no model tuning is performed on them. The evaluation set is generated independently and used only for testing, so there is no development/test confound."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides a per-task and per-graph-size breakdown for each model, showing performance on CONSENSUS, MATCHING, LEADER ELECTION, COLORING, and VERTEX COVER separately at 4, 8, and 16 nodes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 provides qualitative analysis with three findings about failure modes: strategy coordination failures, agents accepting erroneous information, and conflict resolution. Appendix E provides additional transcript examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly shows that performance drops severely for all models on 100-agent networks (near zero), and many models perform poorly on VERTEXCOVER and COLORING tasks. These are negative results reported candidly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'some frontier LLMs demonstrate strong performance for small networks but fall off as network scales' is borne out in Table 2 and Figures 4-5. The claim about scaling to 100 agents is demonstrated in Figure 5."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims — it is a benchmark paper that reports performance scores. No language like 'improves', 'causes', or 'leads to' is used to draw causal inferences from the benchmark results."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately scoped to the specific models and graph distributions tested. Results are presented as benchmark performance on specific tasks, not generalized to all multi-agent systems or all coordination problems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The qualitative analysis in Section 5.4 describes failure patterns but does not systematically discuss alternative explanations for why certain models fail. For example, whether failures stem from context length limits, prompt formatting issues, or fundamental reasoning gaps is not distinguished."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 in Appendix A explicitly lists API providers and exact model versions for all evaluated models (e.g., claude-3-5-haiku-20241022, claude-3-7-sonnet-20250219, gpt-4.1-mini, gemini-2.5-flash-preview-04-17)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt template is provided in Appendix A (Section 'Full System Prompt'), including all template variables. Task-specific prompt texts ([task1] and [task2]) for each task are provided in Appendix B, and examples are given in Section 4."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report LLM API hyperparameters such as temperature or top-p settings used during evaluation. These are omitted from both the main paper and appendices."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendix A describe the message-passing protocol in detail, including the synchronous round structure, JSON formatting requirements, retry logic for invalid JSON, and Algorithm 1 pseudocode. This constitutes a thorough scaffolding description."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes graph generation: 27 topologies, 3 per graph size (4, 8, 16 nodes) per distribution type (small-world, scale-free, Delaunay), 3 graphs per combination. Appendix D details the three graph models."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and discusses the LOCAL model's synchronous message-passing constraint and JSON parsing issues. Appendix G provides further elaboration."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G discusses specific threats: (1) synchronous communication model limits ecological validity for real-world transfer, (2) binary metric may obscure partial progress, (3) homogeneous agents miss heterogeneous deployment challenges, (4) no adversarial/faulty agents tested. These are specific to this study's design choices."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper is appropriately scoped to its benchmark tasks, it does not explicitly state what the results do NOT show (e.g., that performance on AGENTSNET does not predict performance on real-world multi-agent tasks)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace at https://huggingface.co/datasets/disco-eth/AgentsNet, allowing independent verification of the benchmark instances."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Graph generation is procedurally described in Section 5.1 and Appendix D, including the three graph models (small-world, scale-free, Delaunay) and size parameters. The generation process is reproducible from the code."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; the study uses procedurally generated graphs and LLM API calls. Recruitment is not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from graph generation to model evaluation is described: graph sampling, agent instantiation, message-passing rounds, response extraction, and scoring. Algorithm 1 provides pseudocode."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding disclosure section is present in the paper. Two authors are affiliated with Google Research, which produces Gemini models evaluated in the benchmark."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: Florian Grötschla (ETH Zurich), Luis Müller and Jan Tönshoff (RWTH Aachen), Mikhail Galkin and Bryan Perozzi (Google Research). The Google affiliation is visible."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of five authors are from Google Research, which develops Gemini models — which are evaluated in the benchmark. This represents a potential non-independent relationship between researchers and outcomes, and no acknowledgment of this conflict is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement in the paper. The Google Research affiliation of two authors and their potential interest in Gemini model performance is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLM models on the benchmark but does not state the training data cutoff dates for any of the models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark tasks or graph structures could appear in training data. However, because tasks are procedurally generated, contamination risk is likely low — but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is novel and the paper claims it was created in 2025, but no analysis is provided of whether similar distributed computing problems or graph structures exist in training data. The paper does not address this."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 plots model performance versus API cost per repeat in USD (as of May 15, 2025), showing costs ranging from ~$1 to ~$200 per repeat. Pareto-optimal cost-performance models are identified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-repeat API costs are shown in Figure 1, the total computational budget for the full benchmark (total number of API calls, total cost) is not explicitly stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing multi-agent benchmarks cover at most 2-5 agents, while AGENTSNET scales to practically unlimited network sizes, with experiments involving up to 100 coordinating agents.",
       "evidence": "Abstract and Section 1 state this; Figure 5 shows Gemini 2.0 Flash performance on networks of 20-100 agents.",
       "supported": "strong"
     },
     {
       "claim": "Some frontier LLMs demonstrate strong performance for small networks (4 nodes) but performance degrades as network size increases.",
       "evidence": "Table 2 and Figure 4 show per-task performance broken down by graph size (4, 8, 16 nodes). Figure 5 shows near-zero performance at 100 nodes for all tasks.",
       "supported": "strong"
     },
     {
       "claim": "Claude 3.7 Sonnet, Gemini 2.5 Pro, and Gemini 2.5 Flash are the best performing models overall on AGENTSNET.",
       "evidence": "Table 2 shows AGENTSNET aggregate scores: Claude 3.7 Sonnet (0.70), Gemini 2.5 Pro (0.80), Gemini 2.5 Flash (0.69), all higher than other models.",
       "supported": "strong"
     },
     {
       "claim": "Gemini 2.5 Flash achieves roughly comparable performance to Claude 3.7 Sonnet at approximately 1/20th the API cost.",
       "evidence": "Figure 1 plots cost vs. performance, showing Gemini 2.5 Flash as Pareto-optimal relative to Claude 3.7 Sonnet. AGENTSNET scores are 0.69 vs. 0.70 respectively.",
       "supported": "moderate"
     },
     {
       "claim": "Strategy coordination is an essential challenge: agents sometimes fail to agree on strategies or agree too late in the message-passing rounds.",
       "evidence": "Section 5.4, Finding 1 describes this pattern with transcript analysis. Appendix E contains full qualitative examples.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AGENTSNET is a new multi-agent benchmark built on five distributed computing problems (graph coloring, vertex cover, maximal matching, leader election, consensus) that evaluates LLM agents' ability to coordinate through synchronous message-passing on graph topologies. Frontier models perform reasonably on small networks (4-16 nodes) but performance collapses near zero for 100-agent networks across all tasks. Among evaluated models, Gemini 2.5 Pro achieves the highest overall score (0.80), while Gemini 2.5 Flash offers the best cost-performance tradeoff. Qualitative analysis reveals key failure modes: poor strategy coordination, acceptance of erroneous neighbor information, and difficulty maintaining global consistency through purely local communication.",
   "red_flags": [
     {
       "flag": "Conflict of interest: Google Research authors evaluate Gemini models",
       "detail": "Two of five authors (Mikhail Galkin and Bryan Perozzi) are affiliated with Google Research, which develops the Gemini family of models. Gemini models (2.0 Flash, 2.5 Flash, 2.5 Flash Thinking, 2.5 Pro) constitute 4 of the 10 evaluated models. No competing interests statement is present, and no funding is disclosed."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature and other sampling hyperparameters for LLM API calls are not reported anywhere in the paper. These significantly affect output variability for multi-agent coordination tasks."
     },
     {
       "flag": "Limited number of repeats per graph",
       "detail": "The paper uses 'at least one repeat per graph' for the main results (Table 2), with some cells showing as few as 5-6 samples total. The standard errors for some model-task combinations are large (e.g., 0.09 for Gemini 2.5 Flash on VERTEXCOVER), suggesting high variance with insufficient repetitions."
     },
     {
       "flag": "No ablation of message-passing rounds",
       "detail": "The number of message-passing rounds is set heuristically (2D+1 for global tasks, size-based for local tasks). No ablation studies examine whether different numbers of rounds significantly affect results, making it unclear whether performance differences between models reflect coordination ability or sensitivity to round count."
     }
   ],
   "cited_papers": [
     {
       "title": "Agentverse: Facilitating multi-agent collaboration and exploring emergent behaviors",
       "authors": [
         "Chen, W.",
         "Su, Y.",
         "Zuo, J."
       ],
       "year": 2024,
       "relevance": "Multi-agent LLM collaboration framework directly relevant to the survey scope on agentic AI systems."
     },
     {
       "title": "GPTSwarm: Language agents as optimizable graphs",
       "authors": [
         "Zhuge, M.",
         "Wang, W.",
         "Kirsch, L.",
         "Faccio, F.",
         "Khizbullin, D.",
         "Schmidhuber, J."
       ],
       "year": 2024,
       "relevance": "Graph-based approach to organizing LLM agents that motivates the AGENTSNET benchmark design."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": [
         "Liu, X.",
         "Yu, H.",
         "Zhang, H."
       ],
       "year": 2024,
       "relevance": "A major multi-agent benchmark that AGENTSNET is explicitly positioned as complementary to."
     },
     {
       "title": "LLM-Coordination: Evaluating and analyzing multi-agent coordination abilities in large language models",
       "authors": [
         "Agashe, S.",
         "Fan, Y.",
         "Reyna, A.",
         "Wang, X. E."
       ],
       "year": 2024,
       "arxiv_id": "2310.03903",
       "relevance": "Directly evaluates multi-agent coordination in LLMs, a core topic for this survey."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": [
         "Du, Y.",
         "Li, S.",
         "Torralba, A.",
         "Tenenbaum, J. B.",
         "Mordatch, I."
       ],
       "year": 2023,
       "relevance": "Seminal work on multi-agent debate as a method to improve LLM reasoning quality."
     },
     {
       "title": "Scaling large-language-model-based multi-agent collaboration",
       "authors": [
         "Qian, C.",
         "Xie, Z.",
         "Wang, Y."
       ],
       "year": 2024,
       "arxiv_id": "2406.07155",
       "relevance": "Studies scaling properties of multi-agent LLM collaboration, directly relevant to AGENTSNET's scalability focus."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": [
         "Schaeffer, R.",
         "Miranda, B.",
         "Koyejo, S."
       ],
       "year": 2023,
       "relevance": "Methodological paper on emergent behaviors in LLMs; cited for its relevance to discontinuous evaluation metrics used in AGENTSNET."
     },
     {
       "title": "Adding error bars to evals: A statistical approach to language model evaluations",
       "authors": [
         "Miller, E."
       ],
       "year": 2024,
       "arxiv_id": "2411.00640",
       "relevance": "Provides statistical methodology for LLM evaluation used directly in this paper's analysis."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": [
         "Park, J. S.",
         "O'Brien, J.",
         "Cai, C. J.",
         "Morris, M. R.",
         "Liang, P.",
         "Bernstein, M. S."
       ],
       "year": 2023,
       "relevance": "Foundational paper on LLM-based agent systems that motivates multi-agent research."
     },
     {
       "title": "Talk like a graph: Encoding graphs for large language models",
       "authors": [
         "Fatemi, B.",
         "Halcrow, J.",
         "Perozzi, B."
       ],
       "year": 2024,
       "relevance": "Studies graph reasoning in LLMs, directly relevant to AGENTSNET's graph-based evaluation tasks."
     },
     {
       "title": "Tau-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": [
         "Yao, S.",
         "Shinn, N.",
         "Razavi, P.",
         "Narasimhan, K."
       ],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Another agentic benchmark that AGENTSNET is positioned alongside."
     }
   ]
 }