ai-research-survey

scan.json (25217B)

---

 {
   "paper": {
     "title": "LLM-Coordination: Evaluating and Analyzing Multi-agent Coordination Abilities in Large Language Models",
     "authors": ["Saaket Agashe", "Yue Fan", "Anthony Reyna", "Xin Eric Wang"],
     "year": 2023,
     "venue": "North American Chapter of the Association for Computational Linguistics",
     "arxiv_id": "2310.03903",
     "doi": "10.18653/v1/2025.findings-naacl.448"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in abstract: https://github.com/eric-ai-lab/llm_coordination"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The CoordinationQA suite of 198 questions is described as part of the benchmark contribution and the code repository is provided. The games used (Overcooked-AI, Hanabi) are publicly available benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency versions mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided in the paper. The GitHub link is given but no README instructions or reproduction scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1, 3, and 5 report ± values (e.g., '198.8 ± 4.06', '13.33 ± 0.88')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests reported. Claims like 'LLM agents match or outperform RL baselines' are based on point comparisons without significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, etc.) reported. Raw score differences are shown but no effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 3 trials per condition for most experiments with no justification for why 3 trials is sufficient. GPT-4-turbo cross-play uses a single trial per position 'due to cost and time constraints' (footnote 1)."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported via ± notation in Tables 1, 3, and 5 across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines: PPO, PBT for Overcooked; BAD, SAD, OBL for Hanabi; Greedy baseline for Collab games; BC, PPO_BC, HSP for cross-play; Random baseline for CoordinationQA."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include HSP (Yu et al., 2023), OBL (Hu et al., 2021a), and established MARL methods that are standard for these benchmarks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablation on ToM reasoning and verification steps in Table 3/6 and Table 2 (GPT-4-turbo w/out ToM Reasoning)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: total score (Overcooked, Hanabi), capture/escape rate, average turns, accuracy on EC/ToM/JP questions, Pearson correlation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of LLM agent outputs. Evaluation is entirely automated through game scores and MCQ accuracy. Human proxy agents are behavior cloning models, not actual humans."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation of dev/test sets mentioned. The CoordinationQA questions and game scenarios are used directly without held-out splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per game (Overcooked, Hanabi, CollabCapture, CollabEscape), per layout (5 Overcooked layouts), and per question type (EC, ToM, JP) in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Discussion of LLM failures in Hanabi (Section 5): 'LLM agents seem to struggle... GPT-4-turbo performs reasonably well, while other LLMs can barely complete the games.' Also discusses failure modes: misplays, bombing out, and open-source LLMs performing worse than random on JP."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: LLMs fail at Hanabi compared to RL (Table 3), open-source LLMs perform worse than random on Joint Planning (Figure 3), GPT-3.5 and Mixtral struggle across most games."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about LLMs excelling in environment-focused games but struggling with ToM are supported by Tables 1 (Overcooked success) and 3 (Hanabi failure). ZSC robustness claim supported by Tables 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study (Table 3/6) provides controlled single-variable manipulation for causal claims about ToM reasoning and verification steps. 'We attribute this failure to two factors' is supported by correlation analysis (Figure 2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title claims 'Multi-agent Coordination Abilities in Large Language Models' broadly, but results are limited to 4 specific games and 4 LLMs. The paper does not explicitly bound its claims to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. The claim that failure in Hanabi is due to ToM requirements could also be explained by partial observability, action space complexity, or prompt design. These alternatives are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Game scores are used as proxies for 'coordination ability' without discussing whether performance on these specific games generalizes to real coordination. The correlation between CoordinationQA and agentic performance (Figure 2) is presented as validation but the proxy gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions provided: 'gpt-4-0125-preview', 'GPT-3.5-turbo-0125', 'Mixtral 8x7B', 'GPT-4o' (Section 4.1.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided in Appendices A, B, and C including game descriptions, state description formats, system prompts, ToM reasoning prompts, and verification prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, or other API parameters for LLM calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed agentic framework described in Section 4.1.1: Memory (long-term, working, episodic), Reasoning (LLM), Grounding (action translation). ToM reasoning and verification steps described. Based on Cognitive Architectures for Language Agents (Sumers et al., 2023)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "State description conversion process documented in detail (Appendix A.2): how game states are converted to natural language, distance calculations, inventory descriptions, cooker states, etc."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' discusses latency/compute requirements, initial prompt configuration challenges, and manual curation of edge cases."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations discussed: 'effective reasoning for coordination is achievable primarily with larger LLMs like GPT-4-turbo' (latency constraint), prompt sensitivity ('careful manual configuration of the initial prompts'), and scalability of CoordinationQA ('can hinder the ability to scale')."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. No explicit boundaries on what games, settings, or LLMs the conclusions extend to. Section 8 discusses limitations of the approach but not boundaries of the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental logs, game transcripts, or LLM responses made available. Only aggregated scores reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "CoordinationQA creation process described in Section 3.2: 'We manually sampled edge cases from all 4 pure coordination games... generated a total of N=66 scenarios (25 from Overcooked, 28 from Hanabi, and 13 from the two Collab Games) and created 3 questions per scenario, resulting in 198 unique questions.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLMs on game environments."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline from scenario selection to question creation documented: edge case sampling → question creation (3 types per scenario) → ambiguity filtering → final dataset of 198 questions (Section 3.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 Acknowledgements: 'This research project has benefitted from the Microsoft Accelerate Foundation Models Research (AFMR) grant program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with University of California, Santa Cruz. No affiliation with the LLM providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft funded the research and Microsoft's products (through OpenAI partnership) are among those evaluated (GPT-4-turbo, GPT-3.5, GPT-4o). This is not acknowledged as a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the models used. The games (Overcooked, Hanabi) are well-known and their rules/strategies exist extensively in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LLMs have seen Hanabi strategies, Overcooked solutions, or the game rules in their training data. This is particularly relevant since these are well-known research benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Hanabi and Overcooked-AI are well-documented research benchmarks published in 2019-2020. The models tested (GPT-4, etc.) were trained after these benchmarks were published. No contamination discussion."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Cost mentioned as a limitation ('significant latency and require substantial computational resources') and as reason for limited trials ('due to cost and time constraints' in footnote 1), but no actual cost figures reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, GPU hours, or computational budget stated despite using paid API models across many experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 states 'All agents play three games with different seeds (same seeds across agents).' Variance across seeds reported via ± notation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of trials stated: '3 trials' for CollabCapture/Escape (Table 2), '3 trials per model' for CoordinationQA (Section 4.2), 'three games with different seeds' for Hanabi cross-play (Table 5). Single trial noted for GPT-4-turbo cross-play in Overcooked (footnote 1)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search described. Prompt configurations appear hand-tuned but no search process documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the final prompt configurations or reasoning strategies were selected. The paper states prompt engineering is left to 'future works focused on performance improvement' but doesn't describe how current configs were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across games, layouts, and models without any correction applied. No significance tests at all, let alone corrected ones."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors implemented the LLM agent framework and compare against established RL baselines from prior work. No acknowledgment that their implementation choices could favor their approach."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 notes LLMs are 'significantly slower and larger than RL models' but no quantitative comparison of compute requirements. The compute asymmetry between LLM and RL agents is acknowledged but not measured."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses why these specific games were selected: 'carefully selected for their ability to isolate and highlight specific coordination challenges.' Section 5 analyzes what each game tests (environment-focused vs ToM-focused coordination) with correlation analysis (Figure 2)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different scaffolding used for different games (ToM reasoning and verification for Hanabi, not for Overcooked). Different LLMs compared using the same scaffold, but the scaffold is designed by the authors and its effect vs model capability is not disentangled."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Hanabi (2020) and Overcooked-AI (2019) were published well before GPT-4's training cutoff. The models may have seen game strategies and solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether game descriptions in the prompt leak strategy information that wouldn't be available to RL agents learning from scratch."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the CoordinationQA questions or game scenarios could overlap with training data content."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM agents (GPT-4-turbo) match or outperform RL baselines in environment-focused coordination games (Overcooked-AI)",
       "evidence": "Table 1 shows GPT-4-turbo outperforming or matching PPO and PBT across 3 of 5 Overcooked layouts (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents struggle in games requiring active Theory of Mind reasoning (Hanabi)",
       "evidence": "Table 3 shows GPT-4-turbo scoring 13.33 vs RL methods at 23-24 in Hanabi. Other LLMs score near zero.",
       "supported": "strong"
     },
     {
       "claim": "ToM reasoning and verification steps improve LLM agent performance in Hanabi",
       "evidence": "Table 3/6 ablation: removing verification drops from 13.33 to 10.33, removing both drops to 4.33 with 100% bomb rate.",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents are robust to unseen partners (Zero-Shot Coordination) unlike RL methods",
       "evidence": "Table 4: GPT-4-turbo matches/outperforms HSP with human proxies. Table 5: GPT-4-turbo maintains performance with OBL partners (15.0 vs 13.66 self-play) while SAD drops from 23.66 to 11.33/8.00.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform best on Environment Comprehension, worst on Joint Planning in CoordinationQA",
       "evidence": "Figure 3 shows GPT-4-turbo >80% on EC but <40% on JP. Open-source LLMs perform worse than random on JP.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM agents (particularly GPT-4-turbo) can match or outperform RL baselines in fully observable coordination games (Overcooked-AI) without any training, but significantly underperform in games requiring Theory of Mind reasoning (Hanabi). LLMs show robustness to unseen partners (zero-shot coordination), unlike RL methods that suffer performance degradation. Fine-grained analysis via CoordinationQA reveals LLMs excel at environment comprehension but struggle with joint planning, with open-source models performing worse than random on planning questions.",
   "red_flags": [
     {
       "flag": "Very small trial counts",
       "detail": "Only 3 trials per condition for most experiments. GPT-4-turbo Overcooked cross-play uses a single trial per position, acknowledged as a cost constraint. With such small N, reported ± values may not reflect true variance."
     },
     {
       "flag": "No contamination analysis for well-known benchmarks",
       "detail": "Overcooked-AI and Hanabi are established benchmarks with extensive online strategy discussions. GPT-4 could have absorbed game strategies during training, which would inflate performance on environment-focused tasks. This is never discussed."
     },
     {
       "flag": "Microsoft funding conflict not acknowledged",
       "detail": "Research funded by Microsoft AFMR while evaluating OpenAI models (GPT-4-turbo, GPT-3.5, GPT-4o). Microsoft is a major OpenAI investor. This potential conflict is not acknowledged."
     },
     {
       "flag": "Unfair compute comparison with RL baselines",
       "detail": "LLM agents use orders of magnitude more compute than RL agents but this asymmetry is noted qualitatively ('significantly slower and larger') without quantification. Claiming LLMs 'match or outperform' RL without accounting for compute disparity is misleading."
     }
   ],
   "cited_papers": [
     {
       "title": "The Hanabi Challenge: A New Frontier for AI Research",
       "authors": ["Nolan Bard", "Jakob N. Foerster"],
       "year": 2020,
       "relevance": "Core benchmark used for evaluating multi-agent coordination requiring Theory of Mind."
     },
     {
       "title": "On the Utility of Learning about Humans for Human-AI Coordination",
       "authors": ["Micah Carroll", "Rohin Shah"],
       "year": 2019,
       "relevance": "Overcooked-AI benchmark for human-AI coordination, core evaluation environment in this paper."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "LLM-based agent showing planning abilities in virtual environments."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-LLM framework for collaborative task completion."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian"],
       "year": 2024,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent LLM framework for software development coordination."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Language Model Society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Multi-agent LLM communication and collaboration framework."
     },
     {
       "title": "Theory of Mind for Multi-Agent Collaboration via Large Language Models",
       "authors": ["Huao Li"],
       "year": 2023,
       "relevance": "Directly relevant — investigates ToM in LLM multi-agent collaboration."
     },
     {
       "title": "ProAgent: Building Proactive Cooperative AI with Large Language Models",
       "authors": ["Ceyao Zhang"],
       "year": 2023,
       "arxiv_id": "2308.11339",
       "relevance": "LLM-based cooperative agent architecture for Overcooked-AI."
     },
     {
       "title": "Cognitive Architectures for Language Agents",
       "authors": ["Theodore R. Sumers", "Shunyu Yao"],
       "year": 2023,
       "arxiv_id": "2309.02427",
       "relevance": "Design framework for LLM agent architectures, used as basis for this paper's agent design."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Reasoning strategy used as baseline for LLM agent implementations."
     },
     {
       "title": "Exploring Large Language Models for Communication Games: An Empirical Study on Werewolf",
       "authors": ["Yuzhuang Xu"],
       "year": 2024,
       "arxiv_id": "2309.04658",
       "relevance": "Empirical study of LLMs in multi-agent game settings."
     }
   ]
 }