ai-research-survey

scan-v5.json (26719B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LLM-Coordination: Evaluating and Analyzing Multi-agent Coordination Abilities in Large Language Models",
     "authors": [
       "Saaket Agashe",
       "Yue Fan",
       "Anthony Reyna",
       "Xin Eric Wang"
     ],
     "year": 2023,
     "venue": "North American Chapter of the Association for Computational Linguistics",
     "arxiv_id": "2310.03903",
     "doi": "10.18653/v1/2025.findings-naacl.448"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by results: environmental coordination strength (Tables 1–2), ToM deficits (Table 3, Figure 3), joint planning weaknesses (Figure 3), and ZSC robustness (Tables 4–5).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study (Table 6) removes ToM reasoning and verification steps independently, providing adequate design to support the causal claim that each component improves Hanabi performance.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are scoped to 'pure coordination games' throughout; the conclusion explicitly states findings underscore areas for improvement in pure coordination setups, not general AI coordination.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes Overcooked success to full observability and Hanabi failure to ToM demands, but does not consider alternative explanations such as training data overlap with game rules or prompt sensitivity effects.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Game scores (delivery points, card sequences, escape rates) are used directly as coordination performance metrics; the paper treats these as direct measures of coordination ability within well-defined game objectives without conflating them with broader constructs.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is a dedicated Limitations section covering three specific issues: latency/compute, prompt configuration, and manual curation of CoordQA.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific: e.g., effective reasoning requires GPT-4-turbo scale, prompt quality affects results, and manual edge-case curation limits scalability — not boilerplate disclaimers.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly restricts scope to pure coordination games (no mixed incentives), distinguishes this from multi-LLM orchestration work, and notes that performance improvements via prompt engineering are left to future work.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discloses funding from the Microsoft Accelerate Foundation Models Research (AFMR) grant program.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors list University of California, Santa Cruz affiliations with email addresses in the paper header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft funded the research; Microsoft holds a major investment in OpenAI, whose GPT-4 models are the primary beneficiary of the paper's positive findings, creating a financial interest in the outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure beyond the funding acknowledgment appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Pure Coordination Games, Theory of Mind, Environment Comprehension, Joint Planning, and Zero-Shot Coordination are all explicitly defined in context in Sections 1 and 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly enumerated at the end of the introduction: the benchmark, the holistic LLM-vs-RL evaluation, and the component-level CoordQA analysis.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Related Work (Section 2) differentiates this work from multi-LLM orchestration frameworks (MetaGPT, ChatDev) by focusing on innate coordination ability of individual LLMs in established pure coordination benchmarks.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The abstract explicitly links to https://github.com/eric-ai-lab/llm_coordination, a GitHub repository for the benchmark and agent implementations.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "CoordQA (198 manually curated questions) is released with the code; Hanabi and Overcooked-AI are established public benchmarks available independently.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency specification is mentioned in the paper; only a GitHub link is provided without explicit environment details.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions appear in the paper; the appendices detail prompts and game setups but not how to run experiments end-to-end.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Tables 1, 2, 3, and 5 all report ± values alongside mean scores across trials.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, ANOVA, etc.) are used to support comparative claims between LLMs and RL baselines, despite substantial comparative claims throughout.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute performance differences are reported (e.g., GPT-4-turbo 13.33 vs GPT-3.5 1.33 in Hanabi; 80% vs 33% escape rates) providing meaningful context for comparison.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Three trials per model-scenario combination is stated as the protocol but is not justified via power analysis or discussion of why three trials is sufficient for the variance observed.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviations (±) are reported across all main result tables (Tables 1–5), though it is unclear if these are across seeds or trials.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "MARL baselines (PPO, PBT for Overcooked; BAD, SAD, OBL for Hanabi) and greedy baselines for Collab games are included throughout.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include state-of-the-art MARL methods for each game (OBL, HSP, PBT) from 2021–2023, appropriate to the paper's timeframe.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 6 (Appendix C.4) and Table 3 ablate ToM reasoning and answer verification steps independently for the Hanabi LLM agent, showing the contribution of each component.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Overcooked uses delivery score, Hanabi uses card sequence count, Collab games report success rate and average turns — multiple metrics across evaluation tasks.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation of system outputs; the study uses automated game environments and MCQ accuracy with ground-truth labels.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "All 198 CoordQA questions constitute both the benchmark and the evaluation set; there is no held-out portion, and Agentic Coordination uses the full game environments without a separate test split.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Figure 3 breaks CoordQA results by EC/ToM/JP category for each model; Tables 1–3 break agentic results by game and layout.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Specific failure modes are discussed: LLMs 'bombing' in Hanabi (losing all lives), open-source LLMs performing below random on joint planning, and GPT-3.5 failing Collab games.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper clearly reports that most LLMs fail badly at Hanabi compared to RL, that open-source models score below random on joint planning, and that ToM reasoning in CollabCapture shows minimal benefit.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are named: gpt-4-0125-preview, GPT-3.5-turbo-0125, Mixtral 8x7B, and GPT-4o are all explicitly identified in Section 4.1.1.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full verbatim prompts for Overcooked (Appendix A), Hanabi (Appendix B), ToM reasoning step, and answer verification step are all provided in the appendices.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Temperature, top-p, max tokens, and other API hyperparameters are not reported anywhere in the paper.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The agentic framework is described in detail in Section 4.1.1: Memory (long-term, working, episodic), Reasoning (LLM), and Grounding modules with game-specific implementations.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A describes distance preprocessing for Overcooked states (shortest-path distances replacing grid coordinates); Appendix B describes Hanabi knowledge state construction.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "CoordQA questions and game environments are released via GitHub; game interaction logs are implicitly available through the open-source code and publicly available game simulators.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 and Appendix D describe the process: 66 edge-case scenarios manually sampled from games, 3 question types per scenario, filtered for ambiguity, yielding 198 questions.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; evaluations use automated game environments and LLM API calls.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from game scenario sampling to question construction to MCQ evaluation with fuzzy string matching is described across Sections 3.2, 4.2, and Appendix D.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-4-turbo, GPT-3.5, Mixtral, GPT-4o).",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether game rules, strategies, or CoordQA-style questions could have appeared in LLM training data, despite Hanabi and Overcooked being publicly documented games.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Hanabi and Overcooked rules and strategies are publicly available and likely present in LLM pretraining data; this potential contamination is never addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The limitations mention 'significant latency and substantial computational resources' qualitatively, and note GPT-4-turbo cross-play was limited to single trials due to cost, but no dollar or token cost figures are provided.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total compute budget, API call counts, or GPU hours are reported anywhere in the paper.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Zero-shot LLM agents (GPT-4-turbo) match or outperform self-play RL baselines (PPO, PBT) in Overcooked-AI coordination tasks.",
       "evidence": "Table 1 shows GPT-4-turbo outperforming PPO/PBT in 3 of 5 layouts (e.g., Asymmetric Advantages: 260 vs 190.1); Table 4 shows it outperforming BC and PPOBC in zero-shot coordination.",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents significantly underperform specialized RL methods (BAD, SAD, OBL) on Hanabi due to Theory of Mind reasoning demands.",
       "evidence": "Table 3: best LLM (GPT-4-turbo) scores 13.33 vs RL methods scoring 23–24; without ToM+Verification, score drops to 4.33 with 100% bomb rate.",
       "supported": "strong"
     },
     {
       "claim": "LLMs perform well at Environment Comprehension but fail at Joint Planning, with most models scoring below random on JP questions.",
       "evidence": "Figure 3: GPT-4-turbo achieves >80% EC accuracy but <40% JP accuracy; open-source models (Vicuna, Mistral) score below random baseline on JP.",
       "supported": "strong"
     },
     {
       "claim": "Adding explicit ToM reasoning and answer verification steps substantially improves LLM coordination performance in Hanabi.",
       "evidence": "Table 6: GPT-4-turbo with ToM+Verification scores 13.33 vs 10.33 (Verification only) vs 4.33 (neither); bomb rate goes from 100% to 0% with Verification.",
       "supported": "strong"
     },
     {
       "claim": "LLM agents exhibit zero-shot coordination robustness to unseen partners, unlike self-play RL methods.",
       "evidence": "Table 5: GPT-4-turbo scores 15.00 cross-play with OBL-1 vs 13.66 self-play; SAD degrades from 23.66 self-play to 11.33 cross-play. Table 4 shows LLM outperforming BC/PPOBC with human proxies.",
       "supported": "moderate"
     },
     {
       "claim": "CoordQA performance correlates with agentic coordination performance, validating the benchmark as a diagnostic tool.",
       "evidence": "Figure 2 reports high Pearson correlations (0.85–0.98 for EC vs game performance) across 4 LLMs, but only 4 data points makes these correlations statistically unreliable.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "The LLM-Coordination Benchmark reveals a clear capability split: LLM agents (especially GPT-4-turbo) match or exceed RL baselines in fully-observable, environment-focused coordination tasks (Overcooked) without any task-specific training, but substantially underperform in Hanabi where active Theory of Mind reasoning is required. CoordQA dissects why — LLMs are competent at environment comprehension but most models, including GPT-4, score below 40% on joint planning, with open-source models performing worse than random. A critical practical finding is that ToM reasoning and answer verification steps are necessary scaffolding components for LLM coordination: without them, GPT-4-turbo bombs every Hanabi game. Unlike self-play RL methods, LLM agents maintain performance with unseen partners, making them naturally robust for zero-shot coordination settings.",
   "red_flags": [
     {
       "flag": "4-point correlation",
       "detail": "Pearson correlations in Figure 2 are computed across only 4 LLM models, making all reported correlation coefficients (r=0.88, 0.95, etc.) statistically meaningless — you cannot establish a correlation pattern with 4 data points."
     },
     {
       "flag": "Funder conflict",
       "detail": "Microsoft (AFMR grant) funds research that shows Microsoft-invested OpenAI GPT-4 models outperforming all competitors; no competing interests statement addresses this."
     },
     {
       "flag": "Inconsistent trial counts",
       "detail": "GPT-4-turbo cross-play in Overcooked uses only a single trial 'due to cost and time constraints' (footnote 1), while other conditions use 3 trials, undermining the reliability of those specific comparisons."
     },
     {
       "flag": "No significance tests",
       "detail": "Comparative claims (LLMs vs RL, model vs model) are made throughout without any statistical significance testing, despite high variance in some conditions (e.g., Mixtral ±14.40 in Overcooked)."
     },
     {
       "flag": "Contamination unaddressed",
       "detail": "Hanabi and Overcooked rules, strategies, and game-theoretic analysis are extensively documented online and likely present in all tested LLMs' pretraining data; this potential advantage over RL baselines is never discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "The Hanabi Challenge: A New Frontier for AI Research",
       "relevance": "Primary benchmark game; provides RL baselines (BAD, SAD, OBL) and evaluation protocol"
     },
     {
       "title": "On the Utility of Learning about Humans for Human-AI Coordination (Overcooked-AI)",
       "relevance": "Primary benchmark environment and source of PPO/BC/PPOBC baselines for zero-shot coordination comparison"
     },
     {
       "title": "Off-Belief Learning",
       "relevance": "State-of-the-art cross-play method for Hanabi used as unseen partner in ZSC experiments"
     },
     {
       "title": "Cognitive Architectures for Language Agents",
       "relevance": "Design framework for the LLM agent scaffolding (Memory, Reasoning, Grounding)"
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "relevance": "Reasoning strategy used in agent design; key prior work on LLM agents"
     },
     {
       "title": "Theory of Mind for Multi-Agent Collaboration via Large Language Models",
       "relevance": "Direct prior work on ToM in LLM multi-agent settings; differentiates via explicit belief representations"
     },
     {
       "title": "Theory of Mind Might Have Spontaneously Emerged in Large Language Models",
       "relevance": "Motivating claim for using LLMs in coordination; Kosinski's ToM emergence hypothesis"
     },
     {
       "title": "Learning Zero-Shot Cooperation with Humans, Assuming Humans are Biased (HSP)",
       "relevance": "Competitive baseline for zero-shot coordination in Overcooked; LLM matches HSP performance"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners building multi-agent LLM systems get actionable findings: LLMs need explicit ToM scaffolding, excel at observable coordination, and work well with unseen partners."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Zero-shot LLMs matching trained RL in Overcooked is surprising; LLMs failing at joint planning worse than random challenges assumptions about emergent reasoning."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Mild safety relevance: unreliable coordination agents (100% bomb rate without scaffolding) highlights risks of deploying LLMs in high-stakes coordination without adequate design."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "LLMs vs RL framing generates mild interest, but the comparison is mostly constructive rather than adversarial."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Code is publicly released on GitHub; Hanabi and Overcooked are well-known games that can be run interactively, making live demos straightforward."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "GPT-4 is prominently featured as the best-performing system; NAACL is a well-regarded NLP venue."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45492803",
         "title": "OpenZL: An open source format-aware compression framework",
         "points": 434,
         "comments": 107,
         "url": "https://news.ycombinator.com/item?id=45492803",
         "created_at": "2025-10-06T16:01:58Z"
       },
       {
         "hn_id": "40612538",
         "title": "Benchmarking the Energy Costs of Large Language Model Inference (2023)",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40612538",
         "created_at": "2024-06-07T20:17:48Z"
       },
       {
         "hn_id": "38186733",
         "title": "Pipeline Parallelism for DNN Inference with Practical Performance Guarantees",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38186733",
         "created_at": "2023-11-08T04:16:22Z"
       },
       {
         "hn_id": "37896876",
         "title": "Sorting It Out in Hardware: A State-of-the-Art Survey",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37896876",
         "created_at": "2023-10-16T07:49:46Z"
       },
       {
         "hn_id": "38698557",
         "title": "Augmenting LLM with Human-Like Memory for Mobile Task Automation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38698557",
         "created_at": "2023-12-19T17:29:15Z"
       },
       {
         "hn_id": "34332441",
         "title": "Hunter: Using Change Point Detection to Hunt for Performance Regressions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=34332441",
         "created_at": "2023-01-10T22:28:46Z"
       }
     ],
     "top_points": 434,
     "total_points": 441,
     "total_comments": 107
   }
 }