ai-research-survey

scan.json (24383B)

---

 {
   "paper": {
     "title": "Harnessing Language for Coordination: A Framework and Benchmark for LLM-Driven Multi-Agent Control",
     "authors": ["Timothée Anne", "Noah Syrkis", "Meriem Elhosni", "Florian Turati", "Franck Legendre", "Alain Jaquier", "Sebastian Risi"],
     "year": 2024,
     "venue": "IEEE Transactions on Games",
     "arxiv_id": "2412.11761",
     "doi": "10.1109/TG.2025.3564042"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HIVE enables a single human to coordinate up to 2,000 agents via natural language with an LLM. Claude 3.5 Sonnet performed best across five ability tests (19/50 wins), followed by Gemini 2 (18/50). Human-machine collaboration outperforms LLM-alone planning, and LLMs struggle with visual/spatial map understanding, performing better with textual map descriptions than images. Even state-of-the-art LLMs are sensitive to minor prompt wording variations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references a project page at hive.syrkis.com with videos, and the game/framework is described as implemented in JAX. While no explicit GitHub URL is given in the paper text, the project page serves as a public artifact."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link is provided. The benchmark scenarios, maps, and prompts are described in the appendix but no downloadable data archive is referenced."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions JAX and Python but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the system and prompts in detail in the appendix but does not include a reproducibility guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as counts of wins/losses/ties out of 10 prompts and box plots (Fig. 5), but no confidence intervals or error bars on win rates are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 9 LLMs and claims Sonnet performs best, but no statistical significance tests are used. Comparisons are based on raw counts and visual inspection of box plots."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes reported. Results are presented as win/loss/tie counts and percentage metrics in box plots without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "10 prompt variations per ability test per model is used without justification for why 10 was chosen or whether this is sufficient to distinguish model performance."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Box plots in Fig. 5 show the distribution (quartiles and spread) of continuous metrics across the 10 prompt variations for each model and ability test."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine LLMs are compared including Llama3-8B as a small open-source baseline (Sec. IV-B). The paper also compares HIVE with vs. without human help (Sec. V-B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models tested include GPT-4o, Claude 3.5 Sonnet, Gemini 2.0, and o1-mini, which were contemporary state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies: temperature effect (Sec. B/Fig. 11), scaling with number of units (Sec. V-A), with vs. without human help (Sec. V-B), and text vs. image map descriptions (Sec. V-C)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses win/loss/tie counts, percentage of enemies eliminated, covered distance to objective, and plan validity as evaluation metrics across different ability tests."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable here — the benchmark has deterministic game outcomes (win/loss/tie) that serve as ground truth evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a train/test setup. The benchmark evaluates LLMs zero-shot on game scenarios without any training or tuning phase."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all five ability tests (Coordinate, Exploit weakness, Follow markers, Exploit terrain, Strategize points) per model in Table I and Figs. 5-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: LLMs' inability to extract positions from raw data (Sec. VI), visual-spatial reasoning failures (Sec. V-C), sensitivity to prompt wording, and Llama3-8B's inability to follow structured output."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: HIVE without human help performs worse (Sec. V-B), image-based map input degrades performance vs. text (Sec. V-C), Llama3-8B completely fails, most models have <50% success rates on most tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'promising results' on multi-agent tasks and 'critical limitations' — both are supported by Table I showing partial success and the discussion of spatial/strategic planning failures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are modest and supported by controlled ablations: human help improves performance (Sec. V-B, controlled comparison), text descriptions outperform images (Sec. V-C, same prompts/models). The ablation designs are single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to the specific game benchmark and tested models. The title references 'A Framework and Benchmark' rather than making general claims. Discussion acknowledges limitations of the specific setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses alternative explanations: poor Exploit weakness performance may be due to LLMs' inability to process numerical data (not strategic reasoning per se), and visual failures may be due to out-of-distribution map images."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures game win/loss outcomes directly and does not overclaim beyond them. The five ability tests are clearly defined operational measures, and the paper does not frame them as proxies for broader capabilities."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified in Sec. IV-B: 'gpt-4o-2024-11-20', 'claude-3-5-sonnet-20241022', 'gpt-4o-mini-2024-07-18', 'o1-mini-2024-09-12', 'claude-3-5-haiku-20241022', 'gemini-2.0-flash-exp', 'gemini-1.5-pro', 'gemini-1.5-flash'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All 50 player prompts are provided in Appendix Sec. I, and the full system prompt instruction is provided in Appendix Sec. E1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is explicitly discussed with an ablation study (Appendix B), and the paper states temperature was set to 0 for all models except o1-mini (fixed at 1)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The HIVE framework is described in detail in Section III including the plan structured output, behavior tree assignment, objective checking, and game-to-text conversion pipeline (Fig. 2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how game state is converted to text for the LLM (Sec. III-E), how maps are described textually (Sec. III-E2), and how plans are parsed and validated (Sec. III-D1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Discussion) serves as a substantial limitations discussion, covering LLM inference time, visual reasoning challenges, structured output limitations, and numerical processing issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LLMs' sensitivity to prompt wording variations (Sec. IV-C), inability to process mathematical/positional data (Sec. VI), non-determinism even at temperature 0 (Sec. VI citing [38])."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states scope boundaries: single plan inference per game (not real-time replanning), specific game mechanics (4 terrain types, 3 unit types), and notes that results may not transfer to more complex games."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (individual game logs, plan outputs per run) is not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experimental procedure is well-described: 10 prompt variations per ability test per model, deterministic replay with saved plans and random seeds (Sec. III-C), API calls to each LLM."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The benchmark uses automated LLM evaluations on predefined scenarios."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: player prompt → LLM API call → structured output parsing → plan validation → game execution → outcome recording (Sec. III-C, III-D)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in Acknowledgments: 'Funded by the armasuisse S+T project F00-007.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: IT University of Copenhagen and armasuisse Science+Technology. No authors are affiliated with the LLM providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "armasuisse (Swiss defense procurement agency) funds the research but has no financial stake in which LLM performs best on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 9 models tested."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any game scenarios, behavior tree formats, or similar strategy game tasks appeared in LLM training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is novel (reducing contamination risk), but the paper does not discuss this advantage or address contamination at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference wall-clock times are reported in Fig. 10 and Sec. IV-C with medians and quartiles per model (e.g., '4o with 6.4 s [3.3 s, 12.2 s]', 'Sonnet with 12.6 s [9.9 s, 14.2 s]')."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated (total API costs, total number of API calls, GPU hours for game simulation, etc.)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses deterministic game replay with saved random seeds but does not report sensitivity to different random seeds. Variation comes from prompt wording, not seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Clearly stated: 10 prompt variations per ability test per model, for a total of 50 evaluations per model (Sec. IV-C)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Temperature was set to 0 based on an ablation, but no other hyperparameter tuning is discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Temperature selection is justified via the ablation study in Appendix B showing that higher temperatures increase invalid plans. The choice of temperature 0 is data-driven."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Nine models compared across five tasks with no statistical tests at all, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built the HIVE framework and benchmark, which may favor certain LLM behaviors. This author-evaluation bias is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models have very different inference costs (3.9s to 12.6s median) and capabilities, but performance is not analyzed as a function of compute. The paper notes inference times but does not formally relate them to performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the five ability tests actually measure the claimed coordination capabilities or whether game outcomes could be achieved through simpler strategies."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "All models use the same HIVE scaffold, so the scaffold is not a confound in model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether similar strategy game tasks or the structured output format appeared in model training data before the benchmark was created."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the game state information provided to models leaks solution information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 10 prompt variations per task are intentionally similar — the paper acknowledges sensitivity to wording but does not analyze independence of these samples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "HIVE can coordinate swarms of up to 2,000 agents through natural language dialog with an LLM",
       "evidence": "Scaling study in Sec. V-A/Fig. 7 tests up to 4,000 units (2,000 per side). Sonnet produces valid plans at all tested scales.",
       "supported": "moderate"
     },
     {
       "claim": "Claude 3.5 Sonnet is the best-performing LLM on the HIVE benchmark",
       "evidence": "Table I shows Sonnet achieves 19/50 total wins, the highest among all models. Sonnet is the only model to succeed on all five ability tests.",
       "supported": "moderate"
     },
     {
       "claim": "HIVE performs better with human help than without",
       "evidence": "Sec. V-B/Fig. 8 compares with and without human prompts. Win rates decrease and median performance drops without human guidance.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform significantly worse with visual map input than textual descriptions",
       "evidence": "Sec. V-C/Fig. 9 shows win rates and median performance drop when replacing text descriptions with raw images, grid images, or scaffolding images.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are sensitive to minor prompt wording variations",
       "evidence": "Sec. IV-C notes 'slight changes in the prompt result in drastic changes in the plan and the execution.' Fig. 6 shows variable outcomes across prompt variations.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests for model comparisons",
       "detail": "Nine LLMs are compared and ranked with only 10 trials each, using raw win counts without any statistical significance testing. The difference between Sonnet (19/50) and Gemini 2 (18/50) may not be meaningful."
     },
     {
       "flag": "Small sample size for strong claims",
       "detail": "10 prompt variations per ability test per model is a small sample. Given the acknowledged sensitivity to prompt wording, the results may not be stable."
     },
     {
       "flag": "No contamination discussion for a novel benchmark",
       "detail": "While the benchmark is novel (reducing contamination risk), the structured output format and strategy game concepts may appear in training data. This is not discussed."
     },
     {
       "flag": "Potential author-benchmark bias",
       "detail": "The authors designed both the benchmark and the HIVE system. The structured output format, behavior trees, and scenario design may inadvertently favor certain model behaviors without independent validation."
     }
   ],
   "cited_papers": [
     {
       "title": "Human-level play in the game of Diplomacy by combining language models with strategic reasoning",
       "authors": ["E. Dinan", "G. Farina", "C. Flaherty"],
       "year": 2022,
       "relevance": "Seminal work on LLMs achieving human-level strategic game play through language coordination."
     },
     {
       "title": "SwarmBrain: Embodied agent for real-time strategy game StarCraft II via large language models",
       "authors": ["X. Shao", "W. Jiang", "F. Zuo", "M. Liu"],
       "year": 2024,
       "relevance": "Hybrid LLM approach combining high-level strategic planning with low-level tactical execution in RTS games."
     },
     {
       "title": "Large Language Model based Multi-Agents: A Survey of Progress and Challenges",
       "authors": ["T. Guo", "X. Chen", "Y. Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based multi-agent systems covering agent profiling, communication, and environment interaction."
     },
     {
       "title": "LLM as a Mastermind: A Survey of Strategic Reasoning with Large Language Models",
       "authors": ["Y. Zhang", "S. Mao", "T. Ge"],
       "year": 2024,
       "relevance": "Survey of LLM strategic reasoning capabilities relevant to evaluating LLM coordination abilities."
     },
     {
       "title": "BALROG: Benchmarking Agentic LLM and VLM Reasoning On Games",
       "authors": ["D. Paglieri"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLM/VLM reasoning in game environments, showing limitations in complex dynamic settings."
     },
     {
       "title": "Voyager: An open-ended embodied agent with large language models",
       "authors": ["G. Wang", "Y. Xie", "Y. Jiang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "LLM-based agent generating code for game control in Minecraft, an alternative approach to HIVE's structured output."
     },
     {
       "title": "JaxMARL: Multi-Agent RL Environments in JAX",
       "authors": ["A. Rutherford", "B. Ellis", "M. Gallici"],
       "year": 2023,
       "relevance": "Foundation environment for multi-agent RL research using JAX parallelization, directly informs HIVE's implementation."
     },
     {
       "title": "Vision language models are blind",
       "authors": ["P. Rahmanzadehgervi", "L. Bolton", "M. R. Taesiri"],
       "year": 2025,
       "relevance": "Documents VLM failures on trivial spatial reasoning tasks, supporting HIVE's finding that image-based map input fails."
     },
     {
       "title": "An empirical study of the non-determinism of chatgpt in code generation",
       "authors": ["S. Ouyang", "J. M. Zhang", "M. Harman"],
       "year": 2025,
       "relevance": "Studies LLM non-determinism at temperature 0, directly relevant to HIVE's methodology choices."
     },
     {
       "title": "AgentCoord: Visually Exploring Coordination Strategy for LLM-based Multi-Agent Collaboration",
       "authors": ["B. Pan", "J. Lu", "K. Wang"],
       "year": 2024,
       "relevance": "Visual interface for LLM multi-agent coordination strategies, complementary approach to HIVE."
     }
   ]
 }