ai-research-survey

scan.json (26405B)

---

 {
   "paper": {
     "title": "BattleAgentBench: A Benchmark for Evaluating Cooperation and Competition Capabilities of Language Models in Multi-Agent Systems",
     "authors": ["Wei Wang", "Dan Zhang", "Tao Feng", "Boyan Wang", "Jie Tang"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2408.15971",
     "doi": "10.48550/arXiv.2408.15971"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code for BattleAgentBench is available at https://github.com/THUDM/BattleAgentBench.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is the artifact itself — the game environment and stage configurations are released via the GitHub repository. The benchmark does not rely on a separate static dataset; the evaluation data is generated through game interactions using the released code."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of a requirements.txt, Dockerfile, conda environment, or detailed environment setup with library versions found in the paper. The paper does not describe what software dependencies are needed beyond the LLM APIs."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While the code is released on GitHub, the paper itself does not describe how to run the experiments, what commands to execute, or how to replicate the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-5 are reported as point estimates (e.g., scores, accuracy) with no confidence intervals, error bars, or ± notation. The appendix (Section 7.1) mentions running multiple times (3 for API-based, 5 for open-source), but no uncertainty measures are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'claude3.5-sonnet and gpt-4o-mini achieved relatively high scores, while other models scored comparatively lower') but provides no statistical significance tests to support these claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw scores and accuracy values but does not provide effect sizes. The ablation study (Table 5) reports ΔScore but without baseline context or standardized effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The appendix states API-based models run each stage 3 times and open-source models 5 times, but there is no justification for why these numbers were chosen and no power analysis. The asymmetric run counts (3 vs 5) are not explained."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running each stage multiple times (3 or 5 runs), the paper reports only averaged scores in the tables. No standard deviations, ranges, or other variance measures are reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a random baseline in all evaluation tables (Tables 2-4), showing the expected performance of random actions. This provides a lower-bound comparison for all models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary models at the time of writing (2024): Claude 3.5 Sonnet (June 2024), GPT-4o-mini, Gemma2-9b, Qwen2-7b, and Yi-1.5-9b. These represent recent API-based and open-source models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents an ablation study that removes the cooperation interface between agents in Level 3, comparing Table 4 (with cooperation) to Table 5 (without cooperation) to assess the effectiveness of the collaboration component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Forward Distance (F Dis), Format Accuracy (F Acc), Move Accuracy (M Acc) for Level 1, and Score, F Acc, M Acc for Levels 2-3. Goal completion rate is also computed from forward distance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes claims about cooperation quality and strategic behavior (Section 4.4 case study describes cooperation strategies qualitatively), but all evaluation is automated through game metrics (scores, accuracy). No human evaluation of cooperation quality, strategy sophistication, or agent decision-making is included."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark evaluates LLMs as game agents in a live interactive environment. There is no static train/test split — each evaluation generates new game states through agent-environment interaction with randomly sampled initial positions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by stage (7 stages across 3 levels) and by model category (API-based vs open-source), with separate tables for each level (Tables 2, 3, 4). Per-model results are shown for all 11 models plus a random baseline."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses that cooperation among weaker models actually lowers their scores, indicating 'cooperation between these models is ineffective or even harmful.' The case study (Section 4.4, Figure 6) shows qualitative examples of agent behavior, though failure cases could be more detailed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Section 4.3 reports that removing cooperation actually improves scores for most models except claude3.5-sonnet and gpt-4o-mini, demonstrating that cooperation is counterproductive for weaker models. Some open-source models (e.g., llama3-8b-instruct) perform worse than random."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (1) API-based models perform excellently on simple tasks — confirmed in Table 2; (2) open-source small models struggle with simple tasks — confirmed in Table 2 and Figure 5; (3) API-based models have demonstrated some collaborative capabilities but with room for improvement — confirmed in the ablation study (Section 4.3)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that cooperation improves performance for some models (Section 4.3). This is supported by a controlled ablation study comparing performance with and without the cooperation interface (Table 4 vs Table 5), which is an adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'Cooperation and Competition Capabilities of Language Models in Multi-Agent Systems' generally, but the evaluation is specific to a single game environment (Battle City tank game). The paper does not discuss whether findings generalize to other cooperative/competitive multi-agent settings beyond this one game."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, the performance gap between API and open-source models could be due to model size, instruction tuning quality, spatial reasoning ability, or format following capability, but these are not disentangled or discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Most model versions are not fully specified. 'claude3.5-sonnet-0620' is specified with a date. 'gpt-3.5-turbo-0125' has a version. However, 'gpt-4o-mini' lacks a snapshot date. Open-source models like 'internlm2.5-7b-chat', 'mistral-7b-instruct', 'qwen2-7b-Instruct' lack specific version identifiers or checkpoint dates. The specification is inconsistent across models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for all 7 stages are provided in the appendix (Figures 1-7 in Section 7.2), including game state data format, goals, game instructions, operation options, and output format. The templates contain placeholders (%s, %d) for dynamic game state data, but the fill values are the actual game state which varies per turn — the templates themselves are complete and reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other sampling parameters used when calling the LLM APIs or running open-source models. These settings can significantly affect agent behavior."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent scaffolding is described in detail: Section 3.1 describes the agent interaction framework (game server sends observations, agents return actions), the communication interface for cooperation (Section 3.3-3.5), the primary/secondary agent setup (Section 4.1), and the MDP formulation (Section 2). The prompts include thought process templates for agent reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how 'game observation data is converted into text format based on predefined templates.' The appendix (Section 7.1) documents the map generation process (random sampling of agent positions within a small range) and stage settings (Table 6: turns, agents, teams, NPCs per stage)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section 6) is a brief summary without any discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no discussion of specific methodological concerns such as the limited number of runs, the choice of reference model (yi-1.5-9b), or the single-game-environment limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of scope boundaries — e.g., that findings are specific to the Battle City game environment, that the cooperation metric depends on the reference model choice, or that results may not transfer to other multi-agent settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release raw game logs, model outputs, or individual run data. Only averaged results are reported in the tables. The code is released on GitHub, but the actual experimental data (game trajectories, model responses) is not."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and the appendix (Section 7.1) describe how data is collected: each stage runs for a specified number of turns (Table 6), with random map generation, multiple runs per model (3 for API, 5 for open-source), and averaged scores. The evaluation metrics and their computation are formally defined (Equations 2-4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is LLM game agent interactions, and the models evaluated are standard public benchmarks/APIs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: game server loads stage settings → sends observations to agents → agents (LLMs) return actions → server updates state and calculates rewards → process repeats for specified turns → metrics computed from final states. The evaluation setup (Section 7.1) describes the run procedure and averaging."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. The authors are from Tsinghua University's KEG group, but no grants or sponsorship are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all five authors are from 'The Knowledge Engineering Group (KEG), Tsinghua University.' The authors are affiliated with an academic institution, not with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed at all, independence of the funder cannot be assessed. The absence of funding disclosure does not satisfy this criterion."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLM capabilities on game reasoning tasks. While the benchmark is novel, the training cutoffs of the evaluated models are not stated. This matters because models may have been trained on similar game reasoning tasks or the Battle City game itself."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "There is no discussion of whether any evaluated models may have been exposed to Battle City game data, game strategy data, or similar game reasoning benchmarks during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While BattleAgentBench is a novel benchmark with dynamically generated maps, the underlying game is Battle City (a classic 1985 NES game). Models may have been trained on descriptions of Battle City rules, strategies, and gameplay, potentially advantaging some models. The paper does not discuss this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The appendix (Section 7.1) mentions that 'the setting of the number of turns takes into account the cost factor of calling LLM' but does not report actual inference costs, API costs, tokens consumed, or wall-clock time for the evaluations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report total API costs, GPU hours for running open-source models, or overall evaluation time."
       }
     }
   },
   "claims": [
     {
       "claim": "API-based models perform excellently on simple tasks but open-source small models struggle with simple tasks.",
       "evidence": "Table 2 shows API-based models achieve significantly higher Forward Distance scores (e.g., claude3.5-sonnet: 12.8 avg, gpt-4o-mini: 11.5 avg) compared to open-source models (best: internlm2.5-7b at 5.7 avg). Figure 5 shows goal completion rates above 50% only for API-based models.",
       "supported": "strong"
     },
     {
       "claim": "In difficult tasks requiring collaborative and competitive abilities, API-based models have demonstrated some collaborative capabilities but there is still enormous room for improvement.",
       "evidence": "Tables 3-4 show that only claude3.5-sonnet and gpt-4o-mini achieve notable scores in Levels 2-3. The ablation study (Table 5, Section 4.3) shows cooperation only improves scores for these two models. Other API models (gpt-3.5, glm-4-flash) show minimal advantage over open-source models in complex stages.",
       "supported": "strong"
     },
     {
       "claim": "Only claude3.5-sonnet and gpt-4o-mini benefit from cooperation; cooperation is ineffective or harmful for other models.",
       "evidence": "Section 4.3 and Table 5: After removing the cooperation interface, only claude3.5-sonnet and gpt-4o-mini show decreased scores (ΔScore: -2.3 and -0.3 respectively), while all other models show increased scores, indicating cooperation hurts their performance.",
       "supported": "strong"
     },
     {
       "claim": "Claude3.5-sonnet maintains stable spatial awareness ability across increasingly complex stages.",
       "evidence": "Tables 2-4 consistently show claude3.5-sonnet maintaining high Move Accuracy (M Acc ≥ 0.87) across all stages, while other models show declining M Acc as complexity increases.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source small models have poor two-dimensional spatial perception.",
       "evidence": "Table 2 shows open-source models have Move Accuracy around 0.41-0.65, compared to 0.83-0.98 for API-based models. llama3-8b-instruct actually moves backward (negative forward distance: -1.3 avg), performing worse than random.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "BattleAgentBench evaluates 11 LLMs across 7 stages of increasing difficulty in a Battle City tank game, testing navigation, cooperation, and competition capabilities. API-based models (especially Claude 3.5 Sonnet and GPT-4o-mini) significantly outperform open-source 7-9B models on both simple navigation and complex multi-agent tasks. An ablation study removing the cooperation interface shows that only Claude 3.5 Sonnet and GPT-4o-mini genuinely benefit from inter-agent cooperation, while cooperation actually hurts performance for all other models tested.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification despite multiple runs",
       "detail": "The paper runs each stage 3 times (API models) or 5 times (open-source models) and reports averages, but never reports standard deviations, confidence intervals, or any variance measure. This makes it impossible to assess whether observed differences are meaningful or within noise."
     },
     {
       "flag": "Asymmetric evaluation conditions",
       "detail": "API-based models run each stage 3 times while open-source models run 5 times. This asymmetry is not justified and could introduce bias in the averaged results."
     },
     {
       "flag": "Reference model choice may bias multi-agent results",
       "detail": "In Levels 2-3, secondary agents use yi-1.5-9b as a fixed reference model. Since yi-1.5-9b is one of the weakest models tested, the evaluated primary agent is always paired with or competing against weak opponents. This may inflate scores for stronger models and depress scores for weaker models differently than a stronger reference would."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries. Major concerns like single-game generalizability, reference model dependence, and small run counts are not acknowledged."
     },
     {
       "flag": "Broad claims from narrow evaluation",
       "detail": "The paper claims to evaluate 'Cooperation and Competition Capabilities of Language Models in Multi-Agent Systems' but tests only a single game environment (Battle City). The cooperative behavior tested (sending text messages to coordinate tank movements) is a very specific form of cooperation that may not generalize."
     },
     {
       "flag": "Missing LLM hyperparameters",
       "detail": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the 11 models. These settings significantly affect LLM output behavior and make results non-reproducible even with the released code."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Major benchmark for evaluating LLMs as agents across diverse interactive environments, directly relevant to the survey's scope of LLM capability evaluation."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Foundational multi-agent LLM framework relevant to agentic AI architectures and workflows."
     },
     {
       "title": "Large Language Model based Multi-Agents: A Survey of Progress and Challenges",
       "authors": ["Taicheng Guo", "Xiuying Chen", "Yaqi Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of LLM-based multi-agent systems covering progress and challenges, directly relevant to the survey's coverage of multi-agent AI research."
     },
     {
       "title": "A Survey on Large Language Model-based Game Agents",
       "authors": ["Sihao Hu", "Tiansheng Huang", "Fatih Ilhan"],
       "year": 2024,
       "arxiv_id": "2404.02039",
       "relevance": "Survey covering LLM-based game agents across multiple game genres, relevant to benchmark evaluation methodology for agent capabilities."
     },
     {
       "title": "AvalonBench: Evaluating LLMs Playing the Game of Avalon",
       "authors": ["Jonathan Light", "Min Cai", "Sheng Shen"],
       "year": 2023,
       "relevance": "Game-based benchmark for evaluating LLM social deduction and cooperation capabilities, directly comparable to BattleAgentBench."
     },
     {
       "title": "Building Cooperative Embodied Agents Modularly with Large Language Models",
       "authors": ["Hongxin Zhang", "Weihua Du", "Jiaming Shan"],
       "year": 2024,
       "relevance": "ICLR paper on modular cooperative embodied agents using LLMs, relevant to multi-agent cooperation evaluation methodology."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for one of the major LLM families evaluated in agent benchmarks, relevant to understanding model capabilities."
     },
     {
       "title": "Suspicion-Agent: Playing Imperfect Information Games with Theory of Mind Aware GPT-4",
       "authors": ["Jiaxian Guo", "Bo Yang", "Paul Yoo"],
       "year": 2023,
       "arxiv_id": "2309.17277",
       "relevance": "Evaluates LLM agent capabilities in game-theoretic settings, relevant to competitive multi-agent evaluation."
     },
     {
       "title": "MindAgent: Emergent Gaming Interaction",
       "authors": ["Ran Gong", "Qiuyuan Huang", "Xiaojian Ma"],
       "year": 2023,
       "arxiv_id": "2309.09971",
       "relevance": "Assesses multi-agent collaboration efficiency in gaming environments, directly comparable benchmark evaluation methodology."
     },
     {
       "title": "Agent-Pro: Learning to Evolve via Policy-Level Reflection and Optimization",
       "authors": ["Wenqi Zhang", "Ke Tang", "Hai Wu"],
       "year": 2024,
       "arxiv_id": "2402.17574",
       "relevance": "Evaluates LLM agent strategy learning in competitive settings, relevant to agentic AI capability evaluation."
     }
   ]
 }