ai-research-survey

scan.json (24221B)

---

 {
   "paper": {
     "title": "TeamCraft: A Benchmark for Multi-Modal Multi-Agent Systems in Minecraft",
     "authors": ["Qian Long", "Zhi Li", "Ran Gong", "Ying Nian Wu", "Demetri Terzopoulos", "Xiaofeng Gao"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2412.05255",
     "doi": "10.48550/arXiv.2412.05255"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/teamcraft-bench/teamcraft. Paper states 'we open source the entire platform, its training and evaluation code, and release the model checkpoints and training data.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Paper states dataset will be available on Huggingface (Appendix K) and training data and model checkpoints are released via GitHub."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 lists hyperparameters, model architecture details (CLIP ViT-L/14, Vicuna-v1.5), and training used 8 A100 GPUs. However, no requirements.txt or Dockerfile mentioned explicitly in the paper text. The released code repository would contain this. Borderline — the paper itself specifies key dependencies (CLIP, Vicuna, Mineflayer) but not a full environment spec."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper itself. The paper refers to the GitHub repo but does not include a 'Reproducing Results' section with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 6-8 and Figure 5 report single point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., centralized outperforms decentralized, grid-world outperforms VLA) but no statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute success rates with baseline context (e.g., Table 6 shows 0.42 vs 0.00 task success rate for centralized vs decentralized building), allowing readers to assess magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test set has 50 samples per task and 50 per generalization condition (950 total). No justification for why these sizes were chosen or whether they are sufficient for statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported across experimental runs. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Paper compares TeamCraft-VLA (7B and 13B), GPT-4o, and a grid-world text-based LLM baseline across centralized and decentralized settings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o (gpt-4o-2024-08-06) is a contemporary proprietary model. The fine-tuned baselines use Vicuna-v1.5 and CLIP ViT-L/14, which are reasonable for 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Paper performs 15 ablation studies varying dataset size (10%, 50%, 100%), control settings (centralized/decentralized), modalities (multi-modal vs grid-world), and model sizes (7B, 13B)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics used: task success rate, subgoal success rate, and redundancy rate (Equations 2-4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper evaluating automated agent performance in a simulated environment. Human evaluation of agent outputs is not clearly relevant to the claims about agent generalization capabilities."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Paper explicitly separates training, test, and generalization sets. The generalization set uses held-out elements (novel shapes, materials, scenes, 4 agents) excluded from training data (Section 3.7)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 6-8 provide detailed per-task (building, clearing, farming, smelting) and per-generalization-condition (test, shape, material, scene, agents) breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides qualitative failure analysis with specific examples: object mismatching, task allocation failure, and object state recognition failure. Appendix G.4 provides additional detailed failure case studies with figures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings reported: GPT-4o fails almost all tasks, farming crop generalization is 0% across all models, more training data can decrease performance in grid-world setting (overfitting), decentralized agents have high redundancy rates."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that 'existing models continue to face significant challenges in generalizing to novel goals, scenes, and unseen numbers of agents' are well-supported by the experimental results showing low success rates, especially in generalization splits."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily from ablation studies (e.g., removing visual modality in grid-world, varying dataset size, comparing centralized vs decentralized). These are controlled single-variable manipulations and the ablation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the Minecraft TeamCraft environment. The paper frames findings as challenges within the benchmark rather than making broad claims about all multi-agent systems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: grid-world outperformance suggests visual processing is the bottleneck (not planning); more data lowering performance in grid-world suggests overfitting; centralized vs decentralized gap is attributed to information availability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures task success rate, subgoal success rate, and redundancy rate directly — these are well-defined metrics that match the claims about agent performance. No proxy gap exists between measurement and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o version specified as 'gpt-4o-2024-08-06' (Appendix H). VLA models use CLIP ViT-L/14 and Vicuna-v1.5 with specific sizes (7B, 13B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided for GPT-4o in Appendix H (Figures 17, 19, 21) including system prompts and user prompts. Grid-world prompts in Appendix E (Figures 7-10)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports hyperparameters: lr=2e-5, model max length=4096, patch size=14, resolution=336x336, optimizer=AdamW, lr scheduler=constant_with_warmup, warmup ratio=0.03."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The platform architecture is described in Figure 1 and Section 3.2. High-level skills are translated to low-level Mineflayer API calls. Table 4 documents all 8 skill types. Appendix A provides detailed skill descriptions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 describes task generation via rejection sampling and planner-based demonstration generation with the cost function (Equation 1). Appendix D provides planner weight details per task."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Limitations and Future Work' lists four specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this work: reliance on MineFlayer oracle controller, training only on procedural demonstrations (not human), decentralized agents lack explicit communication, no human-AI collaboration tested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly states what was not tested: low-level control (uses oracle controller), human demonstrations, explicit communication between agents, human-AI collaboration."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on Huggingface with 55,000+ demonstrations. Model checkpoints also released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 describes task generation (rejection sampling from diversity pools) and demonstration generation (planner-based with cost function optimization). Appendix D provides detailed planner parameters."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is procedurally generated in a simulated environment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: variables sampled → rejection sampling for solvable tasks → planner assigns actions → demonstrations recorded with prompts, observations, and actions. Statistics in Table 2 and Appendix J."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: UCLA and Amazon AGI. The Amazon author includes a footnote: 'This work does not relate to the author's position at Amazon.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. One author is from Amazon AGI, which has a financial interest in multi-agent AI systems, though the footnote disclaims connection."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used for evaluation but its training data cutoff is not stated. The fine-tuned models use only the TeamCraft data, but GPT-4o's training data is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-4o, no discussion of whether Minecraft-related task solutions might appear in training data. The custom benchmark mitigates this risk but it is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TeamCraft is a new benchmark, which inherently reduces contamination risk for GPT-4o, but this advantage is not explicitly discussed or claimed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported for GPT-4o API calls or VLA model inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training compute stated: 8 A100 GPUs, 7B centralized takes 36 hours, 13B takes 72 hours, decentralized doubles training time, grid-world 7B takes 20 hours (Appendix F)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each model configuration was trained once for 3 epochs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Hyperparameters in Table 5 appear chosen without documented search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Models trained for 3 epochs 'before convergence' but no validation-based selection criterion or early stopping details provided."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own TeamCraft-VLA model against GPT-4o and grid-world baselines without acknowledging self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o uses one-shot prompting vs fine-tuned VLA models trained on 55K demonstrations. This massive compute/data asymmetry is not discussed as a confound when comparing performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the Minecraft-based tasks actually measure the collaborative multi-agent capabilities claimed. No analysis of construct validity or comparison with alternative benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TeamCraft-VLA uses a fine-tuned architecture while GPT-4o uses one-shot prompting with different prompt structures. This scaffolding difference is not addressed when comparing their performance."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data could include Minecraft task solutions or related demonstrations."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., inventory contents revealing optimal strategies)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The procedurally generated tasks share structural templates. No discussion of whether train/test tasks are sufficiently independent despite using the same generation process."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing models face significant challenges in generalizing to novel goals, scenes, and unseen numbers of agents.",
       "evidence": "Tables 6-8 and Figure 5 show performance drops across generalization splits, particularly for novel goals (farming crop generalization is 0% for all models) and 4-agent scenarios.",
       "supported": "strong"
     },
     {
       "claim": "Centralized control significantly outperforms decentralized control across nearly all task variants.",
       "evidence": "Figure 6 and Tables 6-7 show centralized models consistently achieving higher task success rates. Redundancy rates in Table 3 show 0.01 for centralized vs 0.15 for decentralized.",
       "supported": "strong"
     },
     {
       "claim": "Grid-world (text-based) models significantly outperform VLA models in multi-modal settings.",
       "evidence": "Table 8 shows grid-world 7B achieving 1.00 clearing success rate vs 0.64 for VLA-7B, and 0.78 farming test vs 0.36 for VLA-7B. Figure 5 confirms the pattern.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o fails on almost all test cases due to lack of 3D spatial reasoning.",
       "evidence": "Figure 6 shows GPT-4o near-zero task success rates. Section 4.3 notes GPT-4o struggles with mapping block coordinates from visual inputs.",
       "supported": "strong"
     },
     {
       "claim": "Scaling model size from 7B to 13B does not guarantee improved performance, especially with sufficient data.",
       "evidence": "Figure 5 shows 7B model approaching 13B performance as data increases. Tables 6-7 confirm similar patterns.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TeamCraft introduces a 55,000-variant multi-modal multi-agent benchmark in Minecraft with four task types. Experiments reveal that current VLA models struggle significantly with generalization to novel goals (0% on unseen crops) and unseen numbers of agents. Text-based grid-world representations dramatically outperform visual inputs, exposing a major gap in VLA models' visual understanding. Centralized control vastly outperforms decentralized settings, highlighting the challenge of implicit coordination without shared information.",
   "red_flags": [
     {
       "flag": "Unfair GPT-4o comparison",
       "detail": "GPT-4o uses one-shot prompting while TeamCraft-VLA is fine-tuned on 55K demonstrations. This is not an apples-to-apples comparison, yet the paper presents them side by side in Figure 6 without adequately discussing this asymmetry."
     },
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results appear to be from single training runs with no variance reported. Given that VLA training can be sensitive to initialization, single-run results may not be reliable."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Multiple comparative claims (centralized vs decentralized, 7B vs 13B, modality comparisons) are made without any statistical significance testing."
     }
   ],
   "cited_papers": [
     {
       "title": "VIMA: General Robot Manipulation with Multimodal Prompts",
       "authors": ["Y. Jiang", "A. Gupta", "Z. Zhang", "G. Wang"],
       "year": 2022,
       "arxiv_id": "2210.03094",
       "relevance": "Multi-modal prompt-based task specification for embodied agents, which TeamCraft extends to multi-agent settings."
     },
     {
       "title": "MindAgent: Emergent Gaming Interaction",
       "authors": ["R. Gong", "Q. Huang", "X. Ma"],
       "year": 2023,
       "relevance": "Multi-agent collaboration benchmark in Minecraft using LLMs, a direct predecessor to TeamCraft."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["G. Wang", "Y. Xie", "Y. Jiang"],
       "year": 2023,
       "relevance": "LLM-based agent for open-ended Minecraft tasks, relevant to agentic AI capabilities."
     },
     {
       "title": "MineDojo: Building Open-Ended Embodied Agents with Internet-Scale Knowledge",
       "authors": ["L. Fan", "G. Wang", "Y. Jiang"],
       "year": 2022,
       "relevance": "Large-scale Minecraft benchmark for embodied agents with internet-scale knowledge."
     },
     {
       "title": "Visual Instruction Tuning",
       "authors": ["H. Liu", "C. Li", "Q. Wu", "Y. J. Lee"],
       "year": 2024,
       "relevance": "Foundation for vision-language model architecture used in TeamCraft-VLA."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["J. S. Park", "J. O'Brien", "C. J. Cai"],
       "year": 2023,
       "relevance": "Multi-agent simulation with LLM-based agents, relevant to agentic AI collaboration."
     },
     {
       "title": "PARTNR: A Benchmark for Planning and Reasoning in Embodied Multi-Agent Tasks",
       "authors": ["M. Chang", "G. Chhablani", "A. Clegg"],
       "year": 2024,
       "arxiv_id": "2411.00081",
       "relevance": "Contemporary multi-agent embodied benchmark for planning and reasoning."
     },
     {
       "title": "RoCo: Dialectic Multi-Robot Collaboration with Large Language Models",
       "authors": ["Z. Mandi", "S. Jain", "S. Song"],
       "year": 2024,
       "relevance": "LLM-based multi-robot collaboration benchmark relevant to multi-agent coordination."
     },
     {
       "title": "CAMEL: Communicative Agents for Mind Exploration of Large Language Model Society",
       "authors": ["G. Li", "H. Hammoud", "H. Itani"],
       "year": 2023,
       "relevance": "Multi-agent LLM communication framework relevant to agentic AI collaboration."
     },
     {
       "title": "ProAgent: Building Proactive Cooperative Agents with Large Language Models",
       "authors": ["C. Zhang", "K. Yang", "S. Hu"],
       "year": 2024,
       "relevance": "LLM-based proactive cooperative agents, relevant to multi-agent AI systems."
     }
   ]
 }