ai-research-survey

scan.json (25475B)

---

 {
   "paper": {
     "title": "Building Cooperative Embodied Agents Modularly with Large Language Models",
     "authors": ["Hongxin Zhang", "Weihua Du", "Jiaming Shan", "Qinhong Zhou", "Yilun Du", "Joshua B. Tenenbaum", "Tianmin Shu", "Chuang Gan"],
     "year": 2023,
     "venue": "ICLR 2024",
     "arxiv_id": "2307.02485",
     "doi": "10.48550/arXiv.2307.02485"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a project website (https://vis-www.cs.umass.edu/Co-LLM-Agents/) mentioned in the abstract. This serves as a public resource for the project."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The environments used (TDW-MAT and C-WAH) are extensions of publicly available platforms (ThreeDWorld and VirtualHome-Social/Watch-And-Help), and the paper describes the test sets in detail (24 episodes for TDW-MAT, 10 for C-WAH). The environments are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training a Mask-RCNN, using GPT-4 API, LLAMA-2-13b-chat, and LoRA fine-tuning, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides detailed descriptions of the framework, modules, and prompts in the appendix, there are no explicit step-by-step reproduction instructions (e.g., a README with commands to run)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1 and 2 report only point estimates (e.g., Transport Rate of 0.69, Average Steps of 59) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports a t-test p-value (p=0.0003) for the trust score comparison between CoELA and CoELA w/o communication in the user study (Section 5.3.2). However, no significance tests are reported for the main AI-AI cooperation results."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Efficiency Improvement percentages with baseline context throughout (e.g., '45% efficiency improvement' for MHP+CoELA vs MHP alone, trust score of '6.3 vs. 4.7'), providing magnitude context for the improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set sizes (24 episodes for TDW-MAT, 10 for C-WAH) and user study size (8 subjects, 80 trials) are not justified. No power analysis is provided. The user study with 8 subjects is particularly small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports results 'over 5 runs for RHP and 1 run for CoELA due to cost constraints.' CoELA results are single-run with no variance reported. The baselines report averages over 5 runs but without standard deviation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against three baselines: RHP (Rule-based Hierarchical Planner), MHP (MCTS-based Hierarchical Planner), and MAT (Multi-Agent Transformer) across both environments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are adopted from the original challenges (ThreeDWorld Transport Challenge and Watch-And-Help Challenge), representing the strongest known methods for these specific tasks. MAT (2022) is contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4c shows ablation results: removing communication, removing memory module, replacing GPT-4 with GPT-3.5, and removing the execution module. These test the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Transport Rate and Efficiency Improvement for TDW-MAT, Average Steps and Efficiency Improvement for C-WAH, and three subjective rating criteria (effectiveness, helpfulness, trust) for the user study."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3.2 describes a user study with 8 human subjects who cooperated with CoELA and rated the agents on 7-point Likert scales for communication effectiveness, helpfulness, and trust."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses separate training and test sets: '2 floorplans are for the training set and another two are for the test set' (Appendix B.1). CoLLAMA is trained on training set data and evaluated on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-category breakdown for TDW-MAT (Food vs. Stuff subtasks), and results are reported separately for each environment. The user study reports three separate rating dimensions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Failure Cases and Limitations of LLM' discusses three types of failures: limited 3D spatial reasoning, lack of low-level action reasoning, and unstable complex reasoning, with specific examples in Figure 5."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: LLAMA-2 'still underperform' significantly (0.15 TR vs 0.85 for GPT-4 with oracle perception); communication did not significantly improve AI-AI cooperation ('We did not observe a significant performance drop when disabling communication among AI agents'); removing execution module caused all trials to fail."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CoELA 'can surpass strong planning-based methods' (supported by Tables 1-2), 'exhibit emergent effective communication' (supported by qualitative analysis in Figure 3), and CoLLAMA achieves 'promising performance' (supported by Table 1). The trust claim from user study is supported by Figure 4b."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (removing memory module doubles steps, removing communication has minimal effect on AI-AI cooperation). These are controlled single-variable manipulations within the framework, which is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Building Cooperative Embodied Agents Modularly with Large Language Models' is broad, but results are on two specific simulated environments (TDW-MAT and C-WAH) with household rearrangement tasks only. The abstract claims about 'multi-agent cooperation' are not bounded to these specific domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the improvements. For example, the performance gains could partly come from the handcrafted prompts, the action list design, or the specific task structure rather than LLM reasoning. No threats-to-validity section is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4 from the OpenAI API' with a footnote 'Our main experiments are done between 2023.9.1-2023.9.28 and 2023.5.1-2023.5.16' but does not specify the exact model version/snapshot (e.g., gpt-4-0613). It also uses 'GPT-3.5' without a version string, and 'LLAMA-2-13b-chat' which is more specific."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full example prompts are provided in Appendix E (Tables 4 and 6) for both C-WAH and TDW-MAT environments, including the instruction head, goal description, state description, action/dialogue history, and action list with actual fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states 'temperature 0.7, top-p 1, and max tokens 256.' Appendix C.3 reports CoLLAMA training hyperparameters: 'batch size of 384, maximal sequence length of 2048, and a max learning rate of 4e-4 for 30 epochs.' MAT training details are in Appendix C.1: 'hidden layer dim 64, learning rate 7e-4, ppo epoch 10.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper describes the agentic scaffolding in detail: the five-module framework (Perception, Memory, Communication, Planning, Execution) is thoroughly described in Sections 4.1-4.6 with workflow diagrams (Figure 2), a detailed working example (Figure 7), and prompt design."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The perception pipeline (Mask-RCNN training, 3D point cloud construction, semantic map building) is described in detail in Section 4.2 and Appendix A.1. For CoLLAMA, Appendix C.3 documents: '2k trajectories from 10 episodes' filtered to '572 high-quality data with effective communication behavior.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Failure Cases and Limitations of LLM' provides a dedicated discussion of three specific limitations: limited 3D spatial information usage, lack of low-level action reasoning, and unstable complex reasoning."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses specific threats: the inability to incorporate spatial information causing time-consuming plans, the abstraction of high-level plans making the system unaware of low-level execution states (illustrated in Figure 5a), and LLM counting errors (Figure 5b)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to simulated environments, the specific task types tested, or acknowledge that the results may not transfer to real-world physical embodied cooperation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (trajectories, logs, LLM outputs) is made available. Only aggregated results are presented in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The test set construction is described: '6 scenes from the TDW-House dataset and sampled 2 out of the two types of tasks in each of the scenes, making a test set of 24 episodes' (Section 5.1). C-WAH: '2 tasks from each of the five types of activities to construct a test set of 10 episodes.' User study: 8 subjects, 80 trials."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the user study with 8 human subjects, the paper states 'We recruited 8 human subjects' but does not describe recruitment methods, channels, or whether recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from observation to action is well documented through the modular framework description. For CoLLAMA: '2k trajectories → manually filtered 572 high-quality data' (Appendix C.3). The Mask-RCNN training pipeline is documented: '53K 512×512 RGB images' collected by random sampling."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgement section thanks individuals for discussions and reviewers for suggestions, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UMass Amherst, Tsinghua University, Shanghai Jiao Tong University, MIT, and MIT-IBM Watson AI Lab."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Authors are affiliated with MIT-IBM Watson AI Lab, and the paper uses OpenAI's GPT-4 API, but no funding disclosure or independence statement is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 and GPT-3.5 but does not state their training data cutoff dates. This is relevant because the benchmark tasks could potentially be similar to patterns in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4's training data could include information about the ThreeDWorld or VirtualHome environments or similar household task-planning scenarios."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The environments are custom extensions, which reduces contamination risk, but this is not explicitly discussed. The paper does not address whether the LLMs' commonsense knowledge about household tasks constitutes a form of contamination advantage."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the user study with 8 human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the user study involving 8 human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics are reported for the 8 human subjects. Experience level, background, age, gender, etc. are not described."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for participant recruitment. The paper only says 'We recruited 8 human subjects.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that 'We made sure each subject do 10 trials with at least two trials under each scenario' but does not describe how trials were assigned or randomized across conditions."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants likely knew which agent they were cooperating with since the communication style would differ (natural language vs. template language), but this is not discussed."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether any participants dropped out or failed to complete all trials. The paper states 80 trials total from 8 subjects but does not report attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges cost constraints ('1 run for CoELA due to cost constraints') but does not report actual API costs, tokens consumed, or inference time per episode."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Training details for Mask-RCNN, MAT, and CoLLAMA are given, but GPU hours, total API spend, or hardware specifications are not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CoELA driven by GPT-4 surpasses strong planning-based methods, achieving 36% efficiency improvement over RHP+RHP (29%) on TDW-MAT and 45% over MHP+MHP (33%) on C-WAH.",
       "evidence": "Tables 1 and 2 show Transport Rate and Average Steps results across both environments. RHP+CoELA achieves 0.69 TR (36% EI) vs RHP+RHP 0.61 (29%) on TDW-MAT. MHP+CoELA achieves 59 steps (45% EI) vs MHP+MHP 75 steps (33%) on C-WAH.",
       "supported": "moderate"
     },
     {
       "claim": "CoLLAMA (fine-tuned LLAMA-2) achieves competitive performance with GPT-4, reaching 0.70 TR on TDW-MAT compared to GPT-4's 0.71.",
       "evidence": "Table 1 shows CoLLAMA achieving 0.70 total TR vs GPT-4's 0.71, and even surpassing GPT-4 on the Stuff subtask (0.66 vs 0.61). Fine-tuned on only 572 examples (Appendix C.3).",
       "supported": "moderate"
     },
     {
       "claim": "Humans trust CoELA communicating in natural language more than agents without communication (trust score 6.3 vs 4.7, p=0.0003).",
       "evidence": "Section 5.3.2 and Figure 4b report user study results from 8 subjects across 80 trials, with a t-test yielding p=0.0003 for the trust score difference.",
       "supported": "moderate"
     },
     {
       "claim": "Communication among AI agents does not significantly improve cooperation efficiency.",
       "evidence": "Section 5.4 states 'We did not observe a significant performance drop when disabling communication among AI agents' and Figure 4c shows the ablation results.",
       "supported": "moderate"
     },
     {
       "claim": "The Memory Module is essential, with removal nearly doubling the steps needed to finish tasks.",
       "evidence": "Section 5.4 and Figure 4c show the ablation result: 'the steps needed to finish the task for the agent with no Memory Module nearly double.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "CoELA, a modular framework using LLMs for cooperative embodied agents, outperforms strong planning-based baselines (RHP, MHP) by 36-45% efficiency improvement on two multi-agent household task environments (TDW-MAT and C-WAH). Communication between AI agents does not significantly help, though human-agent communication improves both task performance and human trust. Fine-tuned CoLLAMA (LLAMA-2 with LoRA on 572 examples) achieves performance competitive with GPT-4. The Memory Module is the most critical component, while removing the Execution Module causes complete task failure.",
   "red_flags": [
     {
       "flag": "Single-run results for main system",
       "detail": "CoELA (GPT-4) results are from a single run 'due to cost constraints,' while baselines are averaged over 5 runs. This makes it impossible to assess the variance of the main results and raises concerns about result reliability."
     },
     {
       "flag": "Very small user study",
       "detail": "The user study has only 8 participants with no reported demographics, IRB approval, randomization procedure, or blinding. With 4 conditions, each subject did only 2-3 trials per condition, making per-condition sample sizes extremely small."
     },
     {
       "flag": "No variance or uncertainty quantification on main results",
       "detail": "Despite the stochastic nature of both LLM outputs and environment dynamics, no standard deviations, confidence intervals, or error bars are reported for the primary CoELA results."
     },
     {
       "flag": "Communication ineffective for AI-AI but highlighted as contribution",
       "detail": "The paper frames natural language communication as a key contribution, but the ablation shows it does not significantly help AI-AI cooperation. The benefit appears mainly for human-AI interaction, which is tested with only 8 subjects."
     },
     {
       "flag": "Cost transparency missing despite cost-constrained evaluation",
       "detail": "The paper acknowledges that cost constraints limited them to single runs, but never reports the actual API costs or computational budget, making it impossible to assess practicality."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Core LLM evaluated in this paper; relevant to understanding GPT-4 capabilities for agentic tasks."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Open-source LLM used as alternative to GPT-4 and fine-tuned into CoLLAMA for embodied cooperation."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park", "Joseph C. O'Brien", "Carrie J. Cai"],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "LLM-based agent society with memory augmentation; directly related to multi-agent LLM cooperation."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "LLM-driven embodied agent with skill library; related approach to using LLMs for embodied planning."
     },
     {
       "title": "Language Models as Zero-Shot Planners: Extracting Actionable Knowledge for Embodied Agents",
       "authors": ["Wenlong Huang", "Pieter Abbeel", "Deepak Pathak", "Igor Mordatch"],
       "year": 2022,
       "relevance": "Foundational work on using LLMs for embodied planning without fine-tuning."
     },
     {
       "title": "Code as Policies: Language Model Programs for Embodied Control",
       "authors": ["Jacky Liang", "Wenlong Huang", "Fei Xia"],
       "year": 2022,
       "arxiv_id": "2209.07753",
       "relevance": "Uses code generation from LLMs for embodied control, an alternative approach to the prompting-based planning in this paper."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2023,
       "arxiv_id": "2305.14325",
       "relevance": "Multi-LLM debate approach to improve reasoning; related multi-agent LLM cooperation paradigm."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Scale Language Model Society",
       "authors": ["Guohao Li", "Hasan Abed Al Kader Hammoud"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Multi-agent LLM communication framework for collaborative task solving."
     },
     {
       "title": "Sparks of Artificial General Intelligence: Early Experiments with GPT-4",
       "authors": ["Sébastien Bubeck", "Varun Chandrasekaran", "Ronen Eldan"],
       "year": 2023,
       "relevance": "Evaluates GPT-4 capabilities including reasoning and Theory of Mind, which this paper relies on for cooperative planning."
     },
     {
       "title": "A Survey on Large Language Model Based Autonomous Agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2023,
       "arxiv_id": "2308.11432",
       "relevance": "Comprehensive survey of LLM-based agents, covering planning, memory, and tool use capabilities."
     },
     {
       "title": "Cognitive Architectures for Language Agents",
       "authors": ["Theodore Sumers", "Shunyu Yao", "Karthik Narasimhan", "Thomas L. Griffiths"],
       "year": 2023,
       "arxiv_id": "2309.02427",
       "relevance": "Theoretical framework for language agent architectures, directly relevant to the cognitive-inspired modular design in this paper."
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Survey of LLM-based agents covering capabilities and limitations relevant to the survey scope."
     }
   ]
 }