ai-research-survey

scan.json (30032B)

---

 {
   "paper": {
     "title": "WALL-E: World Alignment by Rule Learning Improves World Model-based LLM Agents",
     "authors": [
       "Siyu Zhou",
       "Tianyi Zhou",
       "Yijun Yang",
       "Guodong Long",
       "Deheng Ye",
       "Jing Jiang",
       "Chengqi Zhang"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.07484",
     "doi": "10.48550/arXiv.2410.07484"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "WALL-E proposes a neurosymbolic world model combining LLMs with automatically learned rules for model-based agents using model-predictive control (MPC). On Minecraft TechTree tasks, WALL-E achieves 69% average success rate, surpassing baselines by 15-30% while using 8-20 fewer replanning rounds and 60-80% of the tokens. On ALFWorld, it reaches 95% success rate after only 6 iterations of rule learning. The rule learning process converges quickly (by iteration 4) and the learned rules are compact (6 rules for Minecraft, 11 for ALFWorld).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link at the top: 'Project: https://github.com/elated-sawyer/WALL-E'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: MineDojo TechTree tasks (Fan et al., 2022) and ALFWorld (Shridhar et al., 2020b), both standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the backend models (GPT-4o, GPT-3.5-Instruct) and benchmarks used, but provides no requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The method is described in detail, but there are no specific commands or scripts to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-4 report only point estimates for success rate, replanning rounds, and token costs with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims WALL-E 'outperforms' and 'surpasses' baselines based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: '15-30% in success rate,' '8-20 fewer replanning rounds,' and '60-80% of tokens' compared to baselines, with per-task breakdowns in Tables 1-2 showing both WALL-E and baseline values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 4 tasks per level were selected as test set, or why 30 training and 24 testing tasks are sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any spread measure is reported across experimental runs. ALFWorld mentions 'averaged success rate over several trials' but does not report spread."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are compared: DEPS, GITM, GPT-4V, Jarvis-1, Optimus-1, ReAct, Reflexion, AdaPlanner, RAFA, AutoGen, BUTLER, EMMA, and others across Tables 1 and 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: Optimus-1 (2024), EMMA (2024), RAFA (2023), AdaPlanner (2024), Reflexion (2024). These represent state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents a systematic ablation study examining five configurations: LLM alone, LLM+WM, LLM+rules (no WM), LLM+WM+rules (in WM), and LLM+rules+WM+rules (in both). This isolates the contribution of rules and world model."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: success rate, replanning rounds, and token cost (Tables 1 and 2). ALFWorld additionally uses per-task-type success rates."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs or plans. All evaluation is automated through environment success/failure signals."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'We select four tasks from each level to serve as the testing set and the remaining tasks to construct the training set.' ALFWorld uses 'a set of 134 predefined testing tasks.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task difficulty level (Wood through Redstone) in Table 1, by action type (craft, mine, gather, fight) in Figure 5, and by ALFWorld task type (Pick, Clean, Heat, Cool, Examine, Picktwo) in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While Figure 1 illustrates a failure-recovery example and Appendix H mentions stochastic dynamics as a limitation, there is no systematic error analysis of where the system fails on test tasks or why certain task categories remain difficult."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows that adding a world model without rules does not significantly improve performance (38% vs 37%). Section 4.4 explicitly notes: 'MPC using a world model without applying any rules cannot significantly improve WALL-E's performance.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 15-30% improvement (supported by Table 1: 69% vs 37-54%), 8-20 fewer replanning rounds (Table 1), 60-80% of tokens (Table 2), and 95% ALFWorld success after 6 iterations (Figure 4, Table 3) are all supported by the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about rule learning improving performance are supported by controlled ablation study in Table 4, which systematically adds/removes components (rules, world model) while holding other factors constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 claims WALL-E is 'a general and environment-agnostic method' but tests only on two game environments (Minecraft and ALFWorld). Section 1 mentions applicability to 'medical care, education, autonomous driving' without any evidence. The title 'World Alignment' implies broad generality."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, the paper does not consider whether GPT-4o's strong prior knowledge of Minecraft mechanics (a heavily documented game) could explain the performance, or whether results would hold for environments less represented in LLM training data."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures success rate, replanning rounds, and token cost, and claims these reflect planning ability and sample efficiency. The metrics directly correspond to the claims being made — no significant proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E.1 states 'We utilize GPT-4o as the backend' and E.2 states 'GPT-3.5-Instruct as our backbone model.' These are marketing names without snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B.1 (learning new rules), B.2 (refining rules), and B.3 (translating rules to code), with complete formatting and instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the λ parameter for rule pruning is 'very small to prioritize coverage maximum' and describes the MPC framework, but does not report LLM-specific hyperparameters (temperature, top-p, max tokens) for GPT-4o or GPT-3.5-Instruct calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MPC framework, rule learning pipeline (5 steps in Figure 3), agent-world model interaction loop (Figure 2), state/action spaces (Appendix C), and code-based rule execution are described in extensive detail."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendices E.1-E.2 describe the data pipeline: MineDojo's standard evaluation pipeline, task selection for train/test splits, ALFWorld's conversion of dialogue history to structured JSON format. State and action spaces are fully documented in Appendix C."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H is titled 'Limitation and Future Work' and discusses two specific limitations: the simplicity of learned rules (transition-level only) and inability to handle stochastic dynamics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H identifies study-specific threats: (1) rules are limited to transition-level assessment and cannot capture abstract planning rules; (2) stochastic environments where 'resource gathering at night in Minecraft often fails due to hostile creatures' are mishandled by the deterministic rule learning process."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no statement bounding results to deterministic game environments, or acknowledging that the two test environments both have discrete, well-defined action spaces. The generality claims in Section 4.2 go unbounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw trajectory data, detailed per-task results, or experimental logs are made available. Only aggregated results in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix E describe data collection: MineDojo TechTree tasks with specific task levels (Table 5), 30 training/24 testing tasks for Minecraft, 134 predefined testing tasks for ALFWorld, and the iterative rule learning process (3 tasks per iteration, 10 iterations)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (MineDojo, ALFWorld)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: environment interaction → trajectory collection → comparison with predictions (Eq. 4) → rule learning (Eqs. 5-7) → rule pruning (Eq. 8) → evaluation. Train/test split criteria are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Author affiliations include Tencent (a commercial entity) but no funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Technology Sydney, University of Maryland College Park, and Tencent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot be assessed as no funding is disclosed. Three authors are from Tencent, which has commercial interest in LLM agent capabilities, but no funding relationship is described."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o and GPT-3.5-Instruct but does not state the training data cutoff date for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data includes descriptions of MineDojo tasks, Minecraft crafting recipes, or ALFWorld task solutions, despite the LLM's prior knowledge being central to the method."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MineDojo (2022) and ALFWorld (2020) were published before GPT-4o's training cutoff. The LLM's knowledge of Minecraft mechanics is explicitly leveraged (LLM_request functions), but the paper does not discuss whether this constitutes benchmark contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports inference token counts and API costs in USD for all methods across all task levels. For example, WALL-E averages 60,348 tokens ($0.41) per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-task token costs are reported, the total computational budget for the full rule learning process (10 iterations × 3 tasks) is not stated. No GPU hours, total API spend, or training time is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section E.1 mentions 'randomized seeds for both the environment and its starting position' but does not report results across multiple seeds or analyze seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld mentions 'averaged success rate over several trials' without specifying the exact number. Minecraft does not state how many runs produced the reported results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of hyperparameter search budget. The λ parameter is described as 'very small' and the MPC horizon H is not specified with search details."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents all five configurations tested in the ablation study, showing the full landscape of results. The selected configuration (LLM agent + LLM+rules world model) is justified by the highest average success rate."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper re-implements DEPS and uses its own experimental setup for comparisons. Some baseline results are marked '*-reported in previous work' (different conditions). No acknowledgment of self-implementation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 directly compares token usage and API costs across all methods alongside success rates in Table 1, allowing performance-per-cost assessment."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MineDojo TechTree tasks and ALFWorld actually measure the capabilities the paper claims to evaluate (planning, world modeling, generalization to complex environments)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WALL-E uses MPC with a world model while baselines use different scaffolding approaches (e.g., GITM uses buffered trajectories, ReAct uses reasoning+acting). Cross-method comparisons conflate the scaffold with the model. The ablation study partially addresses this within WALL-E's framework but not across baselines."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. MineDojo (2022) and ALFWorld (2020) were published well before GPT-4o's training data collection. The LLM explicitly uses knowledge of Minecraft mechanics via LLM_request calls."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The LLM_request function calls deliberately use GPT-4's world knowledge to make judgments (e.g., whether a tool is appropriate for mining), which is a design choice but could leak evaluation-relevant information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and testing tasks share structural similarities (e.g., same crafting tree logic, similar action sequences across difficulty levels)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "WALL-E surpasses existing LLM/VLM agents by 15-30% in success rate on Minecraft tasks.",
       "evidence": "Table 1 shows WALL-E achieves 69% average success rate vs. GITM (54%), Jarvis-1 (42%), DEPS (37%). The margin ranges from 15% (vs GITM) to 32% (vs DEPS).",
       "supported": "moderate"
     },
     {
       "claim": "WALL-E requires 8-20 fewer replanning rounds than baselines in Minecraft.",
       "evidence": "Table 1: WALL-E averages 15.77 replanning rounds vs. GITM (25.49) and DEPS (35.36), a reduction of ~10 and ~20 rounds respectively.",
       "supported": "moderate"
     },
     {
       "claim": "WALL-E uses only 60-80% of the tokens compared to baselines in Minecraft.",
       "evidence": "Table 2: WALL-E averages 60,349 tokens vs. GITM (74,639) and DEPS (93,561), which is 81% and 64% respectively.",
       "supported": "moderate"
     },
     {
       "claim": "WALL-E achieves 95% success rate on ALFWorld after only 6 iterations.",
       "evidence": "Figure 4 shows the learning curve reaching 95% at iteration 6. Table 3 confirms 95% overall success rate matching RAFA.",
       "supported": "moderate"
     },
     {
       "claim": "Rule learning converges by the 4th iteration with a compact rule set.",
       "evidence": "Figure 5 shows cover rates plateauing by iteration 4. Figure 6 shows success rate hitting upper bound after 4 iterations. Final rule sets contain 6 rules for Minecraft and 11 for ALFWorld (Appendix D).",
       "supported": "moderate"
     },
     {
       "claim": "WALL-E is the only method that performs better than human players in the MineDojo benchmark.",
       "evidence": "Table 1: WALL-E achieves 69% average success rate vs. 59% for human performance. However, humans outperform WALL-E on Iron (86% vs 63%) and perform comparably or better on Wood (100% vs 98%) and Stone (100% vs 91%).",
       "supported": "weak"
     },
     {
       "claim": "Applying learned rules within the world model contributes ~30% improvement in success rate.",
       "evidence": "Table 4: LLM+WM without rules achieves 38%, while LLM+(LLM+rules)WM achieves 69%, a 31% improvement.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results in Tables 1-4 are point estimates without confidence intervals, standard deviations, or any measure of variability. The number of runs is not specified for Minecraft. This makes it impossible to assess whether observed differences are statistically meaningful."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'outperformance' and 'superiority' are made by comparing raw numbers without any significance testing. Given the stochasticity of both environments (randomized seeds, spawns, mob encounters), the differences could partly be due to chance."
     },
     {
       "flag": "Mixed baseline conditions",
       "detail": "Some baseline results in Tables 1 and 3 are marked '*-reported in previous work' while others appear to be re-run. Results from different papers may use different seeds, hardware, or model versions, making direct comparison unreliable."
     },
     {
       "flag": "Overclaimed generality",
       "detail": "The paper claims WALL-E is 'general and environment-agnostic' based on two game-like environments (Minecraft and ALFWorld) that both have discrete, well-defined action spaces. Section 1 mentions applicability to 'medical care, education, autonomous driving' without any supporting evidence."
     },
     {
       "flag": "Contamination concern deliberately unexplored",
       "detail": "The method explicitly leverages GPT-4o's prior knowledge of Minecraft via LLM_request calls (e.g., asking whether a tool is appropriate for mining). This is a design feature, but the paper never discusses whether the LLM's extensive knowledge of these well-documented games inflates the results and would not transfer to novel environments."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Foundational LLM agent framework combining reasoning and acting that WALL-E is compared against as a baseline."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "relevance": "Key LLM agent baseline using verbal self-reflection for learning from mistakes, directly compared in both benchmarks."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang", "Yiran Wu", "Shaokun Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent conversation framework compared as a baseline on ALFWorld, relevant to agentic LLM workflows."
     },
     {
       "title": "Ghost in the Minecraft: Generally Capable Agents for Open-World Environments via Large Language Models with Text-based Knowledge and Memory",
       "authors": ["Xizhou Zhu", "Yuntao Chen", "Hao Tian"],
       "year": 2023,
       "arxiv_id": "2305.17144",
       "relevance": "Primary LLM agent baseline for Minecraft using buffered trajectories, directly compared against WALL-E's rule learning approach."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique for LLM reasoning that underlies much of the agent planning work in this space."
     },
     {
       "title": "EmbodiedGPT: Vision-Language Pre-Training via Embodied Chain of Thought",
       "authors": ["Yao Mu", "Qinglong Zhang", "Mengkang Hu"],
       "year": 2023,
       "arxiv_id": "2305.15021",
       "relevance": "Represents the approach of fine-tuning LLMs for embodied agent tasks, contrasted with WALL-E's gradient-free rule learning approach."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Foundation model whose capabilities are leveraged as the backbone for WALL-E's world model and agent."
     },
     {
       "title": "Rt-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control",
       "authors": ["Brianna Zitkovich", "Tianhe Yu", "Sichun Xu"],
       "year": 2023,
       "relevance": "VLM-based robotic control that represents an alternative paradigm to rule-based world modeling for embodied agents."
     },
     {
       "title": "Reason for Future, Act for Now: A Principled Framework for Autonomous LLM Agents with Provable Sample Efficiency",
       "authors": ["Zhihan Liu", "Hao Hu", "Shenao Zhang"],
       "year": 2023,
       "arxiv_id": "2309.17382",
       "relevance": "RAFA framework that achieves competitive 95% success on ALFWorld, directly compared as a planning-based agent baseline."
     },
     {
       "title": "Optimus-1: Hybrid Multimodal Memory Empowered Agents Excel in Long-Horizon Tasks",
       "authors": ["Zaijing Li", "Yuquan Xie", "Rui Shao"],
       "year": 2024,
       "arxiv_id": "2408.03615",
       "relevance": "State-of-the-art VLM agent for Minecraft that WALL-E outperforms, demonstrating the advantage of rule-based world alignment."
     },
     {
       "title": "Describe, Explain, Plan and Select: Interactive Planning with Large Language Models Enables Open-World Multi-Task Agents",
       "authors": ["Zihao Wang", "Shaofei Cai", "Guanzhou Chen"],
       "year": 2023,
       "relevance": "DEPS method for LLM-based planning in open-world environments, used as a primary baseline in Minecraft experiments."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2024,
       "relevance": "Key alignment technique that WALL-E's 'world alignment' concept parallels, extending alignment from human preferences to environment dynamics."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The rule learning approach for aligning LLM world models is applicable to other agent scenarios, and code is released on GitHub."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Shows that a few learned rules can bridge the gap between LLM prior knowledge and environment dynamics, somewhat surprising but builds on intuitive foundations."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; focuses on improving agent task completion in game environments."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward benchmark improvement paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with Minecraft and ALFWorld implementations; users could try the approach in these environments."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from UTS, UMD, and Tencent — Tencent is recognizable but not a top-tier AI research brand for this audience."
     }
   }
 }