ai-research-survey

scan.json (25772B)

---

 {
   "paper": {
     "title": "PLAN-AND-ACT: Improving Planning of Agents for Long-Horizon Tasks",
     "authors": ["Lutfi Eren Erdogan", "Nicholas Lee", "Sehoon Kim", "Suhong Moon", "Hiroki Furuta", "Gopala Anumanchipalli", "Kurt Keutzer", "Amir Gholami"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.09572"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PLAN-AND-ACT separates high-level planning from low-level execution for LLM-based agents, achieving 57.58% on WebArena-Lite and 81.36% text-only on WebVoyager. A synthetic data generation pipeline produces 15,000 training examples in under an hour, with dynamic replanning providing the largest single improvement (+10.31%). Even an untrained executor benefits substantially from a well-trained planner (+34.39% over baseline).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic training data (15,000 plans, trajectories) is not released. The benchmarks used (WebArena-Lite, WebVoyager) are public, but the paper's generated training data is not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 5/Figure 5 lists training hyperparameters and mentions '8×A100' and 'torchtune' framework, but no requirements.txt, Dockerfile, or detailed dependency listing is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are reported as point estimates (e.g., 57.58%, 81.36%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SOTA and outperformance over baselines based solely on comparing numbers (e.g., 57.58% vs 49.1%) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context throughout, e.g., '10.31% over the static PLANNER', 'from 9.85% to 29.63%', 'from 45% to 57%' style comparisons in Section 5.2-5.3 and Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "WebArena-Lite has 165 test cases and WebVoyager's size is not stated. No justification for why these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or multi-run statistics are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes multiple baselines: zero-shot LLaMA-3.3-70B, finetuned LLaMA, WebRL-3.1-70B, GPT-4-Turbo, GPT-4o, AWM, WebPilot. Tables 3-4 include additional baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods like WebRL (2024), AgentOccam (2024), WebPilot (2024), and Agent-E (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 is essentially an extensive ablation showing incremental contributions: base planner, finetuning, synthetic trajectories, plan expansion, targeted augmentation, dynamic replanning, and CoT. Each row adds one component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: binary success rate. No other metrics (partial completion, efficiency, step count as metric) are used for evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of plan quality or agent outputs is conducted. The WebArena-Lite benchmark is described as 'human-verified' but that refers to the benchmark construction, not evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "WebArena-Lite provides separate training data (1,113 examples) and 165 test cases. The paper trains on the training split and evaluates on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 and the appendix table (A.11) provide per-website breakdowns across GitLab, Reddit, Shopping Admin, Shopping, Map, and Multiple Websites."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses failure patterns (dynamic content analysis, query refinement failures). Appendix A.2 provides detailed examples of replanning after failures. Section 4.3 discusses targeted augmentation based on failure analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows that a base (unfinetuned) planner actually hurts finetuned executors (row 7 vs row 1). The paper honestly reports that naively finetuning the planner did not improve performance for the finetuned executor (row 8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 57.58% on WebArena-Lite and 81.36% text-only on WebVoyager, both supported by Tables 1 and 4 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation ('adding dynamic replanning improves by 10.31%'). The ablation design in Table 1 uses controlled single-variable manipulation, adding one component at a time while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Long-Horizon Tasks' generally but evaluation is limited to web navigation (WebArena-Lite, WebArena, WebVoyager). The conclusion mentions 'broader applications in various digital environments' without bounding. The paper acknowledges this somewhat in the Limitations section but the title and framing overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for the improvements. For example, the synthetic data pipeline uses GPT-4o as a teacher — improvements could partly reflect knowledge distillation from a stronger model rather than the planning architecture itself. This is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses binary task success rate and does not overframe it — it claims success on specific benchmarks rather than broader 'intelligence' or 'planning capability' in general. The measurements match the claims' granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: LLaMA-3.3-70B-Instruct, LLaMA-3.1-8B-Instruct, QWQ-32B, GPT-4o (as teacher), WebRL-Llama-3.1-70B, ORM-Llama-3.1-8B, DeepSeek-R1-Distill-Llama-70B. These include size and variant specifications."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendices A.3-A.10, covering the Planner system prompt, Executor prompt, Plan Data Annotator, Synthetic Plan Generator, Replanner, and failure classification prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 (Table 5) reports training hyperparameters (learning rate 2e-5, AdamW, cosine scheduler, batch size 32, 1 epoch) and inference hyperparameters (temperature 0, max tokens 4196, max sequence length 32000)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-component architecture (Planner + Executor) is described in detail in Section 3, including dynamic replanning (Section 3.3), information flow between components, and garbage collection by the Executor (Section 3.2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic data generation pipeline is documented in detail in Section 4, including trajectory generation with ORM filtering (Section 4.1), grounded plan generation (Section 4.2), and plan expansion with counts (10,000 + 5,000 synthetic plans). Section 4.1 describes the filtering process using an ORM."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the Conclusion discussing dependency on a baseline model for trajectory generation and the inefficiency of replanning after every action."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific concerns: dependency on a base model that can successfully complete web tasks (limiting applicability to new domains), and computational inefficiency of replanning after every action step."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While the Limitations section mentions some constraints, there is no explicit bounding of scope — e.g., no statement that results are limited to web navigation and may not transfer to other long-horizon task domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw trajectory data, synthetic training data, or per-example results are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the synthetic data collection in detail: Alpaca-style query generation from seed data, teacher LLM trajectory collection, ORM filtering, grounded plan annotation, and plan expansion. Specific counts are provided (923 trajectories, 1,113 WebArena-Lite examples, 10,000 + 5,000 synthetic plans)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Sections 4.1-4.3 document the full pipeline from seed queries through trajectory generation, ORM filtering, plan annotation, and plan expansion, with counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists support from Apple, Nvidia, Microsoft Accelerating Foundation Model Research, Google Cloud/TRC, Intel, BDD, BAIR, Furiosa, KFAS, and JSPS KAKENHI."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UC Berkeley, University of Tokyo, ICSI. No authors are affiliated with companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Intel, Apple, Nvidia, Microsoft, Google) are hardware/platform providers, not producers of the specific models or benchmarks being evaluated. The paper evaluates open-source LLaMA models on academic benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for LLaMA-3.3-70B-Instruct or GPT-4o used as the teacher model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WebArena-Lite test cases or their solutions appeared in LLaMA or GPT-4o training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WebArena was published in 2023. The models used (LLaMA-3.3, GPT-4o) were trained after this. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example API costs reported. The system calls the planner after every executor action, which could be expensive, but costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training hardware is mentioned (8×A100) but total GPU hours, training time, and API costs for generating 15,000 synthetic examples with GPT-4o are not quantified. The paper mentions 'under an hour' for plan expansion but not the full compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper lists chosen hyperparameters (Figure 5) but does not describe how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using a 'held-out validation set' for failure analysis (Section 4.3) but does not describe how the best model configuration was selected or whether final results were on a separate test set from the validation set used for targeted augmentation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their system against baselines without acknowledging potential bias from implementing/running those baselines themselves. Some baseline numbers are cited from papers (marked with *) but others may be their own runs."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison at matched compute budgets. PLAN-AND-ACT uses GPT-4o for synthetic data generation and requires finetuning a 70B model, while some baselines are zero-shot. This compute difference is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether WebArena-Lite's 165 tasks adequately represent 'long-horizon planning' capability. No discussion of construct validity or limitations of the benchmark as a measure of planning ability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different baselines use different scaffolds (WebPilot has 6 agents, AgentOccam uses tree planning, WebRL uses RL), yet comparisons attribute differences to the method rather than the scaffold. The paper does not control for scaffolding differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WebArena was published in 2023 and models trained after could have seen solutions. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic training data is generated from WebArena-Lite training examples. No discussion of whether the generated examples overlap with or are structurally similar to test cases."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "PLAN-AND-ACT achieves 57.58% success rate on WebArena-Lite, a new state-of-the-art.",
       "evidence": "Table 1, last row. Previous SOTA was WebRL-3.1-70B at 49.1%.",
       "supported": "moderate"
     },
     {
       "claim": "PLAN-AND-ACT achieves 81.36% text-only SOTA on WebVoyager with QWQ-32B.",
       "evidence": "Table 4. Previous text-only best was WebVoyager with GPT-4-Turbo at 44.3% (text) and Agent-E at 73.1%.",
       "supported": "moderate"
     },
     {
       "claim": "Dynamic replanning improves performance by 10.31% over static planning.",
       "evidence": "Table 1, row 12 vs row 11: 53.94% vs 43.63% with full executor.",
       "supported": "moderate"
     },
     {
       "claim": "A well-trained planner improves even an untrained executor by 34.39% (from 9.85% to 44.24%).",
       "evidence": "Table 1, comparing row 1 col 1 (9.85%) to row 12 col 1 (44.24%).",
       "supported": "moderate"
     },
     {
       "claim": "15,000 synthetic training examples can be generated in under an hour using GPT-4o.",
       "evidence": "Section 4.3 and Section 6 state this claim. No independent verification provided.",
       "supported": "weak"
     },
     {
       "claim": "CoT reasoning adds 4.36% improvement to the framework.",
       "evidence": "Table 1, row 13 vs row 12: 57.58% vs 53.94%. Table 2 shows 8B model with CoT matches 70B without.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or multi-run reporting",
       "detail": "All results are single point estimates on benchmarks with 165 (WebArena-Lite) test cases. With only 165 binary outcomes, success rate has substantial sampling variance (~3-4% for a 95% CI), making the claimed improvements (e.g., 4.36% for CoT) potentially within noise."
     },
     {
       "flag": "Teacher model knowledge distillation conflated with architecture",
       "detail": "GPT-4o is used as the teacher for synthetic data generation. Performance improvements may partly reflect distillation of GPT-4o's knowledge rather than the planning architecture itself. This is not disentangled."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite being a systems paper with a synthetic data pipeline, neither the code, the 15,000 synthetic training examples, nor the trained model weights are released."
     },
     {
       "flag": "Scaffold confound in cross-method comparisons",
       "detail": "Baselines use very different architectures (WebRL uses RL, WebPilot uses 6 agents, AgentOccam uses tree search), making it unclear whether improvements come from the planning architecture or other factors."
     }
   ],
   "cited_papers": [
     {
       "title": "WebRL: Training LLM Web Agents via Self-Evolving Online Curriculum Reinforcement Learning",
       "authors": ["Qi, Z.", "Liu, X."],
       "year": 2024,
       "arxiv_id": "2411.02337",
       "relevance": "Main baseline and SOTA on WebArena-Lite; RL-based approach to training web agents."
     },
     {
       "title": "AgentOccam: A Simple Yet Strong Baseline for LLM-Based Web Agents",
       "authors": ["Yang, K.", "Liu, Y."],
       "year": 2024,
       "arxiv_id": "2410.13825",
       "relevance": "Hierarchical planning baseline using GPT-4-Turbo for web navigation."
     },
     {
       "title": "WebPilot: A Versatile and Autonomous Multi-Agent System for Web Task Execution with Strategic Exploration",
       "authors": ["Zhang, Y.", "Ma, Z."],
       "year": 2024,
       "arxiv_id": "2408.15978",
       "relevance": "Multi-agent web navigation system with 6 agents; comparative baseline."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Yao, S.", "Zhao, J."],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent framework combining reasoning and action; baseline prompting approach."
     },
     {
       "title": "An LLM Compiler for Parallel Function Calling",
       "authors": ["Kim, S.", "Moon, S."],
       "year": 2023,
       "arxiv_id": "2312.04511",
       "relevance": "LLMCompiler architecture that inspired the planner-executor separation in PLAN-AND-ACT."
     },
     {
       "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions",
       "authors": ["Wang, Y.", "Kordi, Y."],
       "year": 2022,
       "arxiv_id": "2212.10560",
       "relevance": "Pioneering synthetic data generation method used as basis for the data pipeline."
     },
     {
       "title": "DigiRL: Training In-the-Wild Device-Control Agents with Autonomous Reinforcement Learning",
       "authors": ["Bai, H.", "Zhou, Y."],
       "year": 2024,
       "arxiv_id": "2406.11896",
       "relevance": "RL-based approach for training device-control agents with autonomous data collection."
     },
     {
       "title": "NNetscape Navigator: Complex Demonstrations for Web Agents without a Demonstrator",
       "authors": ["Murty, S.", "Bahdanau, D.", "Manning, C. D."],
       "year": 2024,
       "arxiv_id": "2410.02907",
       "relevance": "Interaction-first trajectory collection with retroactive labeling for web agent training data."
     },
     {
       "title": "AdaPlanner: Adaptive Planning from Feedback with Language Models",
       "authors": ["Sun, H.", "Zhuang, Y."],
       "year": 2023,
       "relevance": "Adaptive planning baseline with in-plan and out-of-plan refinement for LLM agents."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Guo, D.", "Yang, D."],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Source of the distilled reasoning model used for CoT trace generation in the pipeline."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Zhou, S.", "Xu, F. F."],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Primary evaluation benchmark environment for web agent research."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Wei, J.", "Wang, X."],
       "year": 2022,
       "relevance": "Foundational work on CoT reasoning applied in the PLAN-AND-ACT framework."
     }
   ]
 }