ai-research-survey

scan.json (24500B)

---

 {
   "paper": {
     "title": "AdaPlanner: Adaptive Planning from Feedback with Language Models",
     "authors": [
       "Haotian Sun",
       "Yuchen Zhuang",
       "Lingkai Kong",
       "Bo Dai",
       "Chao Zhang"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2305.16653"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The implementation of AdaPlanner is available on https://github.com/haotiansun14/AdaPlanner', providing a working GitHub URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available standard benchmarks: ALFWorld and MiniWoB++, both of which are publicly released environments. The paper does not collect proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include a requirements.txt, Dockerfile, or detailed environment setup section listing library versions. The appendix describes experimental setup at a high level but does not provide enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub link is provided, the paper itself contains no step-by-step reproduction instructions. The appendix describes prompts and experimental configurations but does not provide commands or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report success rates as point estimates only (e.g., '91.79%'). No confidence intervals, error bars, or uncertainty measures are reported for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AdaPlanner 'outperforms state-of-the-art baselines by 3.73% and 4.11%' but provides no statistical significance tests (no p-values, t-tests, or equivalent) to support comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: '3.73% improvement' from 88.06% to 91.79% (ALFWorld) and '4.11% improvement' from ~88.76% to 92.87% (MiniWoB++). The schema description explicitly states that 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' Tables 2 and 3 provide the raw baseline scores alongside AdaPlanner scores, giving readers full effect magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 134 ALFWorld tasks and 53 MiniWoB++ tasks are described but there is no justification for why these specific numbers were chosen, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2 and 3 are single point estimates. The paper does not report results averaged over multiple runs with any spread measures (standard deviation, IQR, etc.), making result stability impossible to assess."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines including BUTLER, ReAct, Reflexion (ALFWorld) and CC-Net, WGE, WebN-T5-3B, RCI (MiniWoB++), covering both training-based and prompting-based methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporaneous work (Reflexion, RCI from 2023; ReAct from 2023), and the comparison includes work published within the same year as the paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 includes ablation studies testing: (c) AdaPlanner with and without the code interface (CI), and (d) AdaPlanner with and without skill discovery (SD), demonstrating the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only success rate (%) as the evaluation metric across all experiments. Section 8.1 states 'we use success rate (%) to evaluate the performance of tested methods' and no other metrics are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation is entirely automated through task success/failure in simulated environments (ALFWorld, MiniWoB++). Human evaluation is not relevant to these automated benchmark claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on standard benchmark test sets (134 ALFWorld tasks and 53 MiniWoB++ tasks) following prior work. Demonstrations used for prompting are separate from evaluation tasks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-category results for all six ALFWorld task types (Pick, Clean, Heat, Cool, Examine, Pick two). Table 3 separates MiniWoB++ into feedback and no-feedback subsets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases or provide error analysis. The conclusion section mentions one limitation (needing few-shot demonstrations) but no analysis of where the approach fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (Figure 4c, 4d) show performance drops when components are removed. Table 2 also shows that AdaPlanner with gpt-3.5-turbo underperforms relative to text-davinci-002 in some tasks, which the paper discusses in Section 4."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AdaPlanner outperforms baselines by 3.73% and 4.11% while using 2x and 600x fewer samples. These claims are supported by Tables 2, 3, and Figure 3 in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'without the code interface, AdaPlanner's performance substantially drops'), which are supported by controlled single-variable manipulation ablations shown in Figure 4c and 4d."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on two specific text-based environments (ALFWorld and MiniWoB++) but makes broader claims about 'sequential decision-making tasks' in general. The conclusion does not bound claims to these specific environments."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For instance, performance differences could be attributed to prompt engineering rather than the closed-loop mechanism, but this is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model identifiers: 'gpt-3.5-turbo', 'text-davinci-002', and 'text-davinci-003' are explicitly named in Tables 2 and 3 and Section 8.2."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 8.3 provides the full prompt text for all stages (basic_info, initial_planning, code_check, refinement, start_from determination) with actual example solutions for all six ALFWorld task types and MiniWoB++ tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, max tokens, or other sampling hyperparameters for the LLM API calls. These settings are not mentioned anywhere in the paper or appendix."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the agentic scaffolding in detail: the planner/refiner architecture, in-plan and out-of-plan refinement mechanisms, the ask_LLM() action, assertion-based error detection, and the refine-then-resume mechanism are all described in Sections 3.1-3.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how the environments are set up, how tasks are selected (9 MiniWoB++ tasks with feedback, 53 tasks from RCI evaluation), and how demonstrations are allocated (Table 4). The skill discovery pipeline is also described in Section 3.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusion and Limitations' includes a dedicated limitations discussion, noting that AdaPlanner still requires few-shot expert demonstrations for complex tasks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions only one limitation (need for few-shot demonstrations) and does not discuss threats to validity such as benchmark contamination, sensitivity to prompt phrasing, or the limited scope of the two test environments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no statement bounding results to text-based environments or these specific benchmarks, and the paper implies broader applicability without bounding the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repository is provided, the paper does not describe providing raw experimental logs or trajectories for independent verification of the reported success rates."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experimental setup (Section 8.1) describes the environments used, task types, and how demonstrations were collected (38 human-written + 21 discovered by skill discovery for MiniWoB++, 6 expert samples for ALFWorld)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The evaluation uses automated environments (ALFWorld, MiniWoB++) with no human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendix 8.1-8.2 describe the evaluation pipeline: how tasks are run, how success is determined, and how skill discovery produces additional examples. The pipeline from task assignment to success measurement is sufficiently documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are affiliated with Georgia Institute of Technology, as clearly stated on the first page. No authors appear to be affiliated with OpenAI or other providers of the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. This item is inapplicable in the absence of any funding disclosure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or financial disclosure in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3 (text-davinci-002), GPT-3.5 (text-davinci-003, gpt-3.5-turbo) but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the ALFWorld or MiniWoB++ benchmarks or their task descriptions may have been present in the GPT-3/3.5 training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld (2021) and MiniWoB++ (2018) were both published before GPT-3/3.5 training data cutoffs. The paper does not address the possibility that task descriptions or solutions may have appeared in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs, token counts, or wall-clock time per episode. The paper mentions reducing LLM calls as a benefit but does not quantify actual inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend is reported. The paper does not quantify the computational resources required to run the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "AdaPlanner outperforms state-of-the-art baselines by 3.73% in ALFWorld and 4.11% in MiniWoB++ while using 2x and 600x fewer samples, respectively.",
       "evidence": "Table 2 shows AdaPlanner (GPT-3) achieves 91.79% vs. Reflexion's 88.06% in ALFWorld. Table 3 shows AdaPlanner achieves 92.87% vs. RCI's 91.00% in MiniWoB++. Figure 3 shows sample efficiency comparison.",
       "supported": "moderate"
     },
     {
       "claim": "The code interface (CI) is essential: removing it causes performance to drop from 81% to 46% in ALFWorld and from 93% to 66% in MiniWoB++.",
       "evidence": "Figure 4c shows ablation with and without code interface for both environments. The comparison is controlled—only the code interface is removed.",
       "supported": "strong"
     },
     {
       "claim": "Skill discovery significantly enhances performance: in ALFWorld, success rate nearly doubles with skill discovery (19% to 38%), and in MiniWoB++ increases by about 15% (45% to 60%).",
       "evidence": "Figure 4d shows ablation with and without skill discovery under zero-shot/one-shot settings. Section 4 describes these results.",
       "supported": "strong"
     },
     {
       "claim": "AdaPlanner's closed-loop refinement consistently improves performance with each additional refinement iteration, even with as few as 2 demonstrations total.",
       "evidence": "Figure 4a shows success rate increasing with number of closed-loop corrections across different sample counts (2, 4, 6 samples).",
       "supported": "moderate"
     },
     {
       "claim": "gpt-3.5-turbo underperforms text-davinci-002 on decision-making tasks due to greater hallucination, but AdaPlanner's code prompts mitigate this.",
       "evidence": "Table 2 shows AdaPlanner (GPT-3) at 91.79% vs. AdaPlanner (GPT-3.5) at 80.60%. Section 4 discusses this observation. However, 'gpt-3.5-turbo might be a smaller-scale model' is presented as hypothesis, not established fact.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AdaPlanner is a closed-loop LLM agent framework that uses code-based prompting and adaptive plan refinement (in-plan and out-of-plan) to outperform prior baselines on ALFWorld (91.79% success rate) and MiniWoB++ (92.87% success rate) while requiring fewer demonstrations. Ablation studies show that both the code interface and skill discovery module contribute substantially to performance. The approach operates entirely through prompting without task-specific training, making it broadly applicable.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are reported as single point estimates with no standard deviation, confidence intervals, or indication of how many runs were conducted. Result stability across random seeds or multiple runs is entirely unknown."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparative claims (e.g., outperforming baselines by 3.73%) are made without any statistical tests. The margins are small and statistical significance is never assessed."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "ALFWorld (2021) and MiniWoB++ (2018) predate the training data cutoffs of the GPT-3/3.5 models used. The paper does not discuss whether task descriptions, solutions, or environment behavior could have appeared in training data."
     },
     {
       "flag": "Only success rate reported",
       "detail": "The paper uses a single metric (success rate) across all experiments, which obscures variation in efficiency, number of steps required, and partial task completion."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, max tokens, and other LLM sampling hyperparameters are never reported, making exact reproduction impossible and affecting result interpretation."
     },
     {
       "flag": "No failure case analysis",
       "detail": "The paper does not analyze failure cases or discuss where AdaPlanner breaks down. The limitations section mentions only the need for few-shot demonstrations without discussing systematic failure modes."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik Narasimhan",
         "Yuan Cao"
       ],
       "year": 2023,
       "relevance": "Foundational paper on LLM agents combining reasoning traces with action-taking, directly compared as a baseline."
     },
     {
       "title": "Reflexion: an Autonomous Agent with Dynamic Memory and Self-Reflection",
       "authors": [
         "Noah Shinn",
         "Beck Labash",
         "Ashwin Gopinath"
       ],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Key baseline for LLM agents with self-reflection and memory, compared directly on ALFWorld."
     },
     {
       "title": "Language Models Can Solve Computer Tasks",
       "authors": [
         "Geunwoo Kim",
         "Pierre Baldi",
         "Stephen McAleer"
       ],
       "year": 2023,
       "arxiv_id": "2303.17491",
       "relevance": "RCI method: baseline implicit closed-loop LLM agent for computer tasks, directly compared on MiniWoB++."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": [
         "Jason Wei",
         "Xuezhi Wang",
         "Dale Schuurmans",
         "Maarten Bosma",
         "Brian Ichter",
         "Fei Xia",
         "Ed Chi",
         "Quoc Le",
         "Denny Zhou"
       ],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational open-loop LLM planning method used as a baseline reference."
     },
     {
       "title": "ALFWorld: Aligning Text and Embodied Environments for Interactive Learning",
       "authors": [
         "Mohit Shridhar",
         "Xingdi Yuan",
         "Marc-Alexandre Cote",
         "Yonatan Bisk",
         "Adam Trischler",
         "Matthew Hausknecht"
       ],
       "year": 2021,
       "relevance": "Primary benchmark environment used for evaluation; essential reference for the evaluation design."
     },
     {
       "title": "PAL: Program-aided Language Models",
       "authors": [
         "Luyu Gao",
         "Aman Madaan",
         "Shuyan Zhou",
         "Uri Alon",
         "Pengfei Liu",
         "Yiming Yang",
         "Jamie Callan",
         "Graham Neubig"
       ],
       "year": 2022,
       "arxiv_id": "2211.10435",
       "relevance": "Related approach using code-based prompting for LLM reasoning, motivates AdaPlanner's code interface design."
     },
     {
       "title": "Code as Policies: Language Model Programs for Embodied Control",
       "authors": [
         "Jacky Liang",
         "Wenlong Huang",
         "Fei Xia",
         "Peng Xu",
         "Karol Hausman",
         "Brian Ichter",
         "Pete Florence",
         "Andy Zeng"
       ],
       "year": 2022,
       "arxiv_id": "2209.07753",
       "relevance": "Related code-based prompting method for robot control, relevant to AdaPlanner's code interface approach."
     },
     {
       "title": "Describe, Explain, Plan and Select: Interactive Planning with Large Language Models Enables Open-World Multi-Task Agents",
       "authors": [
         "Zihao Wang",
         "Shaofei Cai",
         "Anji Liu",
         "Xiaojian Ma",
         "Yitao Liang"
       ],
       "year": 2023,
       "arxiv_id": "2302.01560",
       "relevance": "DEPS: only prior method that modifies entire plans based on feedback, directly motivates AdaPlanner's design."
     },
     {
       "title": "Inner Monologue: Embodied Reasoning through Planning with Language Models",
       "authors": [
         "Wenlong Huang",
         "Fei Xia",
         "Ted Xiao",
         "Harris Chan",
         "Jacky Liang",
         "Pete Florence",
         "Andy Zeng",
         "Jonathan Tompson",
         "Igor Mordatch",
         "Yevgen Chebotar",
         "Pierre Sermanet",
         "Noah Brown",
         "Tsung-Yen Yang",
         "Sichun Luu",
         "Sergey Levine",
         "Karol Hausman",
         "Brian Ichter"
       ],
       "year": 2022,
       "arxiv_id": "2207.05608",
       "relevance": "Baseline implicit closed-loop LLM agent using environmental feedback for single-step action selection."
     },
     {
       "title": "Language Models as Zero-Shot Planners: Extracting Actionable Knowledge for Embodied Agents",
       "authors": [
         "Wenlong Huang",
         "Pieter Abbeel",
         "Deepak Pathak",
         "Igor Mordatch"
       ],
       "year": 2022,
       "arxiv_id": "2201.07207",
       "relevance": "Open-loop planning baseline demonstrating limitations of LLMs without feedback adaptation."
     }
   ]
 }