ai-research-survey

scan.json (27176B)

---

 {
   "paper": {
     "title": "Chasing Progress, Not Perfection: Revisiting Strategies for End-to-End LLM Plan Generation",
     "authors": ["Sukai Huang", "Nir Lipovetzky", "Trevor Cohn"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2412.10675"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous code repository is provided: https://anonymous.4open.science/r/official-misconcept-lm-plan-gen-D34B (mentioned in Section 4.1 and Appendix). While anonymous for review, it is a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The dataset will be released via Huggingface Hub' (Section 3.1 footnote and Appendix C), indicating a promise of future release. This counts as NO per evaluation criteria."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides detailed implementation details including hardware (4 Nvidia A100 GPUs), the specific model (Qwen2-7B-Instruct), deepspeed configuration (zero 3), bf16 precision, and Appendix B.1 lists a comprehensive hyperparameter table with learning rates, batch sizes, LoRA configuration, etc."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While hyperparameters are listed in a table and code is provided via an anonymous repository, there are no explicit step-by-step reproduction instructions (e.g., a README with commands). The paper describes the methodology but does not provide a clear recipe to reproduce from scratch."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '98.5% validity rate') across all tables (Tables 1-5) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 caption states 'Improvements of statistical significance are highlighted in green, while significant declines are highlighted in red,' indicating statistical significance testing was performed, though the specific test used is not named."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context throughout, e.g., 'RL boosted the validity rate on the long test set from 34.8% to 41.5% (a 6.7% increase) and the executability rate from 42.3% to 53.6% (9.0%)' (Section 4.7). These provide enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training set is 4000 instances per domain and the test sets are 200 instances per domain, but no justification is given for why these specific sizes were chosen. The paper notes that 4000 is 5.7% of PlanGPT's data but does not justify why this amount is sufficient statistically."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers with no indication of multiple runs or seeds for the main experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes the vanilla fine-tuned model (no strategies) as a baseline (Table 2 row 1), and compares multiple strategies against it. The SFT baseline is also compared against RL using the same data (Figure 6, Section 4.7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The strategies compared (CoT from Wei et al. 2022, self-correction from Kumar et al. 2024 and Ye et al. 2024, RL from Liu et al. 2024, permutation from Allen-Zhu and Li 2023) are contemporary and represent the state of relevant reasoning-enhancement approaches. The paper explicitly positions against PlanGPT (Rossetti et al. 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 is an extensive ablation study with 11 rows showing combinations of permutation, Goal CoT, State CoT, self-correction, and RL strategies, allowing assessment of each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses validity rate and executability rate as two primary metrics, plus introduces goal satisfiability rate (Appendix G) and pass@k (Appendix F). The introduction of executability as a complement to validity is a key contribution."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates automated planning using formal verification (VAL validator). Plan correctness is deterministically verifiable, making human evaluation irrelevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses distinct test sets explicitly separated from training data: 'In-Distrib' (same distribution, different instances), 'Long' (OOD plan lengths 17-32), 'Unseen' (novel domains), and 'Obfuscated' (obfuscated vocabulary). For RL, 10% of the 'long' test set was used for training and 90% for evaluation (Section 4.7)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-domain breakdowns across all 8 training domains plus 2 unseen and 2 obfuscated domains. Table 2 provides breakdowns by test set category (In-Distrib, Long, Unseen, Obfuscated)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis is provided: Section 4.1 discusses failures on OOD and obfuscated sets, Section 4.3 analyzes Goal CoT failures, Section 4.4 discusses self-correction failures with qualitative analysis, and Appendix D provides a detailed failure case study with visualizations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are reported: Goal CoT hinders performance (Section 4.3), self-correction fails to improve validity rates (Section 4.4), fine-tuning fails on OOD scenarios (Section 4.1), and the obfuscated test set shows 0% across all strategies. The paper is structured around understanding what does and does not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (1) fine-tuning does not lead to robust planning (Table 1 OOD results), (2) strategies like CoT enhance executability (Table 2), (3) RL with LCCS is the most effective strategy (Table 2 rows 10-11). All claims match the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about strategy effectiveness through controlled ablation studies (Table 2), where each row changes one variable at a time. The RL comparison uses the same training data with SFT vs RL to control for data (Figure 6, Section 4.7). The plan continuation experiment (Section 4.6) tests the specific hypothesis about CoT's distribution sensitivity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to 'end-to-end plan generation' paradigm (Section 2, Scope), excluding LLM-Modulo, Thought of Search, and hybrid models like AlphaMath. The Discussion section (Section 5) restates the paradigm limitation. Results are specific to Qwen2-7B on the extended PlanBench dataset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for several findings: the CHILDSNACK success might reflect task structure rather than planning ability (Section 4.1), State CoT's improvement may be limited to in-distribution lengths (Section 4.5, verified in Section 4.6), and the RL improvement is verified not to come from additional training data via SFT control (Section 4.7, Figure 6)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'QWEN2-7B-INSTRUCT' (Section 4, also listed in Appendix B.1 hyperparameter table as 'Qwen2-7B-Instruct'), which is a specific model version and size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt examples are provided: Figure 2 shows the PlanBench query format, Figures 3-5 show the augmented formats for each strategy, Figure 9 shows a complete training query example, and Figure 10 shows a complete response with CoT and self-correction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 provides a comprehensive hyperparameter table covering data generation, fine-tuning (learning rate 1.0e-5, cosine scheduler, epochs, batch sizes, deepspeed config), RL (LoRA rank 128, alpha 256, learning rate 3.0e-6, episodes 500000), and evaluation (top_p 0.93, top_k 50, k values 1/3/5)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is end-to-end plan generation via fine-tuned LLM with standard next-token prediction at inference time."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how the extended PlanBench dataset was constructed from IPC benchmarks, with specific train/test sizes (4000 train per domain, 200 test per domain), plan length ranges (3-16 for training, 17-32 for long test), and the serialization format from PDDL to natural language is described with examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) serves as a combined discussion and limitations section, discussing the paradigm limitations of end-to-end plan generation. Additionally, the LCCS reward's limitation (reliance on a single reference plan) is explicitly acknowledged in Section 3.2."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed throughout: the LCCS reward has 'an inherent bias because it relies on a single reference plan' (Section 3.2), the model was trained on only 8 domains which limits generalization claims, and the RL training used 10% of the long test set (Section 4.7). The paper also discusses potential data contamination risk and why Qwen2 was chosen to reduce it (Section 4)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Scope subsection in Section 2 explicitly states what is excluded: LLM-Modulo framework, Thought of Search, hybrid models like AlphaMath with MCTS. The paper limits itself to 'paradigms that utilize basic next-token prediction during inference.' Section 5 reiterates: 'We have limited our scope to the end-to-end plan generation paradigm.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is promised for future release ('will be released via Huggingface Hub once the review process is completed') but is not yet available. Raw experimental outputs (generated plans, per-instance results) are not provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how the PlanBench dataset was extended: new domains were added from IPC benchmarks, longer-plan problems were generated with plan lengths 17-32, and the overall dataset structure is detailed (4000 train instances/domain, 200 test instances/domain across 4 test categories)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is the PlanBench benchmark and IPC planning benchmarks, which are standard resources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: PDDL domains from IPC benchmarks are serialized to natural language via PlanBench templates (Figure 2), then augmented with various strategies (permutation Figure 3, CoT Figure 4, self-correction Figure 5). Plan validation uses the VAL tool. The train/test split sizes and plan length distributions are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding section is present in the paper. No mention of grants or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: all three authors are from the School of Computing and Information Systems, The University of Melbourne. Trevor Cohn is noted as 'Now at Google DeepMind.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Trevor Cohn's affiliation with Google DeepMind is noted, which could be relevant given the paper evaluates LLM capabilities, but no explicit conflict discussion is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the exact training data cutoff date for Qwen2-7B-Instruct. It notes the model 'was pre-trained on general text but not on code' and 'was trained on a smaller dataset' than LLaMA 3 to 'reduce the risk of having exposed to PlanBench' (Section 4), but no specific cutoff date is provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses this: 'QWEN2's architecture shares significant similarities with LLAMA 3, but was trained on a smaller dataset. It thus reduce the risk of having exposed to PlanBench or related data during pre-training.' The paper also uses obfuscated domains and OOD tests to mitigate this concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination risk by: (1) choosing Qwen2 over larger models to reduce exposure risk, (2) using obfuscated domain names to test whether the model relies on memorized concepts, (3) using OOD test sets with longer plan lengths not seen during fine-tuning, and (4) limiting training data to 5.7% of PlanGPT's amount."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of an LLM fine-tuned for planning."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per example are reported despite the approach requiring fine-tuning and multiple strategy evaluations across thousands of instances."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While '4 Nvidia A100 GPUs' are mentioned in Appendix B, no total GPU hours, training time, or total computational budget is stated. The RL training mentions '500000 total episodes' but not wall-clock time."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuning LLMs on planning datasets does not lead to robust planning skills, as shown by poor performance on out-of-distribution test sets.",
       "evidence": "Table 1 shows validity rate drops from 98.5% to 13.5% in Blocksworld when moving from in-distribution to longer plans. Unseen domains show 0% validity and obfuscated domains show 0% validity and executability (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Strategies like chain-of-thought enhance plan executability even when they do not improve validity rates.",
       "evidence": "Table 2 shows State CoT (row 4) achieves 100% executability on unseen domains while validity remains 0%. Permutation augmentation raises unseen executability from 20.1% to 75.5% (row 2) with no validity improvement (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "RL with the proposed LCCS reward is the most effective strategy, improving validity by 6.7% and executability by 9.0% on longer problems.",
       "evidence": "Table 2 row 10 shows validity improving from 34.8% to 41.5% and executability from 42.3% to 53.6% on the long test set. RL also achieves 12.5% validity on the unseen test set where all other strategies achieve 0% (Section 4.7).",
       "supported": "strong"
     },
     {
       "claim": "Goal CoT hinders planning performance due to a complexity paradox and overfitting to training plan lengths.",
       "evidence": "Table 2 rows 3, 5, 7, 9 show Goal CoT consistently reduces performance. Section 4.3 attributes this to the rigidity of fixing step counts before planning and the model's bias toward in-distribution length estimates.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs can recognize planning mistakes with high precision (90.5%) and recall (99.2%) but fail to correct them.",
       "evidence": "Table 3 shows probing test results with precision 87.5-90.5% and recall 97.4-99.2%. Despite high detection, self-correction does not improve validity rates (Table 2 rows 6-9). Appendix D provides qualitative failure analysis (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "State CoT's effectiveness is limited to in-distribution plan lengths.",
       "evidence": "Table 2 row 4 shows State CoT achieves 100% executability on unseen test set (same length distribution as training) but only 43.0% on long test set. The plan continuation experiment (Section 4.6, Figure 7) confirms CoT works within familiar length ranges.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Fine-tuning LLMs on planning datasets achieves high in-distribution performance but fails drastically on out-of-distribution scenarios, with validity dropping from 98.5% to 13.5% on longer problems and 0% on unseen/obfuscated domains. Various reasoning-enhancement strategies (permutation, CoT, self-correction) improve plan executability but not validity, revealing that progress toward better plans occurs even without achieving final correctness. Reinforcement learning with a novel Longest Contiguous Common Subsequence (LCCS) reward emerged as the most effective strategy, improving both validity (+6.7%) and executability (+9.0%) on longer problems and being the only strategy to achieve non-zero validity on unseen domains (12.5%).",
   "red_flags": [
     {
       "flag": "No variance or error bars reported",
       "detail": "All results are reported as point estimates with no standard deviations across runs. Given that both fine-tuning and RL involve stochastic optimization, single-run results may not be stable. The statistical significance testing mentioned in Table 2 caption is not described in detail."
     },
     {
       "flag": "RL evaluated on same distribution as training subset",
       "detail": "RL was trained on 10% of the 'long' test set and evaluated on the remaining 90%. While the SFT control uses the same data split, this means RL results on the 'long' test set are not fully independent of the RL training distribution, and the paper does not discuss how the 10% was selected."
     },
     {
       "flag": "Anonymous repository during review",
       "detail": "The code repository is an anonymous review link (anonymous.4open.science). The dataset is promised for future release on HuggingFace Hub. Neither may remain accessible post-review."
     }
   ],
   "cited_papers": [
     {
       "title": "Position: LLMs Can't Plan, But Can Help Planning in LLM-Modulo Frameworks",
       "authors": ["Subbarao Kambhampati", "Karthik Valmeekam", "Lin Guan", "Mudit Verma", "Kaya Stechly", "Siddhant Bhambri", "Lucas Paul Saldyt", "Anil Bhaskar Murthy"],
       "year": 2024,
       "relevance": "Key position paper arguing LLMs cannot plan autonomously, directly contrasted by this paper's findings on incremental improvements."
     },
     {
       "title": "Planning in Strawberry Fields: Evaluating and Improving the Planning and Scheduling Capabilities of LRM o1",
       "authors": ["Karthik Valmeekam", "Kaya Stechly", "Atharva Gundawar", "Subbarao Kambhampati"],
       "year": 2024,
       "arxiv_id": "2410.02162",
       "relevance": "Evaluates OpenAI o1's planning capabilities, finding even advanced reasoning models fail on long-term planning tasks."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Fei Xia", "Ed Chi", "Quoc V. Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational CoT prompting paper whose strategy is evaluated and found to have limited effectiveness in planning tasks."
     },
     {
       "title": "Training Language Models to Self-Correct via Reinforcement Learning",
       "authors": ["Aviral Kumar", "Vincent Zhuang", "Rishabh Agarwal", "Yi Su", "John D. Co-Reyes", "Avi Singh", "Kate Baumli", "Shariq Iqbal", "Colton Bishop", "Rebecca Roelofs"],
       "year": 2024,
       "arxiv_id": "2409.12917",
       "relevance": "Self-correction learning strategy evaluated in this paper; effective for math but found ineffective for plan validity."
     },
     {
       "title": "Learning General Policies for Planning through GPT Models",
       "authors": ["Nicola Rossetti", "Marco Tummolo", "Alfonso E. Gerevini", "Luca Putelli", "Ivan Serina", "Mattia Chiari", "Matteo Olivato"],
       "year": 2024,
       "relevance": "PlanGPT model: most closely related prior work showing fine-tuned LLMs can generate plans, but evaluated only in-distribution."
     },
     {
       "title": "PlanBench: An Extensible Benchmark for Evaluating Large Language Models on Planning and Reasoning about Change",
       "authors": ["Karthik Valmeekam", "Matthew Marquez", "Alberto Olmo", "Sarath Sreedharan", "Subbarao Kambhampati"],
       "year": 2024,
       "relevance": "The primary benchmark extended and used in this paper for evaluating LLM planning capabilities."
     },
     {
       "title": "Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms",
       "authors": ["Arash Ahmadian", "Chris Cremer", "Matthias Gallé", "Marzieh Fadaee", "Julia Kreutzer", "Ahmet Üstün", "Sara Hooker"],
       "year": 2024,
       "arxiv_id": "2402.14740",
       "relevance": "Provides the RLOO framework used for sequence-level RL optimization in this paper."
     },
     {
       "title": "GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models",
       "authors": ["Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi", "Oncel Tuzel", "Samy Bengio", "Mehrdad Farajtabar"],
       "year": 2024,
       "arxiv_id": "2410.05229",
       "relevance": "Demonstrates limitations of LLM reasoning on out-of-distribution problems, supporting this paper's OOD evaluation design."
     },
     {
       "title": "Physics of Language Models: Part 2.2, How to Learn From Mistakes on Grade-School Math Problems",
       "authors": ["Tian Ye", "Zicheng Xu", "Yuanzhi Li", "Zeyuan Allen-Zhu"],
       "year": 2024,
       "relevance": "Self-correction learning through error exposure, adapted in this paper for planning but found ineffective for plan validity."
     },
     {
       "title": "Chain of thoughtlessness: An analysis of CoT in planning",
       "authors": ["Kaya Stechly"],
       "year": 2024,
       "arxiv_id": "2405.04776",
       "relevance": "Found CoT prompting did not improve LLM planner validity, a finding this paper partially contradicts by showing executability improvements."
     },
     {
       "title": "The pitfalls of next-token prediction",
       "authors": ["Gregor Bachmann"],
       "year": 2024,
       "arxiv_id": "2403.06963",
       "relevance": "Theoretical analysis of snowballing error in autoregressive models, providing the theoretical basis for why LLM planning degrades with sequence length."
     },
     {
       "title": "Qwen2 technical report",
       "authors": ["An Yang", "Baosong Yang", "Binyuan Hui", "Bo Zheng", "Bowen Yu", "Chang Zhou"],
       "year": 2024,
       "arxiv_id": "2407.10671",
       "relevance": "Technical report for the Qwen2 model used as the base LLM in this paper's experiments."
     }
   ]
 }