ai-research-survey

scan.json (24372B)

---

 {
   "paper": {
     "title": "StateFlow: Enhancing LLM Task-Solving through State-Driven Workflows",
     "authors": ["Yiran Wu", "Tianwei Yue", "Shaokun Zhang", "Chi Wang", "Qingyun Wu"],
     "year": 2024,
     "venue": "COLM 2024",
     "arxiv_id": "2403.11322",
     "doi": "10.48550/arXiv.2403.11322"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StateFlow models LLM task-solving as finite state machines, achieving 13% and 28% higher success rates than ReAct on InterCode SQL and ALFWorld respectively, with 3-5x less inference cost. The framework decomposes long prompts into state-specific shorter prompts, reducing token usage while improving focus. Combining StateFlow with Reflexion further improves ALFWorld performance from 84.3% to 94.8%. Ablation studies show the Observe and Error states are the most important components.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is available at https://github.com/yiranwu0/StateFlow (stated in Appendix A.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: InterCode (SQL and Bash) and ALFWorld. The InterCode repository is cited at https://github.com/princeton-nlp/intercode."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Only the AutoGen version (v0.2.17) is mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code repository is linked but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims StateFlow 'outperforms' baselines based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '13% higher success rate compared to ReAct' (from 50.68% to 63.73%), and '5× less cost'. Tables 1-4 provide full baseline comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for using the full InterCode SQL (1034 tasks), Bash (200 tasks), or ALFWorld (134 tasks) benchmarks. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "For ALFWorld, results are 'average success rate of 3 attempts' (Table 4) but no standard deviation or variance is reported across those attempts. InterCode results appear to be single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: Plan & Solve, ReAct, ReAct Refined (for SQL), ALFChat with 2 and 3 agents (for ALFWorld). Tables 1, 2, 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ReAct (2022), Plan & Solve (2023), ALFChat/AutoGen (2023), and Reflexion (2023), all contemporary at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an ablation study removing individual states (Verify, Error, Observe) from the SQL StateFlow model, showing each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: Success Rate, Error Rate, Turns, Cost (Tables 1-2), and Reward (Tables 6-7)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via benchmark metrics (success rate, reward)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmarks (InterCode SQL/Bash, ALFWorld) with their standard evaluation splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "SQL results are broken down by difficulty level (Easy, Medium, Hard, Extra Hard) in Table 5. ALFWorld results are broken down by task type (Pick, Clean, Heat, Cool, Look, Pick 2) in Table 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Appendix B.2 analyze ALFWorld failures: 15/21 failed tasks ended in Pick state, with three failure reasons identified (hallucination, wrong object, loops). Table 9 classifies failures by ending state."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT-4 did not improve over GPT-3.5 on Bash tasks ('Switching to GPT-4-Turbo has little effect on the methods, where the two baselines even suffer from a decrement in accuracy'). Also, GPT-3.5-Instruct showed performance drops for both methods (Table 12)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 13% and 28% improvement over ReAct and 5x/3x cost reduction are supported by Tables 1 and 4. The Reflexion integration claim (84.3% to 94.8%) is supported by Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 3) provides controlled single-variable manipulation to support causal claims about which components matter. Each state is removed individually while keeping others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Enhancing LLM Task-Solving' broadly, but results are limited to three specific benchmarks (InterCode SQL/Bash, ALFWorld) with primarily GPT-3.5 and GPT-4. The paper does not bound its claims to these specific task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, the cost reduction may largely come from shorter prompts rather than the state machine structure itself. The ReAct Refined baseline suggests prompt engineering alone accounts for much of the gain, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures success rate, error rate, turns, and cost on specific benchmarks and reports them as such without inflating claims beyond what these metrics measure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-3.5-Turbo' and 'GPT-4-Turbo (both with the 1106 version)' in Section 4.1, which is a specific API version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all states is provided in the appendix: Tables 13-19 contain the actual instructions used for SQL, Bash, and ALFWorld tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'temperature is set to 0' and 'max of 10 rounds of interaction' for InterCode. Section 4.2 states 'maximum of 50 rounds' for ALFWorld."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper describes the StateFlow scaffolding in detail: state definitions, transition rules, output functions, and Algorithm 1 provides the pseudocode. Figures 1, 3, 4 show the state machine diagrams."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard benchmarks as-is. Section 4.1 describes the InterCode setup (MySQL in Docker) and evaluation protocol. ALFWorld setup is described in Appendix B.1 with BLEU metric mapping."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion briefly mentions that 'StateFlow requires humans to have a good understanding of a given task' but this is a single sentence, not substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential confounds such as prompt engineering effects, benchmark-specific overfitting, or the manual effort required for state machine design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show or what settings are excluded. It does not bound claims to the tested benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (model responses, execution traces) are released. Only aggregate metrics are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks are well-described: InterCode-SQL adapts Spider (1034 tasks), InterCode-Bash has 200 tasks from NL2Bash, ALFWorld has 134 tasks across 6 types. Docker setup is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: benchmarks → model interaction with environment → reward calculation. BLEU metric mapping for ALFWorld is described (Appendix B.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Acknowledgements only thank reviewers and a figure designer."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Pennsylvania State University, MathGPTPro, and Microsoft Research Redmond."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Co-author Chi Wang is from Microsoft Research. The paper uses OpenAI models and the AutoGen framework (from Microsoft). The relationship between Microsoft and the framework being promoted is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Chi Wang is from Microsoft Research, and the implementation is based on AutoGen (a Microsoft project), but this potential conflict is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of GPT-3.5 or GPT-4's training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark tasks (Spider SQL, NL2Bash, ALFWorld) appeared in GPT model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Spider was published in 2018, NL2Bash earlier, and ALFWorld in 2020 — all likely in GPT-3.5/GPT-4 training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "LLM API costs in US dollars are reported for all methods across all benchmarks (Tables 1, 2, 4, 6, 7). Token counts (prompt and completion) are also reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total computational budget (e.g., total API spend for all experiments, hardware used) is not stated. Per-method costs are given but not the total experimental budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 (deterministic), but ALFWorld results are averaged over 3 attempts without reporting variance across those attempts. No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "ALFWorld: 'We report average success rate of 3 attempts' (Table 4). InterCode appears to be single runs with temperature 0."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The state machine designs and prompts appear hand-crafted but no mention of how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The state machine designs are presented without explaining how they were selected or iterated upon. The 10-state ALFWorld variant (Appendix B.3) improves over 7-state, suggesting configurations were explored but the process is not documented."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their StateFlow system against their own implementations/adaptations of baselines (ReAct Refined, ALFChat) without acknowledging self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Cost is reported alongside performance for all methods (Tables 1, 2, 4), and Figure 5 shows cumulative cost vs. success rate for the Reflexion integration."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether InterCode SQL/Bash and ALFWorld actually measure the capabilities claimed. The paper assumes benchmark performance equals 'task-solving' ability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "StateFlow changes both the scaffolding structure and the prompts simultaneously. The ReAct Refined baseline partially isolates the prompt effect but the scaffold vs. prompt contribution is not disentangled."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks (Spider 2018, NL2Bash, ALFWorld 2020) predate GPT-3.5/GPT-4 training. No discussion of whether models saw benchmark solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., the interactive feedback from environments could compensate for leakage effects)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between training data and benchmark tasks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "StateFlow achieves 13% higher success rate than ReAct on InterCode SQL with GPT-3.5, with 5x less cost.",
       "evidence": "Table 1: StateFlow 63.73% SR vs ReAct 50.68% SR; cost $3.82 vs $17.7.",
       "supported": "strong"
     },
     {
       "claim": "StateFlow achieves 28% higher success rate than ReAct on ALFWorld with GPT-3.5.",
       "evidence": "Table 4: StateFlow 83.3% vs ReAct 55.5% overall success rate, with 2.5x less cost ($2.6 vs $6.6).",
       "supported": "moderate"
     },
     {
       "claim": "StateFlow combined with Reflexion improves ALFWorld success rate from 84.3% to 94.8% after 6 iterations.",
       "evidence": "Figure 5 shows the progression. ReAct+Reflexion reaches only 74.6% at 3x higher cost.",
       "supported": "moderate"
     },
     {
       "claim": "The Observe state is the most important component in the SQL StateFlow model.",
       "evidence": "Table 3 ablation: removing Observe drops SR from 63.73% to 57.83% (largest drop), also increases cost and error rate.",
       "supported": "strong"
     },
     {
       "claim": "Further decomposition of states (7→10) improves ALFWorld performance to 88.8% with 15% less cost.",
       "evidence": "Table 10 in Appendix B.3 compares 7-state (83.3%) vs 10-state (88.8%) StateFlow.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest not disclosed",
       "detail": "Co-author Chi Wang is from Microsoft Research, and the implementation is built on AutoGen (a Microsoft project). The paper effectively promotes the AutoGen framework without disclosing this as a potential conflict."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('outperforms', 'improves') are made by comparing point estimates without any significance tests. With only 3 runs on ALFWorld and single runs on InterCode, the observed differences may not be statistically significant."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "Spider (2018), NL2Bash, and ALFWorld (2020) all predate GPT-3.5/GPT-4 training. The models may have memorized solutions. This is especially concerning because StateFlow's structured workflow could exploit memorized patterns differently than ReAct."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. The manual effort required to design state machines for each task type is acknowledged only briefly in the conclusion."
     }
   ],
   "cited_papers": [
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Primary baseline for LLM task-solving with interleaved reasoning and acting."
     },
     {
       "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Iterative self-improvement method combined with StateFlow to demonstrate composability."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang", "Yiran Wu"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent framework used as the implementation foundation for StateFlow."
     },
     {
       "title": "InterCode: Standardizing and benchmarking interactive coding with execution feedback",
       "authors": ["John Yang", "Akshara Prabhakar", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "arxiv_id": "2306.14898",
       "relevance": "Primary benchmark for evaluating interactive code generation with environment feedback."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas L Griffiths", "Yuan Cao", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Alternative LLM reasoning framework modeling thought processes as trees."
     },
     {
       "title": "Graph of thoughts: Solving elaborate problems with large language models",
       "authors": ["Maciej Besta"],
       "year": 2023,
       "arxiv_id": "2308.09687",
       "relevance": "Alternative framework modeling LLM reasoning as directed graphs with transformations."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework for software development using LLMs."
     },
     {
       "title": "CAMEL: Communicative agents for 'mind' exploration of large scale language model society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "relevance": "Framework for autonomous LLM agent cooperation."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Self-improvement method for LLMs through iterative refinement."
     },
     {
       "title": "Language agent tree search unifies reasoning acting and planning in language models",
       "authors": ["Andy Zhou"],
       "year": 2023,
       "arxiv_id": "2310.04406",
       "relevance": "LLM tree search method incorporating reflection and environment feedback."
     },
     {
       "title": "ALFWorld: Aligning text and embodied environments for interactive learning",
       "authors": ["Mohit Shridhar"],
       "year": 2020,
       "arxiv_id": "2010.03768",
       "relevance": "Text-based embodied environment benchmark used for evaluation."
     }
   ]
 }