ai-research-survey

scan.json (28716B)

---

 {
   "paper": {
     "title": "Least-to-Most Prompting Enables Complex Reasoning in Large Language Models",
     "authors": [
       "Denny Zhou",
       "Nathanael Schärli",
       "Le Hou",
       "Jason Wei",
       "Nathan Scales",
       "Xuezhi Wang",
       "Dale Schuurmans",
       "Claire Cui",
       "Olivier Bousquet",
       "Quoc Le",
       "Ed Chi"
     ],
     "year": 2022,
     "venue": "International Conference on Learning Representations (ICLR 2023)",
     "arxiv_id": "2205.10625",
     "doi": "10.48550/arXiv.2205.10625"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Least-to-most prompting, a two-stage approach that decomposes problems into subproblems then solves them sequentially, substantially outperforms chain-of-thought prompting on easy-to-hard generalization. On SCAN length split, code-davinci-002 with least-to-most prompting achieves 99.7% accuracy using only 14 exemplars versus 16.2% for chain-of-thought. On last-letter-concatenation, least-to-most maintains 74.0% at length 12 versus chain-of-thought's 31.8%. On GSM8K, improvement is marginal overall (62.4% vs 60.9%) but meaningful for problems requiring 5+ steps (45.2% vs 39.1%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or GitHub link is provided. The paper includes all prompts in the appendix but no implementation code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The main evaluation benchmarks (SCAN, GSM8K, DROP) are publicly available standard datasets. The custom last-letter-concatenation dataset is constructed from a publicly available Wiktionary frequency list with the procedure fully described in Appendix 7.3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements file, or dependency list is provided. The paper names the GPT-3 model variants used but provides no computational environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While all prompts are provided in the appendices (a significant aid), there are no step-by-step instructions, scripts, or README describing how to run the experiments end-to-end. A researcher would need to reverse-engineer the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 8, 11, 12, 13 are single point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly claims one method 'significantly outperforms' another (e.g., abstract, Section 6) based solely on comparing raw accuracy numbers. No statistical significance tests (p-values, bootstrap tests, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results tables provide both baseline and proposed method accuracies side by side (e.g., Table 4: chain-of-thought 31.8% vs least-to-most 74.0% at L=12; Table 8: 16.2% vs 99.7% on SCAN), giving clear context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 500 test lists per length for last-letter-concatenation is stated but not justified. No power analysis or sample size rationale is provided for any experiment."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be single-run numbers. There is no mention of running experiments multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard few-shot prompting and chain-of-thought prompting are included as baselines across all experiments. For SCAN, comparisons with specialized neural-symbolic models from the literature are also discussed (Section 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Chain-of-thought prompting (Wei et al., 2022) was the state-of-the-art prompting method at the time. For SCAN, the paper compares against the best known neural-symbolic approaches (Chen et al., 2020; Liu et al., 2020; Nye et al., 2020; Shaw et al., 2021)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between least-to-most and chain-of-thought prompting effectively ablates the decomposition stage (both use the same exemplars for subproblem solving). Table 13 further varies the number of exemplars (2, 4, 8-shot) and prompt styles, testing component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (exact match) is used as the evaluation metric across all tasks. No secondary metrics such as decomposition quality, partial credit, or efficiency measures are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "All tasks (SCAN, last-letter-concatenation, GSM8K, DROP) have well-defined correct answers amenable to exact-match automated evaluation. Human evaluation is not relevant to the accuracy claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SCAN, GSM8K, and DROP have established test splits. The last-letter-concatenation test data is freshly generated from a word list, completely independent of the prompt exemplars."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down by list length (4-12), Table 8 by model, Table 11 by DROP subset (football/non-football), Table 12 by number of reasoning steps required, and Table 13 by prompt configuration and model variant."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed error analysis is provided for all three task types: Section 3.1 and Appendix 7.4 for last-letter-concatenation (concatenation errors, wrong template), Section 3.2 and Appendix 8.2 for SCAN (13 errors catalogued by type), and Appendix 9.5 for DROP."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 shows chain-of-thought actually outperforms least-to-most on 2-step GSM8K problems (76.68% vs 74.53%). Overall GSM8K improvement is marginal (62.39% vs 60.87%). Section 5 honestly discusses domains where decomposition fails. Table 19 shows Least-to-Most (best) at 68.01% vs Chain-of-Thought (best) at 68.61%."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'at least 99%' on SCAN (actual: 99.7%, Table 8), '16% accuracy with chain-of-thought' (actual: 16.2%, Table 8), and '14 exemplars' (confirmed in Appendix 8.1.2). All specific claims are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims decomposition 'enables' complex reasoning. The controlled comparison between least-to-most and chain-of-thought uses identical exemplars for subproblem solving, isolating the decomposition variable. This single-variable manipulation is adequate for the causal claim about decomposition's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Despite the broad title, Section 5 explicitly bounds the claims: 'Decomposition prompts typically don't generalize well across different domains' and 'Generalizing decomposition can even be difficult within the same domain.' The abstract scopes to specific tasks. Results are model-specific (GPT-3 variants)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Table 13 address the main confound that least-to-most prompts contain more tokens by testing chain-of-thought with 4 and 8 exemplars, showing the advantage persists. They also compare using identical vs independent exemplars (Section 7.2.4), isolating the recursive structure as the key factor."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exact-match accuracy on specific benchmarks and claims performance on those specific tasks (compositional generalization on SCAN, length generalization on last-letter-concatenation, multi-step math on GSM8K). Claims closely match measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact GPT-3 model versions are specified throughout: code-davinci-002, text-davinci-002, code-davinci-001. These are specific API model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All prompts are provided in full in the appendices: Appendix 7 (last-letter-concatenation), Appendix 8 (SCAN), Appendix 9 (DROP), Appendix 10 (GSM8K). The paper explicitly states: 'We have included prompts for all the tasks in the Appendix.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters are reported: temperature, top-p, max tokens, and other generation settings are not mentioned despite their significant impact on LLM output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a pure prompting paper with sequential API calls."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 7.3 details the last-letter-concatenation data construction: Wiktionary top 10k words, profane word removal yielding 9694 words, random list generation of 500 examples per length. SCAN, GSM8K, and DROP use established public benchmark splits. Python expression postprocessing for SCAN is described in Section 3.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is a dedicated 'Limitations' section with substantive discussion of when the approach fails."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific limitations: decomposition prompts don't transfer across domains (math → common sense), decomposition within a domain can fail even with correct sub-solutions, and exceptional SCAN/last-letter results occur because decomposition is 'relatively straightforward' in those tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states what the method does NOT do: 'a prompt that demonstrates decomposing math word problems isn't effective for teaching large language models to break down common sense reasoning problems, such as \"Did Aristotle use a laptop?\"'. The boundary between easy and hard decomposition is acknowledged."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw model outputs, individual predictions, or per-example results are available. Only aggregate accuracy numbers are reported in the tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The last-letter-concatenation dataset construction is described in detail in Appendix 7.3 (source word list, filtering steps, sampling procedure). The other benchmarks (SCAN, GSM8K, DROP) are well-known public datasets with published documentation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks or algorithmically generated test cases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "For last-letter-concatenation: Wiktionary list → profanity filter (10000→9694 words) → random list sampling → evaluation. For SCAN: established length split. For DROP: numerical reasoning subset. The Python expression postprocessing step for SCAN is described in Section 3.2 and Appendix 8.4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgement section thanks individuals but does not mention grants, funding agencies, or corporate funding. All authors are from Google Research, Brain Team, implying Google funding, but this is not explicitly stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are clearly identified as 'Google Research, Brain Team' in the author list. Since they evaluate OpenAI's GPT-3 (not a Google product), the standard company-evaluating-own-product conflict does not apply."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Google (the implicit funder via author employment) does not have a direct financial stake in GPT-3's performance with this prompting technique. The researchers are evaluating a competitor's product, making the funding source reasonably independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3 models (code-davinci-002, text-davinci-002, code-davinci-001) without stating any training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SCAN (2018), GSM8K (2021), or DROP (2019) examples appeared in GPT-3's pre-training data. All three benchmarks predate the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All benchmarks (SCAN published 2018, DROP published 2019, GSM8K published 2021) were publicly available before GPT-3's training. No discussion of contamination risk for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Least-to-most prompting requires multiple sequential API calls (decomposition + one call per subproblem) versus a single call for chain-of-thought. No inference cost, latency, or token consumption is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API costs, or hardware resources are mentioned despite running thousands of API calls across multiple GPT-3 models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No seed sensitivity analysis or results across multiple random seeds are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state whether results are from single runs or averaged across multiple runs. This is particularly concerning for LLM API calls where temperature settings affect variability."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The prompts appear manually designed. No reporting of how many prompt designs were tried, what alternatives were considered, or what search process led to the final prompts."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "While Table 13 shows multiple configurations, the main results use specific configurations (e.g., 2-shot for Table 4) without justifying why these were selected as the primary comparison. Table 19 shows engineered prompts outperform the 1-shot versions but the main paper highlights the simpler versions."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so correction for multiple comparisons does not arise. The absence of statistical testing is captured by the significance_tests item."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors design and evaluate their own prompting method against baselines. No acknowledgment of author-evaluation bias, and no independent evaluation is reported."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Least-to-most prompting requires O(n) API calls for a problem decomposed into n subproblems, while chain-of-thought uses a single call. This substantial compute difference is never discussed or controlled for. The performance advantage may partly reflect the additional compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SCAN, GSM8K, and DROP without discussing whether these benchmarks adequately measure 'complex reasoning' as claimed. SCAN tests a narrow form of compositional generalization; whether success on it implies general complex reasoning capability is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a pure prompting approach."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SCAN (2018), DROP (2019), and GSM8K (2021) were all publicly available before GPT-3 model training. The paper does not discuss whether the models may have seen benchmark solutions during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompt design or evaluation setup leaks information about correct answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3's training data includes SCAN grammar rules, GSM8K problem patterns, or DROP reading comprehension passages that overlap with test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, decontamination) are employed."
       }
     }
   },
   "claims": [
     {
       "claim": "Least-to-most prompting achieves 99.7% accuracy on SCAN length split using only 14 exemplars, compared to 16.2% for chain-of-thought prompting.",
       "evidence": "Table 8 (Section 3.2) shows code-davinci-002 results: least-to-most 99.7%, chain-of-thought 16.2%, standard prompting 16.7%. The 14 exemplars are the command-mapping prompt (Appendix 8.1.2).",
       "supported": "strong"
     },
     {
       "claim": "Least-to-most prompting significantly outperforms chain-of-thought on length generalization for last-letter-concatenation, with the gap widening as length increases.",
       "evidence": "Table 4 (Section 3.1): at L=4, least-to-most 94.0% vs chain-of-thought 84.2%; at L=12, 74.0% vs 31.8%. Table 13 extends this across multiple prompt sizes and models, showing consistent advantage.",
       "supported": "strong"
     },
     {
       "claim": "Least-to-most prompting improves chain-of-thought on GSM8K, particularly for problems requiring 5+ steps (45.23% vs 39.07%).",
       "evidence": "Table 12 (Section 3.3) breaks down by step count. Overall improvement is marginal: 62.39% vs 60.87% (Table 11). The step-count breakdown shows the advantage concentrates on harder problems. However, no statistical tests confirm the 5+ step difference is significant.",
       "supported": "moderate"
     },
     {
       "claim": "Least-to-most prompting outperforms chain-of-thought prompting on DROP by a large margin.",
       "evidence": "Table 11: non-football subset 82.45% vs 74.77%, football subset 73.42% vs 59.56%. These are substantial absolute differences but no statistical testing is provided.",
       "supported": "moderate"
     },
     {
       "claim": "Least-to-most prompting can solve SCAN in any split with at least 99% accuracy.",
       "evidence": "Section 3.2 states: 'We also test least-to-most prompting on all other splits and even the full SCAN dataset. We find that its solving rate remains the same.' However, detailed per-split results are not shown in the paper.",
       "supported": "moderate"
     },
     {
       "claim": "The advantage of least-to-most over chain-of-thought is not simply due to more information in the prompt.",
       "evidence": "Table 13 (Appendix 7.3) shows 2-shot least-to-most (123 tokens) outperforms 8-shot chain-of-thought (573 tokens) at L=12: 74.0% vs 38.4%. This controls for prompt information content.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single point estimates with no error bars, confidence intervals, standard deviations, or statistical tests. The word 'significantly' is used throughout without statistical basis. For a method involving stochastic LLM API calls, result variability is unknown."
     },
     {
       "flag": "Compute cost asymmetry not addressed",
       "detail": "Least-to-most prompting requires multiple sequential API calls (1 decomposition + N subproblem calls) versus 1 call for chain-of-thought. For SCAN with decomposition into 3-4 steps, this is 4-5x the compute. This cost difference is never acknowledged, making the comparison unfair from a practical standpoint."
     },
     {
       "flag": "No contamination discussion",
       "detail": "All benchmarks (SCAN 2018, DROP 2019, GSM8K 2021) were publicly available before GPT-3's training data collection. GPT-3 code-davinci-002 could have seen SCAN grammar rules, GSM8K problems, or DROP passages during pre-training. This is especially concerning for SCAN where 99.7% accuracy is achieved with only 14 exemplars."
     },
     {
       "flag": "Missing API hyperparameters",
       "detail": "Temperature, top-p, and other generation settings are not reported for any experiment. These can significantly affect LLM output variability and quality, making exact reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Ed Chi", "Brian Ichter", "Fei Xia", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Core baseline method; established chain-of-thought prompting as a paradigm for LLM reasoning that least-to-most extends."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Introduced GPT-3 and few-shot prompting, the foundation for all prompting methods evaluated in this paper."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed Chi", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Proposed self-consistency decoding for chain-of-thought prompting; complementary technique that can be combined with least-to-most."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Large language model that also uses chain-of-thought prompting; relevant to understanding scale effects on reasoning capabilities."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduced the GSM8K benchmark used for math reasoning evaluation in this paper."
     },
     {
       "title": "Generalization without systematicity: On the compositional skills of sequence-to-sequence recurrent networks",
       "authors": ["Brenden Lake", "Marco Baroni"],
       "year": 2018,
       "relevance": "Introduced the SCAN benchmark for compositional generalization, the primary evaluation benchmark where least-to-most achieves near-perfect accuracy."
     },
     {
       "title": "AI Chains: Transparent and controllable human-AI interaction by chaining large language model prompts",
       "authors": ["Tongshuang Wu", "Michael Terry", "Carrie Jun Cai"],
       "year": 2022,
       "relevance": "Proposed chaining LLM steps for complex tasks; conceptually related to least-to-most's sequential subproblem solving approach."
     },
     {
       "title": "Unsupervised question decomposition for question answering",
       "authors": ["Ethan Perez", "Patrick Lewis", "Wen-tau Yih", "Kyunghyun Cho", "Douwe Kiela"],
       "year": 2020,
       "relevance": "Prior work on task decomposition for multi-hop QA; least-to-most extends decomposition to few-shot prompting without training."
     },
     {
       "title": "Compositional generalization via neural-symbolic stack machines",
       "authors": ["Xinyun Chen", "Chen Liang", "Adams Wei Yu", "Dawn Song", "Denny Zhou"],
       "year": 2020,
       "relevance": "Neural-symbolic approach achieving 100% on SCAN but requiring full training set and specialized architecture, contrasting with least-to-most's 14-exemplar prompting approach."
     }
   ]
 }