ai-research-survey

scan-v5.json (28607B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
     "authors": [
       "Wei, J.",
       "Wang, X.",
       "Schuurmans, D.",
       "Bosma, M.",
       "Ichter, B.",
       "Xia, F.",
       "Chi, E.",
       "Le, Q.",
       "Zhou, D."
     ],
     "year": 2022,
     "venue": "NeurIPS 2022",
     "arxiv_id": "2201.11903",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are directly supported: CoT improves arithmetic/commonsense/symbolic reasoning (Sections 3–5), and PaLM 540B achieves SOTA on GSM8K (Table 1, Figure 2) with exact numbers provided.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 ablation study isolates causal role of sequential natural language reasoning by testing equation-only, variable-compute-only (dots), and reasoning-after-answer variants, ruling out competing explanations for gains.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 and Appendix A.3 explicitly bound generalization: CoT is an emergent ability requiring ~100B+ parameter models, gains are minimal for easy single-step tasks, and conditions for benefit are stated.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly tests and rules out: variable computation alone, equation-only intermediate steps, and chain-of-thought provided only after the answer as alternative explanations for performance gains.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states 'chain of thought emulates the thought processes of human reasoners' but 'does not answer whether the neural network is actually reasoning,' clearly distinguishing benchmark accuracy from genuine reasoning.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) contains a dedicated limitations paragraph listing four specific limitations: the open question of actual reasoning, annotation costs for finetuning, no guarantee of correct reasoning paths, and the large-scale requirement.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are addressed: CoT hurts models below ~10B parameters (Table 2), prompt engineering sensitivity is quantified with variance across annotators (Tables 6-7), and incorrect reasoning paths leading to correct answers are identified as a validity concern (Appendix D.1).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 explicitly states conditions where CoT helps vs. does not: requires challenging multi-step reasoning, large model, and flat scaling curve; gains are minimal for easy problems where models already score >90%.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure statement is present; all authors are Google Research employees but no formal funding acknowledgment appears in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All nine authors are explicitly identified as 'Google Research, Brain Team' in the paper header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Google employees evaluating Google's proprietary PaLM model, which achieves the headline SOTA result; Google benefits directly from positive outcomes.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests is included anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly defines 'chain of thought' as 'a series of intermediate natural language reasoning steps that lead to the final output' and distinguishes it from standard prompting with a concrete example.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it explores chain-of-thought prompting—providing CoT demonstrations as few-shot exemplars—to unlock reasoning abilities in LLMs without finetuning, with the contribution positioned as combining rationale-augmented training and few-shot prompting.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 and Appendix C (Extended Related Work) engage substantively with five prior directions, explicitly situating CoT as orthogonal to instruction-following approaches (augments outputs vs. inputs) and distinct from finetuning-based rationale methods.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No source code is released; Appendix E.1 provides prompts and supplementary LaMDA predictions but no code for reproducing the experimental pipeline. GPT-3 experiments can be attempted via API with provided prompts but constitute replication, not released code.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All benchmarks used (GSM8K, SVAMP, ASDiv, AQuA, MAWPS, CSQA, StrategyQA, BIG-bench tasks, SayCan) are publicly available; synthetic datasets (last letter concatenation, coin flip) are provided in supplementary materials.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix E.2 mentions TPU v3 (8x8) for LaMDA and TPU v4 (4x4x12) for PaLM inference but provides no software environment specs, dependency versions, or configuration files.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "LaMDA and PaLM are proprietary and inaccessible; while prompts are in Appendix G, Appendix E.1 acknowledges reproducibility is limited, and no step-by-step instructions for running experiments are provided even for GPT-3.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviations across 5 random seed orderings are reported for LaMDA 137B in Tables 6 and 7; for GPT-3 and PaLM, single runs are used due to API cost, explicitly acknowledged.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, ANOVA, p-values) are applied to any comparative claims; all comparisons are presented as raw accuracy numbers.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute percentage point improvements are consistently reported throughout (e.g., PaLM 540B improves +39pp on GSM8K from 17.9% to 56.9%), providing meaningful effect size context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Benchmark evaluation set sizes are not justified statistically; the manual error analysis of 50 correct and 50 incorrect examples (Appendix D) has no power analysis or justification for the sample size.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviation across random seed orderings is reported for LaMDA 137B (Tables 6-7); for other models, single exemplar orders are used with explicit acknowledgment of this limitation.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Standard few-shot prompting (without chain of thought) is used as the primary baseline throughout all experiments, and prior supervised SOTA numbers from published work are included.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include contemporaneous finetuned models (Cobbe et al. 2021 for GSM8K, Jie et al. 2022 for SVAMP, Lan et al. 2021 for MAWPS) and the same underlying LLMs with standard prompting.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 presents three ablations with LaMDA and PaLM: equation-only prompting, variable-compute-only (dots equal to equation length), and chain-of-thought provided only after the answer.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": false,
           "justification": "All quantitative evaluation uses accuracy (solve rate) as the sole metric; no efficiency, calibration, partial credit, or diversity metrics are reported.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No formal human evaluation with external raters is conducted; the manual error analysis in Appendix D (50 correct, 50 incorrect outputs) is author inspection without inter-rater reliability measurement or formal evaluation protocol.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Standard benchmark evaluation splits are used; for BIG-bench tasks without training sets, the first 10 examples serve as exemplars and remaining examples form the evaluation set.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by model family, scale, and dataset; MAWPS is stratified into SingleOp/SingleEq/AddSub/MultiArith subsets (Table 3) showing differential benefits by difficulty level.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.2 provides detailed categorized failure analysis of 50 incorrect LaMDA 137B outputs: 8% calculator errors, 16% symbol mapping errors, 22% one-step-missing errors, 54% semantic understanding failures, with concrete examples for each.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 shows CoT hurts performance for models below ~10B parameters across all model families; Table 3 shows minimal or negative gains for easy single-step MAWPS tasks; CSQA shows minimal gains for GPT-3.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "GPT-3 API names (text-ada-001, text-babbage-001, text-curie-001, text-davinci-002) and Codex (code-davinci-002) are specified; PaLM and LaMDA parameter counts (8B/62B/540B and 420M/2B/8B/68B/137B) are stated.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix G (Tables 20-28) provides complete few-shot prompts for all nine task types including math word problems (three annotator variants), AQuA, last letter concatenation, coin flip, CSQA, StrategyQA, date understanding, sports understanding, and SayCan.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Greedy decoding is specified for all models; number of few-shot exemplars per task is stated (8 for most, 4 for AQuA, 6 for SayCan); token constraints on exemplar sampling are documented (≤60 tokens, ≤2 steps).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; the method is standard few-shot prompting with CoT exemplars, which is the core method being studied rather than a scaffold around another system.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Exemplar selection criteria are documented (random sampling from training sets with length constraints); symbolic dataset generation is described using top-1000 names from namecensus.com; benchmark splits are specified.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Only LaMDA 137B inputs/targets/predictions are provided as supplementary zip; PaLM and GPT-3 raw outputs are not released, making the headline SOTA PaLM results independently unverifiable.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Prompt construction process is described (manual composition with no special instructions to annotators, random sampling from GSM8K training for robustness experiments); symbolic dataset generation procedure documented with source.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; standard public benchmarks are used and annotators are paper co-authors.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from prompt construction to model inference to accuracy evaluation is clearly described; calculator augmentation post-processing is detailed; LaMDA exact inputs/outputs provided in supplementary.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for PaLM, LaMDA, GPT-3, and Codex are not stated anywhere in the paper.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential overlap between model pretraining corpora and benchmark test sets; this is a significant omission given that proprietary model training data is undisclosed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The possibility that GSM8K, SVAMP, CSQA, or other benchmark examples appeared in pretraining data of any evaluated model is not addressed despite this being a known issue with large pretrained models.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved; NeurIPS checklist confirms N/A.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved; NeurIPS checklist confirms N/A.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Appendix E.2 describes hardware used (TPU v3 8x8 for LaMDA, TPU v4 4x4x12 for PaLM) but explicitly states 'we did not estimate the total amount of compute'; no inference cost, latency, or per-query cost is reported.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware configurations are described (TPU v3 and v4 chip counts) but total compute budget in FLOPs, GPU-hours, or API cost is not provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Chain-of-thought prompting significantly improves LLM performance on arithmetic reasoning benchmarks",
       "evidence": "PaLM 540B improves from 17.9% to 56.9% on GSM8K (+39pp), 69.4% to 79.0% on SVAMP; GPT-3 175B from 15.6% to 46.9% on GSM8K (Table 1)",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought reasoning is an emergent ability that only appears in models with approximately 100B+ parameters",
       "evidence": "Table 2 shows CoT hurts or shows no benefit for models below ~10B parameters across LaMDA, GPT, and PaLM families; large gains only emerge at 100B+",
       "supported": "strong"
     },
     {
       "claim": "PaLM 540B with CoT achieves state-of-the-art on GSM8K, surpassing finetuned GPT-3 with a verifier",
       "evidence": "Figure 2 and Table 1: PaLM 540B CoT achieves 56.9% vs. prior best of 55% (finetuned GPT-3 + verifier, Cobbe et al. 2021)",
       "supported": "strong"
     },
     {
       "claim": "Sequential natural language reasoning steps drive CoT gains, not variable computation or equation generation alone",
       "evidence": "Section 3.3 ablation: variable-compute-only (dots) matches baseline; equation-only improves less than full CoT; reasoning-after-answer matches baseline",
       "supported": "strong"
     },
     {
       "claim": "CoT prompting enables out-of-distribution length generalization in symbolic reasoning tasks",
       "evidence": "Figure 8: PaLM 540B CoT achieves 94.8% on 3-word and 63.0% on 4-word last letter concatenation (OOD), vs. near 0% for standard prompting",
       "supported": "moderate"
     },
     {
       "claim": "CoT improvements are robust to different annotators, exemplar sets, and exemplar orderings",
       "evidence": "Section 3.4 and Tables 6-7: all annotator variants and GSM8K-sampled exemplars outperform standard prompting baseline, though coin flip variance across annotators is high (71.4%–99.6%)",
       "supported": "strong"
     },
     {
       "claim": "CoT generalizes to commonsense reasoning, surpassing prior SOTA on StrategyQA and sports understanding",
       "evidence": "Figure 7: PaLM 540B CoT achieves 77.8% on StrategyQA (vs. 69.4% prior best) and 95.4% on sports understanding (vs. 84% unaided human enthusiast)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Chain-of-thought prompting—augmenting few-shot exemplars with intermediate natural language reasoning steps—dramatically improves LLM performance on arithmetic, commonsense, and symbolic reasoning, with PaLM 540B achieving SOTA on GSM8K by surpassing finetuned models. The effect is an emergent property of model scale: CoT actually hurts models below ~10B parameters and only yields gains at ~100B+. Ablations rule out variable computation and simple equation generation as explanations, attributing gains to sequential natural language reasoning. Error analysis reveals 54% of incorrect outputs involve fundamental semantic understanding failures, and the paper frankly acknowledges that benchmark accuracy does not prove genuine reasoning capability.",
   "red_flags": [
     {
       "flag": "Proprietary model lock-in",
       "detail": "The headline SOTA result (PaLM 540B) relies on a proprietary model inaccessible to outside researchers, making independent replication of the most important claim impossible."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are made without p-values or confidence intervals; only LaMDA results include error bars (5 random seed orderings), and no significance test is applied to any comparison."
     },
     {
       "flag": "Training contamination unaddressed",
       "detail": "No discussion of whether GSM8K, SVAMP, CSQA, or other benchmark test examples appeared in pretraining data; particularly concerning for proprietary models with undisclosed training corpora."
     },
     {
       "flag": "Self-evaluation of own model",
       "detail": "All authors are Google Research employees, and the headline result showcases Google's PaLM model achieving SOTA; no independent verification of PaLM results is possible."
     },
     {
       "flag": "Single accuracy metric throughout",
       "detail": "All quantitative results rely solely on accuracy (solve rate); no efficiency, calibration, robustness, or partial-credit metrics are reported across any of the 10+ benchmarks evaluated."
     },
     {
       "flag": "Causal mechanism largely speculative",
       "detail": "Despite ablations ruling out proximate alternatives, Section A.1 acknowledges why model scale improves CoT remains 'certainly multi-faceted' and the preliminary error analysis is done on only 45 examples."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "relevance": "Foundational few-shot prompting paper (GPT-3); establishes the standard prompting paradigm CoT extends and provides primary baseline; introduces GPT-3 models used in evaluation"
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems",
       "relevance": "Introduces GSM8K benchmark and finetuned verifier approach; primary prior SOTA that CoT surpasses; establishes the evaluation setting for the headline result"
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "relevance": "Companion paper providing theoretical framework for understanding CoT as an emergent ability of scale; CoT prompting contributes a key case study to this work"
     },
     {
       "title": "Show Your Work: Scratchpads for Intermediate Computation with Language Models",
       "relevance": "Closest prior work using intermediate steps via finetuning; CoT demonstrates similar gains with prompting alone, without gradient updates"
     },
     {
       "title": "Program Induction by Rationale Generation: Learning to Solve and Explain Algebraic Word Problems",
       "relevance": "Pioneering natural language rationale approach for math via training from scratch; CoT achieves comparable results with few-shot prompting only"
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "relevance": "Direct follow-up work improving CoT via majority voting over sampled generations; cited by this paper as showing CoT can be further enhanced"
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "relevance": "Establishes scaling law context; CoT findings on emergent ability complicate simple scaling predictions by showing prompting technique matters beyond raw model size"
     },
     {
       "title": "Scaling Language Models: Methods, Analysis & Insights from Training Gopher",
       "relevance": "Shows scaling alone is insufficient for arithmetic and reasoning tasks, directly motivating the need for CoT prompting as an additional intervention"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "CoT requires only prompt modification with no finetuning or model access; immediately applicable by any practitioner with API access to a large LLM using the exact prompts in Appendix G."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The emergent ability finding—that CoT actually hurts small models—challenged the assumption that prompting improvements scale smoothly, surprising the field at publication."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper acknowledges factual incorrectness in generated chains but frames this as a limitation rather than a safety concern."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild competitive angle between Google PaLM and OpenAI GPT-3/Codex with PaLM claiming SOTA, but no explicit controversy or disagreement with prior work."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Practitioners can immediately reproduce CoT with GPT-3 API using the exact 8-exemplar prompts provided in Appendix G; no additional resources or infrastructure required."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Google Brain team at NeurIPS 2022; became one of the most cited papers in the LLM prompting literature and underpins most subsequent work on reasoning with LLMs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42711991",
         "title": "Show HN: QwQ-32B APIs – o1 like reasoning at 1% the cost",
         "points": 17,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=42711991",
         "created_at": "2025-01-15T15:29:12Z"
       },
       {
         "hn_id": "30988904",
         "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=30988904",
         "created_at": "2022-04-11T14:08:01Z"
       },
       {
         "hn_id": "30112147",
         "title": "Plume: Differential Privacy at Scale",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30112147",
         "created_at": "2022-01-28T08:34:48Z"
       },
       {
         "hn_id": "34053182",
         "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=34053182",
         "created_at": "2022-12-19T15:29:09Z"
       }
     ],
     "top_points": 17,
     "total_points": 22,
     "total_comments": 4
   }
 }