ai-research-survey

scan.json (31245B)

---

 {
   "paper": {
     "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
     "authors": [
       "Jason Wei",
       "Xuezhi Wang",
       "Dale Schuurmans",
       "Maarten Bosma",
       "Brian Ichter",
       "Fei Xia",
       "Ed H. Chi",
       "Quoc V. Le",
       "Denny Zhou"
     ],
     "year": 2022,
     "venue": "NeurIPS 2022",
     "arxiv_id": "2201.11903"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. The authors provide inputs/outputs/targets for LaMDA and GPT-3 in supplementary material but do not release evaluation code or scripts."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (GSM8K, SVAMP, ASDiv, AQuA, MAWPS, CSQA, StrategyQA, BIG-bench tasks). The coinflip and last letter concatenation datasets are new and are provided in supplementary material (Appendix E.3). LaMDA inputs/outputs/targets are also provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E.2 states TPU v3 (8x8) for LaMDA 137B and TPU v4 (4x4x12) for PaLM 540B inference, and GPT-3 via public API. However, no software environment details (library versions, dependencies) are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While full prompts are given in Appendix G and model outputs are in supplementary material, there are no scripts or instructions to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported on main results. Standard deviations are only provided for LaMDA 137B ablation/robustness experiments in Tables 6 and 7, but not for the primary results across models and benchmarks in Tables 1-5."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims of improvement are based solely on comparing point estimates (e.g., '56.9% vs 17.9%') without any formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance improvements are reported with baseline context throughout (e.g., Table 1 shows '+39.0' for PaLM 540B on GSM8K going from 17.9% to 56.9%). Absolute and relative gains are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of evaluation examples or the choice of 50 examples for error analysis (Sections 3.2, Appendix D). The evaluation set sizes are inherited from existing benchmarks without discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across five random seeds (different exemplar orderings) are reported for LaMDA 137B in Tables 6 and 7. For other models, single exemplar orders are used to save compute, which is explicitly stated in Section 3.1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard few-shot prompting is used as the primary baseline throughout. Prior supervised state-of-the-art results are also compared against (e.g., finetuned GPT-3 with verifier for GSM8K from Cobbe et al. 2021)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary prior best results from Cobbe et al. (2021), Jie et al. (2022), Pi et al. (2022), and others. These were recent at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 presents a detailed ablation study with three variations: equation only, variable compute only (dots), and chain of thought after answer. Results shown in Figure 5 and Tables 6-7."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only solve rate (accuracy) is used as the evaluation metric across all benchmarks. No secondary metrics are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual error analysis of 50 correct and 50 incorrect chain-of-thought outputs from LaMDA 137B on GSM8K (Section 3.2, Appendices D.1-D.2). Also manual analysis of 45 PaLM 62B errors (Appendix A.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard evaluation splits of each benchmark. The few-shot exemplars were manually composed or drawn from training sets, while results are on evaluation/test splits. Section 3.1 notes 'most of the datasets only have an evaluation split.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by individual benchmark (Tables 1-5), by model size (Figures 4, 7, 8), and by MAWPS subsets (Table 3: SingleOp, SingleEq, AddSub, MultiArith). In-domain vs OOD results provided for symbolic reasoning."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 provides detailed error analysis categorizing 50 incorrect outputs into calculator errors (8%), symbol mapping errors (16%), one-step missing (22%), and semantic understanding/incoherence errors (54%). Specific examples are shown in Tables 10-11."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that chain-of-thought prompting hurts performance for small models (<10B parameters), shown in Figure 4 and Table 2. Also reports minimal gains on CSQA (Section 4) and easy MAWPS subsets (Section 3.2, Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CoT improves reasoning, emerges in sufficiently large models, and achieves SOTA on GSM8K. All are supported: Figure 4 shows emergence, Table 1 shows GSM8K SOTA (56.9% vs 55% prior best with PaLM 540B)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('chain-of-thought prompting improves performance') and supports them with controlled ablation studies (Section 3.3): equation only, variable compute only, and reasoning after answer. These ablations isolate specific factors, supporting the causal claim that the content of the chain of thought matters."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly scopes claims to specific tasks (arithmetic, commonsense, symbolic reasoning), specific models (GPT-3, LaMDA, PaLM), and specific model scales (≥100B). Section 6 explicitly discusses when CoT helps (Appendix A.3 gives three conditions) and states it only helps 'sufficiently large' models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Section 3.3) tests three alternative explanations: (1) equations alone suffice, (2) more computation is the key factor, (3) CoT just activates knowledge. All are refuted with evidence. Section 6 also acknowledges models may not be 'actually reasoning.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently measures solve rate/accuracy on specific benchmarks and does not overclaim beyond these measurements. Claims are tied to specific datasets (e.g., 'on the GSM8K benchmark') rather than broad ungrounded claims about 'reasoning ability' in general."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 specifies exact model versions: GPT-3 text-ada-001, text-babbage-001, text-curie-001, text-davinci-002 with presumed parameter counts. LaMDA model sizes (422M to 137B), PaLM sizes (8B, 62B, 540B), UL2 20B, and Codex code-davinci-002 are specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all tasks is provided in Appendix G (Tables 20-28). This includes all few-shot exemplars with chain-of-thought annotations. Alternate annotator prompts are in Appendix H (Tables 29-30)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'We sample from the models via greedy decoding.' For LaMDA, 'averaged results over five random seeds' with different exemplar orderings. Input context window limited to 1024 tokens (Appendix D.3)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is simple few-shot prompting with no tool use, retry logic, or agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The exemplar selection process is documented: 8 manually composed exemplars for most benchmarks (Section 3.1), 4 from AQuA training set, and specific criteria for GSM8K exemplars (≤60 tokens, ≤2 steps, Section 3.4). Symbolic reasoning data generation is described in Section 5."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) contains a dedicated paragraph on limitations, covering: (1) CoT doesn't prove reasoning, (2) annotation costs for finetuning, (3) no guarantee of correct reasoning paths, (4) only works at large scale making it costly to serve."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: generated chains of thought are not always factual (Section 6, Appendix D.1), incorrect reasoning can accidentally lead to correct answers especially for classification tasks (Appendix D.1), and prompt sensitivity affects results (Section 3.4, Appendix A.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 explicitly states when CoT helps and when it doesn't: three conditions must be met (challenging task, large model, flat scaling curve). Section 6 states CoT emergence 'only at large model scales makes it costly to serve.' The paper scopes to arithmetic, commonsense, and symbolic reasoning tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.1 states: 'we make exact inputs, targets, and predictions for LaMDA 137B for each task available as a zip file in the supplementary material.' GPT-3 results are reproducible via the public API."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks are publicly available with citations. The synthetic symbolic reasoning datasets are described in Section 5 with generation procedures (random concatenation from top-1000 names from namecensus.com). Chain-of-thought annotations are fully provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates language models on standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is straightforward: benchmark datasets → prompt construction with exemplars → model inference via greedy decoding → answer extraction → accuracy comparison. External calculator variant is also described (Python eval function, Table 1). The process for creating chain-of-thought annotations is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The paper is from Google Research, Brain Team, but no specific grant numbers or funding acknowledgments are provided beyond naming individual colleagues who gave feedback."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are identified as Google Research, Brain Team with email addresses {jasonwei,dennyzhou}@google.com on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors work at Google, which develops and markets the LaMDA and PaLM models being evaluated. Google has a commercial interest in demonstrating that large language models (especially their own) can perform better with prompting techniques."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Given all authors work at Google and evaluate Google's proprietary models (LaMDA, PaLM), financial interests are plausible but undisclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five model families evaluated (GPT-3, LaMDA, PaLM, UL2, Codex). The reader cannot assess whether benchmark data appeared in training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation benchmark examples could have appeared in the training data of any model. This is a significant omission given the use of publicly available benchmarks with large-scale models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (GSM8K 2021, CSQA 2019, SVAMP 2021, MAWPS 2016) were publicly available before model training. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study of language models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, or token consumption are reported for any experiment despite using multiple large models (up to PaLM 540B). Section 6 mentions large scale 'makes it costly to serve' but provides no quantification."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E.2 describes hardware used (TPU v3 for LaMDA, TPU v4 for PaLM) but explicitly states 'we did not estimate the total amount of compute.' No GPU hours, total API spend, or wall-clock time are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "For LaMDA, results are averaged over five random seeds (different exemplar orderings) with standard deviations reported in Tables 6-7. However, for GPT-3 and PaLM, single exemplar orderings are used, justified by LaMDA's low variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states: 'For LaMDA, we report averaged results over five random seeds.' For other models, 'to save compute we report results for a single exemplar order.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The paper uses greedy decoding and manually composed prompts. Section 3.1 notes 'These particular exemplars did not undergo prompt engineering' but no search budget is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a single set of 8 exemplars across most benchmarks (Section 3.1), and Section 3.4 demonstrates robustness across different annotators, exemplar sets, and orderings. No cherry-picking of configurations is apparent."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable. The paper reports only point estimates and standard deviations."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not discuss the bias of evaluating their own prompting technique. While they compare against prior work, they do not acknowledge that their implementations of baselines could be disadvantaged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute. CoT prompting generates more tokens (longer outputs) than standard prompting, increasing compute cost, but this trade-off is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmarks actually measure 'reasoning ability' as claimed, or whether solve rate on these tasks is a valid proxy for reasoning. The connection between benchmark performance and the claimed cognitive ability is assumed rather than argued."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The approach is direct few-shot prompting without any agentic framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several benchmarks (MAWPS 2016, CSQA 2019) were published years before model training and could be in training data. No temporal analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The few-shot exemplars provide the reasoning structure that the model then imitates, but whether this constitutes a form of answer leakage for the test examples is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data are independent. Given the models were trained on large web corpora that could include these benchmarks, independence is not verified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference, decontamination, or temporal splits are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Chain-of-thought prompting significantly improves the ability of large language models to perform complex reasoning tasks.",
       "evidence": "Demonstrated across arithmetic (5 benchmarks), commonsense (5 benchmarks), and symbolic reasoning (2 tasks) with three model families. PaLM 540B on GSM8K: 17.9% → 56.9% (Section 3.2, Table 1, Figure 4).",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought reasoning is an emergent ability of model scale, not positively impacting small models.",
       "evidence": "Performance gains only appear at ~100B parameters across LaMDA, GPT-3, and PaLM (Figure 4, Tables 2-5). Small models produce 'fluent but illogical chains of thought' leading to lower performance than standard prompting (Section 3.2).",
       "supported": "strong"
     },
     {
       "claim": "PaLM 540B with chain-of-thought prompting achieves state-of-the-art on GSM8K, surpassing finetuned GPT-3 with a verifier.",
       "evidence": "PaLM 540B CoT achieves 56.9% on GSM8K vs prior best of 55% from Cobbe et al. (2021) finetuned GPT-3 with verifier (Figure 2, Table 1).",
       "supported": "moderate"
     },
     {
       "claim": "The benefit of chain-of-thought prompting comes from the semantic reasoning content, not just additional computation or knowledge activation.",
       "evidence": "Ablation study (Section 3.3, Figure 5): variable compute only (dots) performs same as baseline; reasoning after answer performs same as baseline; equation only helps partially but not as much as full CoT.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought prompting is robust across different annotators, exemplar sets, and exemplar orderings.",
       "evidence": "Three annotators, three GSM8K-sampled exemplar sets, varying numbers of exemplars, and multiple orderings all outperform standard prompting (Section 3.4, Figure 6, Tables 6-7, Figure 11).",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought prompting facilitates length generalization to longer sequences in symbolic reasoning.",
       "evidence": "Models trained on 2-step exemplars generalize to 3- and 4-step problems for last letter concatenation and coin flip tasks (Section 5, Figure 8, Table 5). Performance lower than in-domain but scales upward with model size.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Chain-of-thought prompting—providing intermediate reasoning steps in few-shot exemplars—dramatically improves large language model performance on arithmetic, commonsense, and symbolic reasoning tasks, but only at model scales of ~100B+ parameters. PaLM 540B with CoT achieved state-of-the-art on GSM8K (56.9%) surpassing finetuned approaches. Ablation studies show the benefit comes from the semantic content of reasoning steps, not merely from additional computation or knowledge activation. The approach is robust across different annotators, exemplar sets, and model families.",
   "red_flags": [
     {
       "flag": "Company evaluating own models",
       "detail": "All authors are from Google Research, Brain Team. Two of the five model families evaluated (LaMDA and PaLM) are Google proprietary models. The paper demonstrates that Google's largest models benefit most from the technique."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Multiple benchmarks used (MAWPS 2016, CSQA 2019, GSM8K 2021) were publicly available before model training, yet no contamination analysis is performed. Training data cutoffs are not stated for any model."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only solve rate (accuracy) is reported across all experiments. No secondary metrics (e.g., reasoning chain quality, calibration, partial credit) are used to evaluate the approach."
     },
     {
       "flag": "Incomplete variance reporting",
       "detail": "Standard deviations across seeds are only reported for LaMDA 137B. For GPT-3 and PaLM (including the SOTA-claiming PaLM 540B results), only single-run numbers are reported, justified by LaMDA's low variance—but variance may differ across model families."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": [
         "Tom Brown",
         "Benjamin Mann",
         "Nick Ryder"
       ],
       "year": 2020,
       "relevance": "Foundational work on few-shot prompting with GPT-3, the baseline approach that CoT prompting extends."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems",
       "authors": [
         "Karl Cobbe",
         "Vineet Kosaraju",
         "Mohammad Bavarian"
       ],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduced GSM8K benchmark and the finetuned GPT-3 verifier approach that CoT prompting surpasses."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and code evaluation methodology; one of the five model families evaluated with CoT prompting."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": [
         "Jason Wei",
         "Yi Tay",
         "Rishi Bommasani"
       ],
       "year": 2022,
       "relevance": "Provides theoretical framing for the emergence of CoT reasoning at scale, directly tied to this paper's core finding."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": [
         "Xuezhi Wang",
         "Jason Wei",
         "Dale Schuurmans"
       ],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Follow-up work showing majority voting over sampled CoT generations further improves performance."
     },
     {
       "title": "Show Your Work: Scratchpads for Intermediate Computation with Language Models",
       "authors": [
         "Maxwell Nye",
         "Anders Johan Andreassen",
         "Guy Gur-Ari"
       ],
       "year": 2021,
       "arxiv_id": "2112.00114",
       "relevance": "Closest prior work using intermediate computation steps for program execution; CoT generalizes this to natural language."
     },
     {
       "title": "Program Induction by Rationale Generation: Learning to Solve and Explain Algebraic Word Problems",
       "authors": [
         "Wang Ling",
         "Dani Yogatama",
         "Chris Dyer"
       ],
       "year": 2017,
       "relevance": "Pioneered natural language rationales for math problem solving, the training-based predecessor to CoT prompting."
     },
     {
       "title": "Do As I Can, Not As I Say: Grounding Language in Robotic Affordances",
       "authors": [
         "Michael Ahn",
         "Anthony Brohan",
         "Noah Brown"
       ],
       "year": 2022,
       "arxiv_id": "2204.01691",
       "relevance": "SayCan robot planning benchmark used to evaluate CoT prompting for commonsense reasoning in robotic instruction following."
     },
     {
       "title": "Scaling Language Models: Methods, Analysis & Insights from Training Gopher",
       "authors": [
         "Jack W. Rae",
         "Sebastian Borgeaud",
         "Trevor Cai"
       ],
       "year": 2021,
       "arxiv_id": "2112.11446",
       "relevance": "Documented that scaling alone is insufficient for reasoning tasks, motivating CoT prompting as an alternative approach."
     },
     {
       "title": "STaR: Bootstrapping Reasoning with Reasoning",
       "authors": [
         "Eric Zelikman",
         "Yuhuai Wu",
         "Noah D. Goodman"
       ],
       "year": 2022,
       "arxiv_id": "2203.14465",
       "relevance": "Extends CoT idea to self-training: models generate rationales, filter correct ones, and finetune on them."
     },
     {
       "title": "Finetuned Language Models Are Zero-Shot Learners",
       "authors": [
         "Jason Wei",
         "Maarten Bosma",
         "Vincent Y. Zhao"
       ],
       "year": 2022,
       "relevance": "Instruction tuning work (FLAN) that augments inputs with task instructions; CoT takes the orthogonal approach of augmenting outputs."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": [
         "Jacob Austin",
         "Augustus Odena",
         "Maxwell Nye"
       ],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Evaluates LLMs for code generation, related to using intermediate steps in program synthesis."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Chain-of-thought prompting is a directly usable technique that any developer working with LLMs can apply immediately to improve reasoning outputs."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The emergent scaling finding—that CoT hurts small models but dramatically helps 100B+ models—was genuinely surprising and reshaped how the field thinks about prompting."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper focuses on improving reasoning accuracy with no safety, security, or risk angle."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper presents a new technique without challenging specific claims or companies."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The technique is immediately reproducible via the GPT-3 API with the exact prompts provided in the appendix, though it requires access to large-scale models."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "From Google Brain with prominent authors (Jason Wei, Quoc Le, Denny Zhou), evaluating GPT-3 and PaLM—all household names in the AI community."
     }
   }
 }