ai-research-survey

scan.json (25533B)

---

 {
   "paper": {
     "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
     "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed H. Chi", "Sharan Narang", "Aakanksha Chowdhery", "Denny Zhou"],
     "year": 2022,
     "venue": "International Conference on Learning Representations (ICLR 2023)",
     "arxiv_id": "2203.11171"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Self-consistency, a decoding strategy that samples diverse reasoning paths and takes majority vote, significantly improves chain-of-thought prompting across arithmetic and commonsense reasoning benchmarks. Gains are striking: +17.9% on GSM8K, +11.0% on SVAMP, +12.2% on AQuA with PaLM-540B or GPT-3. The method requires no training, fine-tuning, or additional annotation — it is purely a test-time technique. Self-consistency also helps when chain-of-thought hurts performance compared to standard prompting, and the consistency level correlates with accuracy, providing an uncertainty estimate.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or link is provided in the paper. The reproducibility statement mentions model checkpoints and APIs but no code for the self-consistency method itself."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used are publicly available (GSM8K, SVAMP, AQuA, CommonsenseQA, ARC, etc.). The paper also provides full prompts in Appendix A.3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions TPU configurations (e.g., TPU v3 2x2 for UL2, TPU v4 4x4x12 for PaLM) in Appendix A.2 but does not provide software environment details, library versions, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The reproducibility statement points to model checkpoints and APIs but does not provide scripts or a README for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across 10 runs. Figure 2 and Figures 7-8 show error bars. The paper notes 'standard deviation of self-consistency is ≤0.5 for all tasks' in Table 2 footnote."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims of 'X outperforms Y' are based on comparing accuracy numbers without any hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute accuracy improvements are reported with baseline context throughout, e.g., '+17.9%' on GSM8K (from 56.5% to 74.4%). Tables 2 and 3 show both baseline and improved numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for using 40 sampled paths or 10 runs. These choices appear arbitrary with no power analysis or justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are averaged over 10 runs with standard deviations reported. The paper states this explicitly in Section 3.2 and shows variance in figures and tables (e.g., Tables 6, 7)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: greedy CoT decoding (Wei et al., 2022), sample-and-rank, beam search, prompt-order ensembles, multi-prompt ensembles, model ensembles, and previous SoTA methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include the then-contemporary CoT prompting (Wei et al., 2022) and prior SoTA results. The comparison methods (sample-and-rank, beam search, ensembles) are standard approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 ablates different aggregation strategies (weighted avg, weighted sum, majority vote). Section 3.5 ablates sampling strategies, scaling, imperfect prompts, equation prompts, and zero-shot CoT."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is used as the evaluation metric across all tasks (with EM/F1 for HotpotQA in Table 5). No other metrics like calibration or per-example analysis are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant — the tasks have fixed ground-truth answers and automated evaluation is the appropriate method."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in footnote 3: 'By default we use the test split for all datasets if the labels are available for evaluation. For CommonsenseQA we use the dev split; for StrategyQA we use the question-only set from BIG-bench.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (Tables 2, 3), per model (4 models), per aggregation strategy (Table 1), and per number of sampled paths (Figure 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4, 12, and 13 show specific examples including incorrect reasoning paths. The paper discusses that 'language models can sometimes generate incorrect or nonsensical reasoning paths' (Section 5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows that weighted average aggregation performs much worse. Table 6 shows beam search underperforms sampling. The paper notes self-consistency gains are 'relatively lower for smaller models' (Section 3.5). Symbolic reasoning shows minimal gains for small models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims specific accuracy gains (+17.9% GSM8K, +11.0% SVAMP, etc.) which are all supported by Table 2 and Table 3 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily about self-consistency improving performance. The ablation design (controlled comparisons changing only the decoding strategy while keeping prompts/models fixed) adequately supports these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across 4 models, 12+ benchmarks, and multiple reasoning types. Section 5 notes the limitation that 'self-consistency can be applied only to problems where the final answer is from a fixed answer set.' The scope is reasonably bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against alternative explanations for the improvement: sample-and-rank (Section 3.4), beam search diversity, model ensembles, and prompt ensembles. Table 1 analyzes whether the gain comes from probability weighting vs. majority vote."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on specific benchmarks and claims improvements on those specific benchmarks. It does not overclaim — it says 'reasoning accuracy' in the context of the tested benchmarks, not 'general reasoning ability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are named: UL2-20B, GPT-3 code-davinci-001 and code-davinci-002, LaMDA-137B, PaLM-540B. The GPT-3 engines are specific enough to reproduce via the API."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix A.3 (Tables 14-21) for all tasks, including the exact chain-of-thought exemplars used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports sampling parameters: T=0.5/k=40 for UL2 and LaMDA, T=0.7/k=40 for PaLM, T=0.7 without top-k for GPT-3. Appendix A.2 reports max tokens (128) and no frequency/presence penalty for GPT-3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Self-consistency is a pure decoding strategy applied to language model outputs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how answers are parsed: 'we parse the first numerical part as the final answer after the model generates \"The answer is\"' (footnote 1). Dataset splits are documented in footnote 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Conclusion and Discussion) contains substantive discussion of limitations including computational cost, applicability to fixed-answer-set problems only, and nonsensical reasoning paths."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific limitations: higher computation cost, restriction to fixed answer sets, models generating incorrect/nonsensical reasoning paths (with specific examples in Table 4). These are specific to this work."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'self-consistency can be applied only to problems where the final answer is from a fixed answer set' and notes it cannot be directly applied to open-text generation without a consistency metric."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw model outputs (the sampled reasoning paths and answers) are not released. Only aggregate accuracy numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: all benchmarks are public with specific splits documented (footnote 3). The sampling procedure is detailed in Section 3.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: prompt model with CoT exemplars → sample 40 outputs → parse answers → aggregate via majority vote. Answer parsing is described in footnote 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. All authors are from Google Research, Brain Team, but no funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Google Research, Brain Team, with email addresses at google.com clearly shown on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google employees evaluate Google's own models (LaMDA-137B, PaLM-540B) alongside others. Google has a financial interest in demonstrating the capabilities of its language models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models used. This is relevant as some benchmarks (e.g., AddSub from 2014, MultiArith from 2015) likely appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap despite using public benchmarks with models that could have seen them during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (AddSub 2014, MultiArith 2015, CommonsenseQA 2019) were published well before these models' training. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges the computational cost limitation ('self-consistency incurs more computation cost') but does not quantify the cost per example or total API spend."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 states hardware used (TPU v3/v4 configurations) and approximate inference times: '1 to 4 hours' per task on UL2/LaMDA, '2 to 12 hours' on PaLM, with some tasks up to 2 days."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are averaged over 10 runs with standard deviations reported. Figures 2, 7, 8 show error bars across runs. The paper explicitly states 'averaged over 10 runs' (Section 3.2)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2: 'We report the results of self-consistency averaged over 10 runs, where we sampled 40 outputs independently from the decoder in each run.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The sampling parameters (temperature, top-k) appear chosen without reporting how they were selected. Section 3.5 ablates these choices but doesn't report the search budget that led to the default settings."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 and Figure 4 show that self-consistency is robust across sampling strategies and parameters. Table 1 justifies the choice of majority vote over other aggregation strategies by comparing all options."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite many comparisons across tasks, models, and configurations."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge any bias from evaluating their own proposed method. However, the method is simple enough (majority vote over samples) that implementation bias is less of a concern."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 plots accuracy as a function of the number of sampled reasoning paths (1, 5, 10, 20, 40), directly showing the compute-performance tradeoff. Section 5 discusses this tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the reasoning benchmarks actually measure reasoning ability vs. pattern matching or memorization. The paper takes benchmark validity at face value."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Self-consistency is a decoding strategy, not a scaffolded system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' training data includes the benchmark problems. Many benchmarks predate the models' training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The few-shot exemplars could prime the model differently for different problems, but this is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and test benchmarks share structural similarities or overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Self-consistency improves chain-of-thought prompting by +17.9% on GSM8K, +11.0% on SVAMP, +12.2% on AQuA, +6.4% on StrategyQA, and +3.9% on ARC-challenge.",
       "evidence": "Tables 2 and 3 show these gains with PaLM-540B or GPT-3 code-davinci-002. Results averaged over 10 runs with std dev ≤0.5.",
       "supported": "strong"
     },
     {
       "claim": "Self-consistency achieves new state-of-the-art results on arithmetic reasoning benchmarks without any task-specific training.",
       "evidence": "Table 2 compares against previous SoTA including fine-tuned models (GPT-3 175B finetuned + verifier). Self-consistency with PaLM-540B or GPT-3 surpasses these on most tasks.",
       "supported": "strong"
     },
     {
       "claim": "Self-consistency significantly outperforms sample-and-rank, beam search, and ensemble-based approaches.",
       "evidence": "Figure 3 compares to sample-and-rank, Table 6 to beam search, Table 7 to prompt-order and multi-prompt ensembles, Table 10 to model ensembles. Self-consistency dominates in all comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Self-consistency helps recover performance when chain-of-thought hurts compared to standard prompting.",
       "evidence": "Table 5 shows that on ANLI-R1, e-SNLI, and RTE, CoT hurts but self-consistency recovers and outperforms standard prompting.",
       "supported": "strong"
     },
     {
       "claim": "Majority vote (unweighted) performs comparably to normalized weighted sum for answer aggregation.",
       "evidence": "Table 1 shows majority vote at 74.4% vs. normalized weighted sum at 74.1% on GSM8K, with similar results across tasks. The paper explains this is because normalized probabilities are close to each other.",
       "supported": "strong"
     },
     {
       "claim": "Self-consistency is robust to sampling strategies and parameters.",
       "evidence": "Figure 4 (left) and Figure 6 show results across various temperature and top-k settings, with consistent improvements regardless of sampling parameters.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own models",
       "detail": "All authors are Google Research employees evaluating Google's LaMDA-137B and PaLM-540B models. While the method is also tested on public models (UL2, GPT-3), the largest gains are showcased on Google's proprietary models."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Many benchmarks used (AddSub 2014, MultiArith 2015, ASDiv 2020, CommonsenseQA 2019) were publicly available well before the models' training. No training cutoff dates or contamination analysis is provided for any model."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite many comparative claims across 12+ benchmarks and 4 models, no statistical tests are performed. Claims of superiority rely solely on comparing accuracy numbers."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "The foundational work that self-consistency builds upon; introduces chain-of-thought prompting for LLM reasoning."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen", "Heewoo Jun", "Lukasz Kaiser", "Matthias Plappert", "Jerry Tworek", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman"],
       "year": 2021,
       "relevance": "Introduces GSM8K benchmark and the verifier approach that self-consistency outperforms without requiring training."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot prompting paradigm; one of the four models evaluated."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Introduces PaLM-540B, the largest model evaluated; key to understanding scaling effects of self-consistency."
     },
     {
       "title": "Large language models are zero-shot reasoners",
       "authors": ["Takeshi Kojima", "Shixiang Shane Gu", "Machel Reid", "Yutaka Matsuo", "Yusuke Iwasawa"],
       "year": 2022,
       "relevance": "Introduces zero-shot chain-of-thought prompting; self-consistency is shown to work with this approach as well."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Codex paper; code-davinci-001/002 engines used as two of the four model families evaluated."
     },
     {
       "title": "The unreliability of explanations in few-shot prompting for textual reasoning",
       "authors": ["Xi Ye", "Greg Durrett"],
       "year": 2022,
       "relevance": "Shows chain-of-thought can hurt performance on some tasks; self-consistency addresses this limitation."
     },
     {
       "title": "Scaling language models: Methods, analysis & insights from training Gopher",
       "authors": ["Jack W Rae"],
       "year": 2021,
       "relevance": "Discusses limitations of scaling for reasoning; provides context for why prompting strategies like self-consistency are needed."
     },
     {
       "title": "Beyond the imitation game: Measuring and extrapolating the capabilities of language models",
       "authors": ["BIG-bench collaboration"],
       "year": 2021,
       "relevance": "Major LLM benchmark suite; StrategyQA evaluation set sourced from BIG-bench."
     },
     {
       "title": "LaMDA: Language models for dialog applications",
       "authors": ["Romal Thoppilan"],
       "year": 2022,
       "arxiv_id": "2201.08239",
       "relevance": "Introduces LaMDA-137B, one of the four models evaluated; also describes the re-ranker approach that self-consistency outperforms."
     }
   ]
 }