ai-research-survey

scan.json (24332B)

---

 {
   "paper": {
     "title": "Large Language Models are Better Reasoners with Self-Verification",
     "authors": ["Yixuan Weng", "Minjun Zhu", "Fei Xia", "Bin Li", "Shizhu He", "Shengping Liu", "Bin Sun", "Kang Liu", "Jun Zhao"],
     "year": 2022,
     "venue": "Findings of EMNLP 2023",
     "arxiv_id": "2212.09561",
     "doi": "10.18653/v1/2023.findings-emnlp.167"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Self-verification improves LLM reasoning by using backward verification to validate candidate answers generated via chain-of-thought prompting. The method achieves consistent improvements across 8 reasoning datasets (e.g., +4.33% on GSM8K with Instruct-GPT), combines additively with self-consistency and PAL, and shows self-verification is an emergent ability of larger models. Condition Mask Verification outperforms True-False Item Verification on arithmetic tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in abstract: https://github.com/WENGSYX/Self-Verification."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 8 datasets used are publicly available standard benchmarks (GSM8K, SingleEq, AddSub, MultiArith, AQUA-RAT, SVAMP, CSQA, Date Understanding). URLs provided in Appendix A.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided. The paper describes the method and hyperparameters but lacks a reproducibility guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 are point estimates only. No confidence intervals or error bars reported despite running each experiment three times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests performed. Claims of improvement are based solely on comparing raw accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports absolute improvements in parentheses with baseline context (e.g., '60.81 → 65.14 (+4.33)'), providing both magnitude and baseline reference."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of K=5 candidate answers or P=10 verification rounds beyond 'we recommend choosing an appropriate value for P (e.g. P=10).' No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Paper states 'we ran each experiment three times and calculated the average result' (Section 4.4) but does not report standard deviation or any spread measure across those runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against CoT baseline, self-consistency (Wang et al., 2023c), PAL (Gao et al., 2023), and previous fine-tuned SOTA results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-consistency (2023) and PAL (2023) were contemporary methods at time of submission. Previous SOTA fine-tuned results also included."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation-style analyses: single vs. multiple condition masks (Figure 5), CMV vs. TFV (Figure 6), varying P values (Figure 7), varying number of few-shot prompts (Figure 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (problem solve rate) is reported as the evaluation metric. No other metrics used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for measuring accuracy on reasoning benchmarks with ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard test splits of established benchmarks (GSM8K test set of 1319 examples, etc., as shown in Table 6)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down across 8 individual datasets spanning 3 reasoning task types (arithmetic, commonsense, logical). Table 2 provides further per-dataset analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 in the appendix shows specific failure examples for each dataset with incorrect self-verification outputs marked with [✗] and ground truth answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3b shows self-verification has negative impact on smaller models (0.4B and 1.3B). Paper acknowledges 'it is challenging to augment the reasoning performance of smaller language models' in Limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of improvement on arithmetic, commonsense, and logical reasoning datasets are supported by Table 1 results. The specific numbers cited (60.8→65.1 on GSM8K, 91.01→93.40 on SingleEq) match the table."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about self-verification improving performance. The ablation studies (Figures 5-7) use controlled single-variable manipulation, which is adequate for the claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title claims LLMs 'are Better Reasoners with Self-Verification' but experiments only use GPT-3 variants (code-davinci-001, code-davinci-002, text-ada/babbage/curie/davinci). No testing on other LLM families."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, self-verification uses additional compute (multiple decoding passes), but no comparison against simply generating more candidate answers with the same compute budget."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Paper measures accuracy on reasoning benchmarks and claims improvements on reasoning benchmarks — no proxy gap. Claims match measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers provided: code-davinci-001, code-davinci-002, text-ada-001, text-babbage-001, text-curie-001, text-davinci-002 (Section 4.2, Figure 3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided in Appendix A.5-A.7 (Tables 8-17) for both forward reasoning and backward verification across all task types."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4: K=5 candidate answers, P=10 verification iterations, max token length 168, sampling decoding without top-k truncation. Temperature not explicitly stated but sampling decoding specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used. The method is a prompting strategy with two-stage inference (forward reasoning + backward verification), not an agent scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Answer cleansing strategy described in detail in Appendix A.1 with pseudo code (Table 3) for each answer format (number, multiple choice, true/false, yes/no, free format)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Limitations' section present after the Conclusion, discussing dependency on LLMs, prompt bias, model size constraints, and computational cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section discusses specific threats: prompts are 'artificially constructed and may introduce bias', effectiveness depends on 'accurate answers within the candidate conclusions', and smaller models cannot benefit (citing Figure 3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Paper states method 'is not suitable for evaluating the LLM's inference procedure' (only conclusions), acknowledges it 'is challenging to augment the reasoning performance of smaller language models', and notes arithmetic tasks benefit more than general tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (model predictions, verification scores) are released. Only aggregated accuracy numbers reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are standard public benchmarks. Dataset details including size, average word count, answer format, and license listed in Table 6 with URLs in Appendix A.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline from input to output is documented: CoT prompting → sampling K=5 candidates → rewriting conclusions → condition masking/TFV → scoring → selection. Answer cleansing in Appendix A.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists National Key R&D Program of China, NSFC grants, Strategic Priority Research Program of CAS, Youth Innovation Promotion Association CAS, and OPPO Research Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations listed: CAS, University of Chinese Academy of Sciences, Hunan University, Unisound, Shanghai AI Laboratory."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government research agencies (NSFC, CAS) and OPPO Research Fund. None have a direct stake in whether self-verification improves GPT-3 reasoning."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement provided. One author is affiliated with Unisound (a commercial company) but no declaration of financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of training data cutoff dates for any of the GPT-3 models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark datasets (some dating to 2014-2021) appeared in GPT-3's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks (GSM8K 2021, MultiArith 2016, etc.) were available online before GPT-3's training. No contamination analysis performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or dollar amounts reported. Figure 7 shows performance vs. number of verification iterations P but not in terms of cost. The method requires K×P additional API calls per example but cost is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The Reproducibility Statement (A.3) gives experiment dates but not compute costs or API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Paper states experiments were run 3 times and averaged but does not report variance across runs or seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4: 'we ran each experiment three times and calculated the average result.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how K=5 and P=10 were selected. Figure 7 shows P sensitivity but no systematic search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows performance across different P values and the paper recommends P=10 based on the performance plateau, showing all configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests performed at all, let alone corrections for multiple comparisons across 8 datasets × multiple methods."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors compare their self-verification method against their own implementations of baselines without acknowledging potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 explicitly shows performance as a function of computational resource (number of P verification iterations) on GSM8K. Limitations section acknowledges increased computational cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 8 benchmarks adequately measure 'reasoning ability' as claimed. Benchmarks are used without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. The method is a prompting strategy, not a scaffold comparison."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several benchmarks (MultiArith 2016, SingleEq 2015, AddSub 2014) predate GPT-3's training by years."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether few-shot prompts or evaluation setup leaks information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and test benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Self-verification improves CoT reasoning on Instruct-GPT by an average of 2.33% across 8 datasets",
       "evidence": "Table 1 shows improvements ranging from +0.41% (CSQA) to +4.33% (GSM8K) with code-davinci-002.",
       "supported": "moderate"
     },
     {
       "claim": "Self-verification can be combined with self-consistency and PAL for further gains",
       "evidence": "Table 1 bottom section shows SC+SV and PAL+SV consistently improve over SC and PAL alone across datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Self-verification is an emergent ability that improves with model scale",
       "evidence": "Figure 3 shows self-verification hurts or has negligible effect on 0.4B-1.3B models but helps 7B and 175B models.",
       "supported": "moderate"
     },
     {
       "claim": "Condition Mask Verification outperforms True-False Item Verification on arithmetic tasks",
       "evidence": "Figure 6 compares CMV vs TFV across 6 arithmetic datasets. CMV is generally better on Instruct-GPT, though TFV sometimes matches or beats CMV on GPT-3.",
       "supported": "moderate"
     },
     {
       "claim": "Multiple condition masks produce better verification scores than single condition masks",
       "evidence": "Figure 5 shows multi-condition generally outperforms single-condition across 6 datasets, though single-condition sometimes underperforms CoT baseline.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All improvements are reported as raw accuracy differences without any statistical tests. Many improvements are small (e.g., +0.06% to +0.46% for SC+SV on Instruct-GPT) and may not be statistically significant."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "Paper states experiments were averaged over 3 runs but never reports standard deviation, making it impossible to assess whether improvements exceed run-to-run variance."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Several benchmarks (MultiArith 2016, SingleEq 2015, AddSub 2014) were publicly available years before GPT-3 training. No contamination analysis performed."
     },
     {
       "flag": "Compute fairness not addressed",
       "detail": "Self-verification uses K×(R×P+1) API calls per example (K=5 candidates, R conditions, P=10 repeats) but is compared against single-pass CoT. The compute-matched comparison is missing."
     },
     {
       "flag": "Limited model diversity",
       "detail": "All experiments use only OpenAI GPT-3 variants. The title claims 'Large Language Models' broadly but no other model families were tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational CoT prompting method that this paper builds upon and extends with backward verification."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc V Le", "Ed H. Chi"],
       "year": 2023,
       "relevance": "Key baseline and complementary method; self-verification is shown to stack with self-consistency decoding."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Proposes trained verifiers for math reasoning; this paper's self-verification aims to achieve similar without training."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yura Burda", "Harrison Edwards", "Bowen Baker", "Teddy Lee", "Jan Leike", "John Schulman", "Ilya Sutskever", "Karl Cobbe"],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Process-level verification for math reasoning using trained reward models; contrasts with this paper's training-free approach."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": ["Luyu Gao", "Aman Madaan", "Shuyan Zhou", "Uri Alon", "Pengfei Liu", "Yiming Yang", "Jamie Callan", "Graham Neubig"],
       "year": 2023,
       "relevance": "Alternative forward reasoning method using programs; shown to be complementary with self-verification."
     },
     {
       "title": "Large language models are zero-shot reasoners",
       "authors": ["Takeshi Kojima", "Shixiang Shane Gu", "Machel Reid", "Yutaka Matsuo", "Yusuke Iwasawa"],
       "year": 2022,
       "relevance": "Zero-shot CoT prompting demonstrating LLM reasoning capabilities without few-shot examples."
     },
     {
       "title": "Chain-of-verification reduces hallucination in large language models",
       "authors": ["Shehzaad Dhuliawala", "Mojtaba Komeili", "Jing Xu", "Roberta Raileanu", "Xian Li", "Asli Celikyilmaz", "Jason Weston"],
       "year": 2023,
       "arxiv_id": "2309.11495",
       "relevance": "Related verification approach for reducing LLM hallucinations through structured verification chains."
     },
     {
       "title": "Least-to-most prompting enables complex reasoning in large language models",
       "authors": ["Denny Zhou", "Nathanael Schärli", "Le Hou", "Jason Wei", "Nathan Scales", "Xuezhi Wang", "Dale Schuurmans", "Claire Cui", "Olivier Bousquet", "Quoc Le", "Ed Chi"],
       "year": 2023,
       "relevance": "Decomposition-based prompting strategy for complex reasoning; mentioned as complementary to self-verification."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot in-context learning capabilities that this work builds upon."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper; the code-davinci models used in this paper's experiments are based on this work."
     }
   ]
 }