ai-research-survey

scan.json (19527B)

---

 {
   "paper": {
     "title": "Dancing with Critiques: Enhancing LLM Reasoning with Stepwise Natural Language Self-Critique",
     "authors": ["Yansi Li", "Jiahao Xu", "Tian Liang", "Xingyu Chen", "Zhiwei He", "Qiuzhi Liu", "Rui Wang", "Zhuosheng Zhang", "Zhaopeng Tu", "Haitao Mi", "Dong Yu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.17363"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in abstract: https://github.com/puddingyeah/PANEL"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available benchmarks: AIME (from Art of Problem Solving wiki) and GPQA Diamond (Rein et al., 2024). No proprietary data collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found in the paper. Only model names are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. Code repository is referenced but no README or reproduction guide is included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-2 and Figure 3 report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PANEL 'outperforms' and 'surpasses' baselines based solely on comparing raw accuracy numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context, e.g., 'NL self-critique improves reasoning accuracy over solution-level self-evaluation by 2.2%' (Section 3.2), and absolute numbers are provided in tables allowing comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "AIME uses only 45 problems (30 + 15) and GPQA Diamond has 198 questions. No justification for these sample sizes or discussion of statistical power given the small N."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or results across multiple runs reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: Self-Consistency, Solution-Level Self-Evaluation, Solution-Level Self-Evaluation with NL Self-Critique, and Step-Level Self-Evaluation (Section 3.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: Self-Consistency (Wang et al., 2023b), Step-Level Self-Evaluation (Xie et al., 2024). These are contemporary and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablation provided: PANEL with and without NL Self-Critique in Figure 3, and comparison of self-critique vs. external critique model in Table 2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is reported as the evaluation metric. No other metrics (e.g., computational efficiency comparisons, token usage) are used for evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated reasoning traces or critiques. All evaluation is automated via answer matching. Given the paper claims about 'rich, human-readable critiques,' human evaluation of critique quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "AIME and GPQA Diamond are standard test sets. No tuning on these benchmarks is described; the method is inference-time only."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-domain breakdowns for GPQA (Biology, Chemistry, Physics) and per-year breakdowns for AIME (2024, 2025)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Only success cases are shown (Figures 2 and 9). No analysis of where PANEL fails or produces worse results than baselines."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows that external 70B critique underperforms self-critique (8B) for step-level evaluation on GPQA (32.3% vs 38.9%), which is a somewhat counterintuitive negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that PANEL 'significantly enhances reasoning performance, outperforming traditional scalar reward-based methods' are supported by Tables 1 results showing improvements on both benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('NL self-critique improves reasoning accuracy') are supported by controlled ablation: PANEL with vs. without self-critique (Figure 3), which is a single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim to enhance 'LLM Reasoning' broadly, but results are only on two Llama models and two STEM reasoning benchmarks. No testing on non-STEM reasoning, other model families, or other task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, the additional inference compute from generating critiques is not controlled for — the improvement could be from additional compute rather than critique quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: 'Llama 3.1-8B-Instruct' and 'Llama 3.3-70B-Instruct' (Section 3.1). These are specific enough to identify the exact models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for NL self-critique provided in Appendix A (Figures 5-8) for Math, Physics, Chemistry, and Biology domains."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is stated as 0.6 (footnote 1) and K=5 candidates. However, other hyperparameters like top-p, max tokens, and search depth/budget are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage PANEL framework (Sampling Candidates, NL Self-Critique, Decision Making) is described in detail in Section 2, including the search algorithm and decision-making process."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how AIME problems were formatted for input, how GPQA Diamond questions were processed, or any preprocessing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after Section 5 (Conclusion), discussing theoretical understanding gaps and challenges in quantifying information richness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses only general theoretical gaps ('a deeper theoretical understanding... is still needed') and methodological challenges of comparing information richness. No specific threats like small sample sizes on AIME or compute fairness are mentioned."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not bound its claims to STEM reasoning with Llama models, nor state what was not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs, model responses, or per-problem results are made available. Only aggregate accuracy numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Benchmarks are described: AIME from American Invitational Mathematics Examination (2024 and 2025-Part1), GPQA Diamond from Rein et al. (2024) with 198 questions across three domains."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how benchmark problems are fed into the system, how outputs are parsed and evaluated, or intermediate processing steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Tencent and Shanghai Jiao Tong University. Internship relationships also disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. Authors are from Tencent, which has commercial interest in LLM reasoning capabilities, but this potential conflict is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3.1 and 3.3 models on AIME and GPQA benchmarks but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AIME 2024 or GPQA Diamond problems could have been in the Llama training data. AIME 2025 Part 1 is likely post-training-cutoff, but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPQA Diamond was published in 2024 and could be in Llama 3.3 training data. AIME problems are publicly available. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "PANEL generates critiques for each candidate at each step, significantly increasing inference cost. No cost, latency, or token consumption numbers are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information on total compute used, GPU hours, or hardware specifications."
       }
     }
   },
   "claims": [
     {
       "claim": "PANEL outperforms traditional scalar reward-based methods on AIME and GPQA Diamond benchmarks.",
       "evidence": "Table 1 shows PANEL achieving 24.4% on AIME (vs 17.8-22.2% baselines) and 54.5% on GPQA (vs 51.0-52.5% baselines) with Llama3.3-70B. Section 3.2.",
       "supported": "moderate"
     },
     {
       "claim": "NL self-critique improves reasoning accuracy in both solution-level and step-level self-evaluation methods.",
       "evidence": "Section 3.2: NL self-critique improves solution-level self-evaluation by 2.2% (8B) and 0.5% (70B) on GPQA. Step-level improvement of 6.6% on AIME and 3.0% on GPQA with 70B model.",
       "supported": "moderate"
     },
     {
       "claim": "Self-generated NL critique from the policy model is more effective than external larger model critique for step-level evaluation.",
       "evidence": "Table 2: 8B self-critique achieves 38.9% vs 32.3% for external 70B critique on GPQA in step-level setting.",
       "supported": "weak"
     },
     {
       "claim": "PANEL scales effectively with increased computational resources (pass@k analysis).",
       "evidence": "Figure 3 shows PANEL consistently outperforming baseline across k=1,2,4,8 for both model sizes.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PANEL introduces natural language self-critique as a replacement for scalar process reward models in step-level tree search for LLM reasoning. On AIME and GPQA Diamond benchmarks with Llama 3.1-8B and 3.3-70B, PANEL outperforms self-consistency and self-evaluation baselines. Self-generated critiques from the policy model itself outperform critiques from a larger external model for step-level evaluation, and NL self-critique primarily influences early reasoning steps.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates on small benchmarks (45 AIME problems, 198 GPQA questions) with no error bars, confidence intervals, or multiple-run statistics. Performance differences could be within noise."
     },
     {
       "flag": "Compute fairness not addressed",
       "detail": "PANEL generates additional critique text and decision-making text for each candidate at each step, substantially increasing inference compute. Baselines are not given equivalent additional compute, making the comparison potentially unfair."
     },
     {
       "flag": "Very small benchmark for key claims",
       "detail": "AIME has only 45 problems total (30+15). A single problem is worth ~2.2% accuracy. Performance differences of 2-6% on AIME could represent 1-3 problems, making claims of 'significant improvement' on AIME unreliable."
     },
     {
       "flag": "No contamination analysis",
       "detail": "AIME problems and GPQA Diamond may be in Llama training data. No discussion of potential contamination despite using publicly available benchmarks with models that may have been trained on them."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling test-time compute optimally can be more effective than scaling LLM parameters",
       "authors": ["Charlie Victor Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2025,
       "relevance": "Core related work on inference-time compute scaling, directly relevant to survey scope on LLM reasoning capabilities."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Key baseline method for LLM reasoning via sampling multiple reasoning paths."
     },
     {
       "title": "Self-evaluation guided beam search for reasoning",
       "authors": ["Yuxi Xie", "Kenji Kawaguchi", "Yiran Zhao"],
       "year": 2024,
       "relevance": "Step-level self-evaluation baseline directly compared against in experiments."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2024,
       "relevance": "Foundational work on tree-structured reasoning for LLMs."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yuri Burda"],
       "year": 2024,
       "relevance": "Process reward models for step-level verification, the approach PANEL aims to replace."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2024,
       "relevance": "Related work on using verbal/natural language feedback for LLM self-improvement."
     },
     {
       "title": "CRITIC: Large language models can self-correct with tool-interactive critiquing",
       "authors": ["Zhibin Gou", "Zhihong Shao", "Yeyun Gong"],
       "year": 2024,
       "relevance": "Related work on LLM self-critique and self-correction capabilities."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Contemporary work on enhancing LLM reasoning capabilities via RL."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought reasoning in LLMs."
     },
     {
       "title": "Generative verifiers: Reward modeling as next-token prediction",
       "authors": ["Lunjun Zhang", "Arian Hosseini", "Hritik Bansal"],
       "year": 2024,
       "relevance": "Related approach using generative models as verifiers, relevant to LLM evaluation methodology."
     }
   ]
 }