ai-research-survey

scan.json (25712B)

---

 {
   "paper": {
     "title": "Evaluating LLM Reasoning Beyond Correctness and CoT",
     "authors": ["Soheil Abbasloo"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.18134"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SIEV, a dialectical evaluation framework based on thesis-antithesis-synthesis interactions, reveals substantial hidden reasoning gaps in state-of-the-art LLMs on saturated benchmarks GSM8K and MMLU. Models with near-identical correctness scores (e.g., GPT-5-chat at 96.4% vs O3 at 97.1% on GSM) diverge dramatically at synthesis (56.2% vs 93.6%), suggesting high static accuracy masks fragile reasoning. Cross-model dialectics show models often benefit from external antitheses more than self-generated ones, raising questions about whether LLM reasoning is a general capability or context-sensitive pattern matching.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'the SIEV source code is publicly available at https://github.com/microsoft/siev' in Section 3."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available GSM8K and MMLU benchmarks. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no reproduction guide is described in the text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports ± values for pT and pS (e.g., '97.1±0.1', '93.6±0.7'), indicating uncertainty across runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes extensive comparative claims between models (rankings, performance differences) but uses no statistical significance tests. Differences are assessed by raw score comparison only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports specific score drops with baseline context, e.g., 'GPT-5-chat loses more than 40 points (out of 100) on GSM' and Δ values showing thesis-to-synthesis change throughout Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 21 models were selected, or why the full GSM8K/MMLU datasets (rather than subsets) were used. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviation across runs using ± notation for both pT and pS columns."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The thesis score (pT) serves as the conventional correctness baseline, and SIEV metrics (pS, DS, Δ) are compared against it across all 21 models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Includes very recent models: O3, GPT-5, GPT-5-chat, O4-mini, Kimi-K2, DeepSeek-R1, alongside older models like GPT-3.5 for range."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of SIEV's components. The λ and γ parameters are set (λ=0.7, γ=1) without exploring alternatives. No ablation of the antithesis stage or synthesis stage independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports four complementary metrics: Synthesis Score (pS), Dialectic Score (DS), Improvement (Δ), and Opposition Compliance (OC)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 explicitly acknowledges the absence: 'some aspects of reasoning... may require expert human judgment. Human-in-the-loop extensions could validate and refine SIEV's automated signals.' Evaluation is entirely automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Uses the standard test sets of GSM8K and MMLU, which are established benchmarks with defined test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1, 6, and 10 provide per-topic MMLU breakdowns. Figure 4 shows per-pattern breakdowns. Table 2 compares overall vs representative topic performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 8 provide detailed qualitative failure examples. Figure 3 shows DeepSeek-R1 failing at synthesis; Figure 8 shows GPT-5 ignoring a valid antithesis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that Δ is negative across all models on average (Table 1), meaning synthesis generally degrades performance. Section 3.1: 'Δ is negative on average, indicating limited refinement at synthesis.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claim 'GPT-5-chat loses more than 40 points on GSM' is confirmed in Table 1 (Δ=-40.2). Claims about substantial gaps are supported by the wide pS ranges shown."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is appropriately hedged with causal language. Section 3.2: 'These patterns do not settle the debate, but they add weight to an existing view.' Section 3.1: 'The performance drops may hint at underlying issues in models training, though diagnosing these is beyond our current scope.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Evaluating LLM Reasoning Beyond Correctness and CoT' and abstract claim SIEV enables 'a clearer foundation for assessing and understanding the reasoning capabilities of LLMs' are broad generalizations from only two benchmarks (GSM8K and MMLU). The limitations section acknowledges this but the framing exceeds the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 'Key Takeaway' extensively discusses whether improvements reflect genuine reasoning or context-sensitive pattern matching, citing prior skeptical work (Dziri et al., Kambhampati, McCoy et al.)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 acknowledges 'these signals do not necessarily certify authentic reasoning, they offer stronger grounds for interpreting whether and how a model's apparent reasoning reflects a stable, integrative process.' The gap between measurement (synthesis accuracy) and claim (reasoning quality) is explicitly discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'O3', 'GPT-5', 'GPT-4', 'DeepSeek-R1', 'Kimi-K2'. No API versions, snapshot dates, or model IDs are provided. This is critical since model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full prompt text for all three stages (thesis, antithesis, synthesis) with both MMLU and GSM parameter specifications."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API parameters are reported. Only the DS formula parameters (λ=0.7, γ=1) are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-agent pipeline is described in detail in Section 2.2 and Figure 2: Agent A produces thesis and synthesis, Agent B produces antithesis. Thinking token handling for R1 is also documented in Figure 3 notes."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how model outputs were parsed, how answer extraction worked, how edge cases (malformed outputs, refusals) were handled, or whether any filtering was applied to the results."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Brief Discussion and Limitations' provides substantive discussion of multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses specific threats: OC measures opposition but not semantic quality of antitheses, synthesis evaluation via correctness alone misses multi-dimensional quality, absence of human-judged reasoning traces limits validation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'it remains to be seen how these findings generalize to emerging benchmarks, multimodal settings, or tasks that demand long-horizon planning or domain-specific symbolic reasoning.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw model outputs, reasoning traces, or per-question results are released. Only aggregate statistics are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No details on when API calls were made, which API endpoints were used, or how the 21 models were accessed. The pipeline structure is described but not the data collection specifics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The TAS pipeline is described conceptually (Figure 2) but operational details are missing: how outputs were parsed, how correctness was determined, how many items failed parsing, whether any were excluded."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned. The author is at Microsoft Research but no funding acknowledgment section exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation 'Microsoft Research, Vancouver, Canada' is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The author works at Microsoft Research. Microsoft has a major partnership with OpenAI whose models (GPT-4, GPT-5, O3, etc.) are extensively evaluated. This financial relationship is not discussed. Several OpenAI models rank highly (O3 ranks #1 on GSM)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 21 models evaluated. This is relevant since GSM8K (2021) and MMLU (2020) predate all tested models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SIEV has 'lower susceptibility to contamination' (Section 1) but does not analyze whether thesis scores are inflated by contamination. GSM8K and MMLU are known to be heavily contaminated in modern LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K and MMLU were both published years before any tested model's training. The paper acknowledges these are 'saturated benchmarks' but does not address whether saturation is partly due to contamination, which would undermine the thesis-score baseline."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "SIEV requires 3 inference passes per question across 21 models on two full benchmarks. No API costs, token counts, or latency figures are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, API spend, or hardware information is provided despite what must be substantial inference costs across 21 models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports ± values across runs for all models (e.g., O3: 97.1±0.1), indicating multiple runs with variance measurement."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The ± values in Table 1 imply multiple runs but the exact number of runs is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The DS formula uses λ=0.7 and γ=1 without reporting how these values were selected or what alternatives were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "λ=0.7 and γ=1 are presented without justification. Table 1 note states these values but no sensitivity analysis or selection rationale is provided."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The author proposes SIEV and evaluates it without acknowledging self-comparison bias or seeking independent validation of the framework."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SIEV triples inference cost (3 passes per question) compared to standard evaluation. This cost overhead is never discussed or quantified relative to the diagnostic value gained."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is motivated by questioning construct validity of correctness-based benchmarks. Section 1 argues that accuracy alone 'reveals little about the process' and SIEV is proposed as having better construct validity for measuring reasoning."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The two-agent scaffold is identical across all model comparisons. Thinking token handling for R1 is explicitly documented: 'we redact thinking tokens when passing them' (Figure 3 notes), addressing a potential confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and MMLU (2020) both predate all 21 tested models. The paper does not discuss whether models trained on these benchmarks affects thesis scores, which serve as the baseline for all SIEV comparisons."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "In the synthesis stage, models receive the antithesis as input, which may contain answer information. This structural feature leakage is not discussed as a potential confound."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K and MMLU items share structural patterns that could advantage certain models."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used despite evaluating on heavily contaminated benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-5-chat loses more than 40 points on GSM when evaluated through SIEV's process-oriented lens",
       "evidence": "Table 1 shows GPT-5-chat pT=96.4 vs pS=56.2, Δ=-40.2 on GSM8K",
       "supported": "strong"
     },
     {
       "claim": "SIEV reveals substantial hidden reasoning gaps (often exceeding 20%) on MMLU, a benchmark often treated as broadly solved",
       "evidence": "Table 1 shows multiple models with >20% drops from pT to pS on MMLU (e.g., GPT-5-chat: 88.2→50.5, GPT-4.1-mini: 84.9→54.7, Llama-3.3-70B: 85.1→58.3)",
       "supported": "strong"
     },
     {
       "claim": "Models with similar static correctness can mask very different reasoning trajectories",
       "evidence": "Table 1: GPT-5-chat (pT=96.4) and O3 (pT=97.1) have similar thesis scores but diverge at synthesis (56.2 vs 93.6 on GSM)",
       "supported": "strong"
     },
     {
       "claim": "Models' reasoning is strongly topic-dependent rather than a uniform general capability",
       "evidence": "Figure 6 shows wide pS variation across MMLU domains. Llama-3.3-70B-Inst performs well in Elementary Math but poorly in Moral Disputes. Section 3.1 discusses topic-dependent patterns.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-model antitheses can improve reasoning performance compared to self-dialectics",
       "evidence": "Figure 7 (middle plot) shows GPT-5 improves +5.4 to +14 points in pS when paired with different antithesis models on GSM",
       "supported": "moderate"
     },
     {
       "claim": "SIEV has lower susceptibility to contamination compared to correctness-based evaluation",
       "evidence": "Claimed in Section 1 as a key contribution but no empirical test of this claim is provided",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating related products",
       "detail": "Author is at Microsoft Research. Microsoft has a major partnership with OpenAI. Multiple OpenAI models (O3, GPT-5, O1) rank highly on SIEV metrics, particularly on GSM. This conflict is not acknowledged."
     },
     {
       "flag": "No model version specificity",
       "detail": "21 models evaluated using only marketing names (O3, GPT-5, etc.) without API versions or snapshot dates. Results are not reproducible since model behavior changes across versions."
     },
     {
       "flag": "Unsupported contamination resistance claim",
       "detail": "The paper claims SIEV has 'lower susceptibility to contamination' as a key contribution but provides no empirical evidence. Meanwhile, thesis scores on heavily contaminated benchmarks serve as the baseline for all SIEV comparisons."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters not reported for any of 21 models. These significantly affect LLM output quality and could influence synthesis/antithesis quality."
     },
     {
       "flag": "Unjustified metric parameters",
       "detail": "DS formula uses λ=0.7 and γ=1 without justification or sensitivity analysis. These parameters directly affect model rankings."
     },
     {
       "flag": "No cost reporting for expensive evaluation",
       "detail": "Evaluating 21 models × 2 benchmarks × 3 passes each represents massive inference cost that is never quantified, making practical adoption assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational work on CoT prompting that SIEV extends beyond by evaluating reasoning process quality."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark used as primary evaluation testbed; relevant to benchmark evaluation methodology."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "GSM8K benchmark used as primary evaluation testbed for SIEV."
     },
     {
       "title": "Faith and fate: Limits of transformers on compositionality",
       "authors": ["Nouha Dziri", "Ximing Lu"],
       "year": 2023,
       "arxiv_id": "2305.10403",
       "relevance": "Key prior work questioning whether LLMs genuinely reason, providing symbolic/graph-based reasoning analysis."
     },
     {
       "title": "GSM-Plus: A comprehensive benchmark for evaluating the robustness of LLMs as mathematical problem solvers",
       "authors": ["Qintong Li", "Leyang Cui", "Xueliang Zhao"],
       "year": 2024,
       "arxiv_id": "2402.19255",
       "relevance": "Related work on probing LLM reasoning robustness through benchmark perturbations."
     },
     {
       "title": "GSM-Symbolic: Understanding the limitations of mathematical reasoning in large language models",
       "authors": ["Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi"],
       "year": 2024,
       "arxiv_id": "2410.05229",
       "relevance": "Introduces symbolic perturbations to test LLM mathematical reasoning sensitivity."
     },
     {
       "title": "Embers of autoregression: Understanding large language models through the problem they are trained to solve",
       "authors": ["R. Thomas McCoy", "Shunyu Yao", "Dan Friedman"],
       "year": 2023,
       "arxiv_id": "2309.13638",
       "relevance": "Foundational work questioning whether LLMs reason or imitate reasoning through pattern matching."
     },
     {
       "title": "Alice in wonderland: Simple tasks showing complete reasoning breakdown in state-of-the-art large language models",
       "authors": ["Marianna Nezhurina", "Lucia Cipolina-Kun", "Mehdi Cherti"],
       "year": 2024,
       "arxiv_id": "2406.02061",
       "relevance": "Demonstrates reasoning failures in frontier LLMs on simple tasks, supporting SIEV's motivation."
     },
     {
       "title": "A peek into token bias: Large language models are not yet genuine reasoners",
       "authors": ["Bowen Jiang", "Yangxinyu Xie", "Zhuoqun Hao"],
       "year": 2024,
       "arxiv_id": "2406.11050",
       "relevance": "Shows LLM reasoning fragility under token-level perturbations, complementary to SIEV's process-level evaluation."
     },
     {
       "title": "MMLU-Pro: A more robust and challenging multi-task language understanding benchmark",
       "authors": ["Yubo Wang", "Xueguang Ma", "Ge Zhang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Addresses benchmark saturation in MMLU with harder variant; relevant to LLM evaluation methodology."
     }
   ]
 }