ai-research-survey

scan-v5.json (22093B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluating LLM Reasoning Beyond Correctness and CoT",
     "authors": [
       "Soheil Abbasloo"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.18134",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's headline claim—GPT-5-chat loses >40 points on GSM—is directly verified in Table 1 (∆=-40.2). Claims about 'substantial gaps' and SIEV surfacing hidden weaknesses on MMLU are supported by Table 1 and Figures 1/6.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly frames synthesis scores as evidence of 'genuine reasoning' vs. 'pattern replay,' but the study design is purely observational—no controlled manipulation demonstrates that lower synthesis scores causally reflect less genuine reasoning rather than a different task format effect.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section appropriately bounds scope to GSM8K and MMLU, but the abstract and conclusion make broad claims about 'LLMs' and 'reasoning capabilities' that extend well beyond the two saturated benchmarks actually tested.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly raises the alternative that cross-model synthesis gains may reflect structural token familiarity rather than improved reasoning, and the authors cite prior skeptical work (Dziri et al., Kambhampati) without dismissing it.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Synthesis correctness (pS) is used throughout as a direct proxy for 'genuine reasoning quality,' but Section 4 itself admits synthesis can be logically coherent yet factually incorrect or vice versa—this distinction is not maintained in the main claims.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Brief Discussion and Limitations' contains four named subsections covering scope of benchmarks, opposition quality, synthesis evaluation granularity, and absence of human-judged reasoning traces.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: OC only measures formal opposition not semantic quality, pS is correctness-only and misses logical coherence, and evaluation intentionally avoids human annotation leaving conceptual-quality judgments unvalidated.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'it remains to be seen how these findings generalize to emerging benchmarks, multimodal settings, or tasks that demand long-horizon planning or domain-specific symbolic reasoning.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure section exists. The impact statement says 'none which we feel must be specifically highlighted here,' with no mention of grants, institutional support, or compute resources.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is clearly stated: 'Microsoft Research, Vancouver, Canada' with contact email.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The sole author is a Microsoft Research employee; the paper evaluates Microsoft's own models (GPT-5, GPT-5-chat, GPT-4, O3, O1, O4-mini, etc.) and positions them favorably in some rankings—a clear conflict of interest that is not disclosed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interests declaration appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Thesis, antithesis, and synthesis are defined in Section 2.1; OC (Opposition Compliance), pS (Synthesis Score), DS (Dialectic Score), and ∆ are formally defined with formulas in Section 2.4.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists four key contributions of SIEV: benchmark/model agnosticism, lower contamination susceptibility, exposing hidden weaknesses, and natural multi-agent compatibility.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 engages substantively with GSM-Plus, GSM-Symbolic, ontology-guided interventions, CoT prompting, and skeptical reasoning literature, explicitly contrasting SIEV from these approaches rather than just listing them.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 argues that dialectical thesis–antithesis–synthesis structure measures coherence, adaptability, and integration—dimensions of reasoning that correctness cannot capture—grounding the claim in Hegelian philosophical tradition.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": false,
           "answer": false,
           "justification": "SIEV is explicitly 'not a benchmark itself' (Section 4) but a framework overlaid on existing benchmarks; it creates no new items and therefore has no difficulty distribution to characterize.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly selects GSM8K and MMLU because thesis scores show near-ceiling clustering, then demonstrates SIEV's synthesis scores produce much wider spread (pS ranging from ~40 to ~93), confirming the framework avoids the ceiling effect.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is provided anywhere in the paper; there is no comparison of how humans perform under the thesis–antithesis–synthesis evaluation protocol.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "The DS formula includes free parameters λ=0.7 and γ=1 (Table 1 footnote) with no ablation study, sensitivity analysis, or justification for why these values were chosen over alternatives.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims SIEV has 'lower susceptibility to contamination' because it evaluates dynamics rather than static answers, but provides no empirical validation—models could learn to produce good antitheses through training exposure to the dialectical format.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section 4 acknowledges that generalization to emerging benchmarks is unseen but provides no plan for updating SIEV, versioning the evaluation protocol, or addressing temporal obsolescence as model capabilities evolve.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 identifies four specific failure modes: OC measures only formal not semantic opposition quality, pS is correctness-only missing logical coherence, synthesis quality is multidimensional, and human judgment is absent for conceptual validity.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper states 'the SIEV source code is publicly available at https://github.com/microsoft/siev' and Appendix A provides full prompt specifications for all three stages.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": false,
           "answer": false,
           "justification": "SIEV does not create a new dataset; it is an evaluation framework applied to existing benchmarks (GSM8K, MMLU). No dataset documentation is applicable.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub link is provided but the paper does not state the license under which the code is released, nor the terms under which SIEV outputs may be used or shared.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 'SIEV as a General Approach' explains that SIEV is for evaluating reasoning processes across benchmarks, and the limitations section explains what SIEV does NOT measure (semantic quality of opposition, human-judged reasoning validity).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-5-chat loses more than 40 points on GSM when evaluated through SIEV's synthesis score compared to its thesis score.",
       "evidence": "Table 1 reports GPT-5-chat pT=96.4, pS=56.2, ∆=-40.2 on GSM8K.",
       "supported": "strong"
     },
     {
       "claim": "Models with near-identical thesis accuracy can exhibit sharply different synthesis performance, revealing hidden reasoning differences.",
       "evidence": "Table 1 shows multiple models cluster near 96-97 on pT (GSM) while pS spans 56–93; Figure 1 illustrates topic-level divergence on MMLU.",
       "supported": "strong"
     },
     {
       "claim": "SIEV has lower susceptibility to contamination than correctness-based metrics.",
       "evidence": "Claimed in Section 1 Key Contributions item (2), but no empirical validation is provided—it is a theoretical argument only.",
       "supported": "weak"
     },
     {
       "claim": "Reasoning capability is strongly topic-dependent rather than a uniform general skill.",
       "evidence": "Figure 6 shows Llama3.3-70B-Instruct scoring high in Elementary Math but low in Moral Disputes; DeepSeek-R1 peaks in quantitative domains but weakens on normative ones.",
       "supported": "moderate"
     },
     {
       "claim": "Models generally show negative ∆, meaning synthesis quality degrades from thesis, indicating limited integrative reasoning.",
       "evidence": "All 21 models show negative ∆ on both GSM and MMLU in Table 1, ranging from -0.7 to -40.2.",
       "supported": "strong"
     },
     {
       "claim": "Cross-model antitheses improve synthesis performance compared to self-generated antitheses for many models.",
       "evidence": "Figure 7 shows pS gains for GPT-5 of +5.4 to +14 points across pairings; similar patterns for DeepSeek-R1 and O4-mini in most settings.",
       "supported": "moderate"
     },
     {
       "claim": "Thesis accuracy (pT) is weakly related to opposition production (OC) and thesis-to-synthesis change (∆).",
       "evidence": "Figure 5 distance correlation analysis shows weak pT–OC and pT–∆ links; correlation matrix in Section 3.1 discussion confirms this.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "SIEV, a dialectical evaluation framework applying thesis–antithesis–synthesis interactions to existing benchmarks, reveals substantial reasoning gaps hidden by correctness-only metrics: models with near-identical thesis accuracy (e.g., O3 vs. GPT-5-chat both at ~96% on GSM) diverge by >35 points on synthesis score. Across 21 LLMs on GSM8K and MMLU, all models show negative average ∆, meaning synthesis quality universally degrades from thesis, with GPT-5-chat losing 40 points. Cross-model dialectical pairing generally improves synthesis performance compared to self-generated antitheses, suggesting that reasoning in LLMs may be more context-sensitive than a stable general capability. Reasoning performance is strongly topic-dependent across MMLU domains, with quantitative and normative subjects producing very different model rankings.",
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "The sole author is a Microsoft Research employee evaluating Microsoft's own models (GPT-5, O3, O1, O4-mini, GPT-4, GPT-4.1 family). No competing interests statement is present."
     },
     {
       "flag": "Proxy conflated with construct",
       "detail": "Synthesis correctness (pS) is treated throughout as a direct measure of 'genuine reasoning,' but the paper itself acknowledges synthesis can be logically coherent yet factually wrong—the distinction between the proxy and the construct is not maintained in the main claims."
     },
     {
       "flag": "Free parameters unjustified",
       "detail": "The Dialectic Score formula uses λ=0.7 and γ=1 with no ablation study or sensitivity analysis explaining why these values are appropriate."
     },
     {
       "flag": "Contamination resistance unvalidated",
       "detail": "The claim that SIEV has 'lower susceptibility to contamination' is theoretical; no empirical test of this claim is provided, and models could learn to produce good antitheses through training exposure to dialectical formats."
     },
     {
       "flag": "No human baseline",
       "detail": "The evaluation framework claims to distinguish genuine reasoning from pattern replay, yet no human performance data is provided to calibrate what 'genuine reasoning' looks like under the SIEV protocol."
     },
     {
       "flag": "Framing-type mismatch",
       "detail": "Section 4 explicitly states 'SIEV is not a benchmark itself, but a dialectical approach to benchmark models'—yet the paper is presented as a benchmark-creation contribution, creating a fundamental ambiguity about the nature of the contribution."
     }
   ],
   "cited_papers": [
     {
       "title": "Training Verifiers to Solve Math Word Problems (GSM8K)",
       "relevance": "Core benchmark (GSM8K) on which SIEV is evaluated; represents the correctness-only evaluation paradigm the paper critiques."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Second core benchmark evaluated; treated as broadly saturated for top models, making SIEV's ability to surface variance particularly notable."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "relevance": "Represents the prior state of process-oriented evaluation that SIEV positions itself against and extends."
     },
     {
       "title": "GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in LLMs",
       "relevance": "Related perturbation-based approach to probing reasoning robustness; SIEV contrasts itself by not altering benchmarks."
     },
     {
       "title": "GSM-Plus: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers",
       "relevance": "Another correctness-under-perturbation approach that SIEV argues remains tied to the static correctness paradigm."
     },
     {
       "title": "Faith and Fate: Limits of Transformers on Compositionality",
       "relevance": "Prior work questioning genuine reasoning capabilities of LLMs; provides theoretical grounding for SIEV's skeptical framing."
     },
     {
       "title": "Can Large Language Models Reason and Plan?",
       "relevance": "Kambhampati's skeptical view on LLM planning and reasoning is cited to contextualize SIEV's motivation."
     },
     {
       "title": "Measuring and Testing Dependence by Correlation of Distances",
       "relevance": "Statistical methodology (distance correlation) used in SIEV's correlation analysis of dialectical metrics across MMLU sub-topics."
     },
     {
       "title": "A Peek into Token Bias: Large Language Models Are Not Yet Genuine Reasoners",
       "relevance": "Prior work on token-level fragility cited as parallel evidence that apparent reasoning may reflect statistical patterns rather than genuine inference."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Code is available and the framework can be applied to any existing benchmark, but the computational overhead of running three-stage dialectical evaluations for 21 models is substantial for most practitioners."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that GPT-5-chat—a top model by correctness—collapses to near-bottom on synthesis scoring is genuinely surprising and challenges the conventional correctness-as-reasoning paradigm."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper raises no AI safety or risk concerns; the impact statement explicitly declines to highlight societal consequences."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The Microsoft-employee-evaluating-Microsoft-models dynamic creates an implicit tension, and the dramatic ranking reversals (GPT-5-chat from rank 1 to near-last) are noteworthy, but the paper presents this matter-of-factly."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code is publicly available on GitHub and prompts are fully specified in Appendix A, making the framework reproducible for those with API access to the evaluated models."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Microsoft Research affiliation plus evaluation of named flagship models (GPT-5, O3, O1, DeepSeek-R1, Kimi-K2) gives the paper high brand-recognition surface area."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45838564",
         "title": "LLMs encode how difficult problems are",
         "points": 174,
         "comments": 38,
         "url": "https://news.ycombinator.com/item?id=45838564",
         "created_at": "2025-11-06T18:29:03Z"
       },
       {
         "hn_id": "46370038",
         "title": "A Search for Radio Technosignatures from Interstellar Object 3I/Atlas",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=46370038",
         "created_at": "2025-12-23T22:07:08Z"
       },
       {
         "hn_id": "46425525",
         "title": "Optimal Software Pipelining and Warp Specialization for Tensor Core GPUs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46425525",
         "created_at": "2025-12-29T20:54:07Z"
       },
       {
         "hn_id": "45751115",
         "title": "DeepSeek-OCR: Contexts Optical Compression",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45751115",
         "created_at": "2025-10-29T18:33:29Z"
       },
       {
         "hn_id": "46069881",
         "title": "Conformal Prediction for Compositional Data",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46069881",
         "created_at": "2025-11-27T15:03:53Z"
       },
       {
         "hn_id": "38152071",
         "title": "Reality3DSketch: Rapid 3D Modeling of Objects from Single Freehand Sketches",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38152071",
         "created_at": "2023-11-05T15:41:49Z"
       },
       {
         "hn_id": "32056080",
         "title": "Data-Driven Offline Optimization for Architecting Hardware Accelerators",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=32056080",
         "created_at": "2022-07-11T13:48:52Z"
       },
       {
         "hn_id": "46021507",
         "title": "World-in-World: World Models in a Closed-Loop World",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46021507",
         "created_at": "2025-11-23T07:25:35Z"
       },
       {
         "hn_id": "46369891",
         "title": "The size of 3I/ATLAS from non-gravitational acceleration",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=46369891",
         "created_at": "2025-12-23T21:51:08Z"
       },
       {
         "hn_id": "38101172",
         "title": "Locomotion Through Step Placement with Straight Legs and Rolling Contacts",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38101172",
         "created_at": "2023-11-01T16:57:11Z"
       }
     ],
     "top_points": 174,
     "total_points": 190,
     "total_comments": 40
   }
 }