ai-research-survey

scan-v5.json (18580B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "GSM-Plus: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers",
     "authors": [
       "Qintong Li",
       "Leyang Cui",
       "Xueliang Zhao",
       "Lingpeng Kong",
       "Wei Bi"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2402.19255",
     "doi": "10.48550/arXiv.2402.19255"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (up to 20% gap, 25 LLMs tested, human performance unaffected) are all directly supported by Table 4 and Figure 6 results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'task-specific optimization… gives better accuracy while robustness depends more on base model and fine-tuning dataset' are stated causally but derived from cross-model comparisons where models differ on multiple confounding dimensions simultaneously.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly bounds scope to grade school level math; broader generalizations about LLM robustness are appropriately hedged to math word problems on GSM8K variants.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges GPT-4 generation bias (Table 3) but does not discuss whether performance drops reflect increased task difficulty rather than robustness failures, or why critical thinking is a different task type entirely rather than a harder version.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly measures answer accuracy on perturbed questions and defines specific metrics (PDR, ASP) that operationalize robustness; the paper does not conflate these with deeper understanding.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated Limitations section lists three explicit limitations: scope limited to elementary-level math, solution chain accuracy not assessed, and no investigation of failure causes.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations are specific but miss key threats: potential data contamination in model training, moderate inter-annotator agreement (Krippendorff's α=0.567) is not discussed as a threat, and prompting experiment using only 120/1319 questions is not flagged.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers only grade school math, only the listed perturbation types, and does not investigate solution chain accuracy or underlying failure reasons.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding from NSFC-RGC joint scheme (grant N_HKU714/21) is disclosed in the Acknowledgements section.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (HKU and Tencent AI Lab) are clearly stated on the first page; the internship relationship of two authors at Tencent is also footnoted.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSFC and RGC are government research funding bodies with no stake in LLM benchmark outcomes; Tencent affiliation is disclosed but the paper evaluates third-party models.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, consulting) is present beyond the funding acknowledgment.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Robustness is operationalized through PDR and ASP metrics (defined with formulas), and all 8 perturbation types are precisely defined in Table 6 with formal descriptions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it contributes GSM-Plus—an adversarial benchmark dataset with 10,552 question variations across 8 perturbation types—and a systematic evaluation of 25 LLMs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a structured comparison of GSM-Plus against six prior datasets, explicitly noting which perturbation types existing work covers and where GSM-Plus extends coverage.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper grounds the 5 perturbation perspectives in Polya's four-stage problem-solving framework, arguing these dimensions capture distinct cognitive capabilities required for mathematical reasoning.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper provides GPT-4 pass rates by perturbation type (Table 7) but does not formally characterize difficulty tiers (easy/medium/hard) or measure item-level difficulty for the 10,552 questions.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "Floor effects are implicitly visible (near-0% accuracy for most models on critical thinking in Figure 6) but the paper does not explicitly discuss these as a discriminability concern or check ceiling effects for the benchmark overall.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": true,
           "justification": "Human performance is reported in Table 4 (96.77% on GSM8K, 98.75% on GSM-Plus, ASP of 92.11%), established using qualified annotators with bachelor's degrees and a qualifying exam.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Exact-match accuracy is the standard for GSM8K; PDR and ASP are formally defined with equations; the critical thinking evaluation provides an explicit list of accepted expressions for the 'no valid answer' case.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The paper cites contamination concerns (Golchin and Surdeanu, 2023) but implements no anti-contamination measures such as temporal splits, canary strings, or dynamic generation in GSM-Plus itself.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether future models could overfit to the fixed 10,552 variations or become obsolete, and provides no update mechanism or versioning plan.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The limitations section notes what GSM-Plus does not measure (solution chains, higher-level math, failure causes), but the paper does not discuss failure modes of the benchmark itself—e.g., that rephrased questions may be gamed by GPT-4-family models.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompts for all four prompting methods (CoT, LTM, COMP, SC) are provided in Appendix C.4; the paper states the dataset and evaluation suite will be released at the project website.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "The paper documents GPT-4 generation prompts, human annotation process (qualifying exam, batch process, 10% cross-annotation, IAA α=0.567), and per-perturbation pass rates in Table 7.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper states the dataset will be released but does not specify licensing terms, usage restrictions, or access conditions beyond the project website URL.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The paper specifies GSM-Plus is intended for evaluating robustness of LLMs on grade school math word problems and explicitly states it should not be used to assess higher-level math reasoning or solution chain quality.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs exhibit a performance gap of up to 20% between GSM8K accuracy and GSM-Plus accuracy, while human performance is unaffected (PDR of -2.05%)",
       "evidence": "Table 4: GPT-4 drops from 93.25% to 85.58% (PDR 8.23%), GPT-3.5 from 73.62% to 61.19% (PDR 16.88%); humans show 96.77% to 98.75%",
       "supported": "strong"
     },
     {
       "claim": "Critical thinking and arithmetic variation are the most challenging perturbation types, with most open-source models showing near-complete failure on critical thinking",
       "evidence": "Figure 6 shows ~100% PDR for most open-source models on critical thinking; Table 8 shows near-0% accuracy for many models on critical thinking variations",
       "supported": "strong"
     },
     {
       "claim": "Math-specific fine-tuning improves benchmark accuracy but does not proportionally improve robustness (PDR remains similar between foundation and SFT models)",
       "evidence": "Table 4: LLaMA-2-13B PDR 34.76%, Abel-13B PDR 31.97%, MAmmoTH-13B PDR 34.58% — minimal improvement despite large accuracy gains",
       "supported": "moderate"
     },
     {
       "claim": "Larger models generally exhibit lower PDR, with base model quality being more important than fine-tuning dataset for robustness",
       "evidence": "Abel series: PDR 37.67% (7B) → 31.97% (13B) → 28.52% (70B); MetaMath-Mistral-7B (PDR 27.69%) outperforms larger LLaMA-2-based models",
       "supported": "moderate"
     },
     {
       "claim": "The compositional prompting method COMP improves robustness over standard CoT, particularly on critical thinking variations",
       "evidence": "Table 5: COMP improves critical thinking from 40.83% (CoT) to 54.17%; COMP+SC to 55.83% on GPT-3.5-Turbo on a 120-question subset",
       "supported": "moderate"
     },
     {
       "claim": "Program-based models are more robust to numerical variation but less robust to distractor insertion than NL-based models",
       "evidence": "Section 5.2: MAmmoTH-Coder-13B PDR 11.80% vs Abel-13B 14.03% for numerical variation; reversed for distractor insertion (23.2% vs 16.1%)",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "benchmark-creation"
   ],
   "key_findings": "GSM-Plus reveals that all 25 tested LLMs show substantial performance degradation (PDR 8–40%) when GSM8K questions are perturbed across 8 variation types, while human annotators show negative PDR (-2.05%), suggesting LLMs rely on superficial patterns rather than genuine mathematical understanding. Critical thinking—requiring models to recognize missing information—is the most catastrophic failure mode, with most open-source models collapsing to near-0% accuracy. Math fine-tuning consistently improves absolute accuracy but shows minimal benefit for robustness, while base model quality and fine-tuning dataset composition matter more than the SFT process itself. None of the four investigated prompting techniques achieves robust performance across all variation types, though the proposed compositional prompting method COMP modestly improves performance, especially on critical thinking.",
   "red_flags": [
     {
       "flag": "GPT-4 self-preference bias",
       "detail": "Question variations are generated by GPT-4 and the primary contamination check (Table 3) only compares GPT-4's performance on its own vs. human-corrected questions — it does not test whether GPT-4-family models systematically outperform on GPT-4-reworded questions vs. true adversarial rephrasing."
     },
     {
       "flag": "Prompting experiments on 9% subset",
       "detail": "The prompting analysis (Section 5.4) uses only 120 seed questions (960 variations) out of 1,319, with no justification for why this subset is representative or power calculation."
     },
     {
       "flag": "Moderate inter-annotator agreement",
       "detail": "Krippendorff's α = 0.567 is only moderate reliability; the paper does not discuss implications for annotation quality or how disagreements were resolved beyond 'authors manually review'."
     },
     {
       "flag": "No contamination resistance",
       "detail": "The benchmark is a fixed public dataset with no temporal splits or dynamic generation, making it susceptible to future model training on GSM-Plus variations, yet the paper provides no mitigation strategy."
     },
     {
       "flag": "Critical thinking as different task type",
       "detail": "The 'critical thinking' perturbation fundamentally changes the answer type (from numerical to 'no valid answer'), making it a different task rather than a variation of the original, which confounds robustness measurement with task-type generalization."
     }
   ],
   "cited_papers": [
     {
       "title": "Training verifiers to solve math word problems (GSM8K)",
       "relevance": "Parent dataset that GSM-Plus extends; provides the 1,319 seed questions"
     },
     {
       "title": "Are NLP models really able to solve simple math word problems? (SVAMP)",
       "relevance": "Prior robustness evaluation benchmark for grade school math; direct predecessor"
     },
     {
       "title": "Large language models can be easily distracted by irrelevant context (GSM-IC)",
       "relevance": "Prior work on distractor insertion perturbations; GSM-Plus extends this with 7 additional perturbation types"
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "relevance": "Baseline prompting method evaluated across all perturbation types in GSM-Plus"
     },
     {
       "title": "MetaMath: Bootstrap your own mathematical questions for large language models",
       "relevance": "Evaluated SFT model; also directly relevant as question bootstrapping approach"
     },
     {
       "title": "MAmmoTH: Building math generalist models through hybrid instruction tuning",
       "relevance": "Evaluated SFT model family across 7B/13B/70B scales"
     },
     {
       "title": "Time travel in LLMs: Tracing data contamination in large language models",
       "relevance": "Motivates robustness evaluation as alternative to contaminated standard benchmarks"
     },
     {
       "title": "Least-to-most prompting enables complex reasoning in large language models",
       "relevance": "Prompting method (LTM) evaluated and compared against proposed COMP method"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for anyone evaluating LLM math capabilities — provides a drop-in replacement for GSM8K with robustness measurement."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the prevailing narrative that >90% GSM8K accuracy means strong mathematical understanding, showing large gaps under minor perturbations."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises mild concern that LLMs deployed in math-adjacent tasks may fail on slight problem variations, but doesn't connect to safety-critical scenarios."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly engages the ongoing community debate about whether LLMs 'truly understand' math or exploit patterns, with concrete evidence for the skeptical side."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Dataset and evaluation code are released publicly; anyone can run models on GSM-Plus immediately and reproduce the robustness analysis."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates GPT-4, GPT-3.5, LLaMA-2, and CodeLlama — well-known models — but the authors are from HKU/Tencent, not a tier-1 AI lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42306766",
         "title": "Beyond Language Models: Byte Models Are Digital World Simulators",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42306766",
         "created_at": "2024-12-03T14:58:15Z"
       },
       {
         "hn_id": "46502322",
         "title": "Beyond Language Models: Byte Models Are Digital World Simulators (2024)",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46502322",
         "created_at": "2026-01-05T18:06:20Z"
       }
     ],
     "top_points": 4,
     "total_points": 5,
     "total_comments": 0
   }
 }