ai-research-survey

scan.json (26681B)

---

 {
   "paper": {
     "title": "GSM-Plus: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers",
     "authors": ["Qintong Li", "Leyang Cui", "Xueliang Zhao", "Lingpeng Kong", "Wei Bi"],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2402.19255",
     "doi": "10.48550/arXiv.2402.19255"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GSM-PLUS reveals that LLMs exhibit significant performance drops (up to 40% PDR) when math questions are perturbed, even when they correctly solve the original GSM8K questions. Critical thinking and arithmetic variation perturbations cause the largest drops, while numerical substitution and problem understanding are more resilient. Task-specific fine-tuning improves accuracy but does not substantially improve robustness (PDR). A compositional prompting method (COMP) shows modest improvements but does not close the gap between standard and adversarial benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a project URL (qtli.github.io/GSM-Plus/) and states 'Dataset and evaluation suits will be released' in the conclusion. The abstract also links to the project page."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The GSM-PLUS dataset of 10,552 question variations is built on the public GSM8K dataset. The paper states 'Dataset and evaluation suits will be released' and provides a project page URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper describes the experimental setup but does not provide scripts or commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, 8, and Figure 6 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GPT-4 exhibits the highest level of robustness') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports PDR (performance drop rate) as a relative measure with baseline context, e.g., 'GPT-4 exhibits the highest level of robustness with the smallest PDR of 8.23%' and provides accuracy on both GSM8K and GSM-PLUS for context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the subset size of 120 seed questions used in the prompting experiments (§5.4). The full dataset size of 10,552 is described but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic predictions, which means single-run results. No variance across runs or seeds is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 25 LLMs including closed-source (GPT-4, GPT-3.5-Turbo), open-source foundation models, and math-specific SFT models, plus a human baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4, Mistral-7B, LLaMA-2 variants, and recent math SFT models (MetaMath, ToRA, MAmmoTH, SEGO), which were contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates by comparing performance across 8 perturbation types, different model sizes, foundation vs SFT models, and different prompting techniques. Table 5 compares COMP components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy, PDR (performance drop rate), and ASP (accurately solved pairs) as distinct evaluation metrics (§3.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human performance baseline was established using 3 qualified annotators on 450 questions (50 seed + 400 variations), reported in Table 4 and Appendix C.1."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GSM-PLUS is built from the GSM8K test set (1,319 questions). The dataset is used purely for evaluation, not for training or tuning any of the models tested."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-perturbation breakdowns are provided in Figure 2, Figure 6, and Tables 8-9 across all 8 perturbation types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed failure case examples are provided in §5.2 (Figure 1) and Appendix C.3 (Examples C.1-C.8) covering critical thinking, adding operation, distractor insertion, reversing operation, and IDF conversion failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that math SFT does not significantly improve robustness (§5.1), that no prompting technique is sufficient to close the performance gap (§5.4), and that compositional prompting has limited impact."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about performance drops, sensitivity to statement additions and target alterations, and limited prompting improvements are all supported by results in Tables 4-5 and Figures 2-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are largely about perturbation effects on performance, which are tested through controlled single-variable manipulations (each perturbation type applied independently). The ablation of SFT datasets in §5.1 uses controlled comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds results to grade school math (§Limitations: 'we mainly focus on the robustness of math reasoning at the elementary school level') and states that other education levels are future work."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses training data leakage as an alternative explanation for high GSM8K performance (§1, citing Golchin and Surdeanu 2023), and GPT-4's potential preference bias in dataset construction (Table 3, §3.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly frames accuracy as measuring robustness of math reasoning rather than making broader claims. PDR and ASP are well-defined metrics that directly measure what is claimed (robustness to perturbations), and the paper acknowledges it does not investigate accuracy of solution chains (Limitations)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4' and 'GPT-3.5-Turbo' without specific version strings or snapshot dates. Open-source models are identified by name and size (e.g., 'Mistral-7B', 'LLaMA-2-7B') but without exact checkpoint versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for the COMP method are provided in Appendix C.4 (Examples C.9-C.14), including premise extraction, subgoal generation, calculation, and verification prompts. The paper also describes 8-shot prompting for open-source models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The decoding temperature is set to 0 for deterministic predictions' (§4) and specifies 8 demonstrations for open-source base models."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The COMP method's scaffolding is described in detail in §5.4 and Figure 5: iterative subgoal generation, calculation, and verification with specific prompts for each step."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The dataset construction pipeline is well-documented in §3.2: GPT-4 generates variations, human annotators verify/correct them with a qualifying exam, batched workloads, and cross-annotation (10% with 3+ annotators, 90.02% IAA). Table 7 shows pass rates per perturbation type."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusion, listing three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) focus only on elementary school level math, (2) comparing only answer accuracy without evaluating solution chains, (3) not investigating underlying reasons for failures."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: 'we mainly focus on the robustness of math reasoning at the elementary school level' and identifies evaluating other education levels, solution chain accuracy, and failure root causes as out of scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GSM-PLUS dataset is stated to be released via the project page (qtli.github.io/GSM-Plus/), which would include the raw question-answer pairs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed data collection procedure is described in §3.2 and Appendix B: GPT-4 generates variations from 8 perturbation types on 1,319 GSM8K seed questions, followed by human verification with qualifying exams and batched annotation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 describes annotator recruitment: qualifying exam of 24 variations, individual review of submissions, selection of 5 evaluators with at least bachelor's degrees. Appendix C.1 describes human performance annotator recruitment via Tencent crowdsource platform with qualification requirements."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: GSM8K seed questions → GPT-4 variation generation → GPT-4 answer generation → human annotator verification/correction (18.85% revised) → cross-annotation quality check (10% with 3+ annotators). Table 7 provides pass rates per perturbation type."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This research was supported in part by the joint research scheme of the National Natural Science Foundation of China (NSFC) and the Research Grants Council (RGC) under grant number N_HKU714/21.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Hong Kong and Tencent AI Lab. No models from these organizations are being evaluated as primary subjects."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSFC and RGC are government research funding bodies with no financial stake in the benchmark evaluation outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 25 LLMs evaluated, despite this being critical for assessing whether models may have seen GSM8K solutions during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses contamination: 'merely solve math word problems based on superficial patterns or even due to training data leakage (Golchin and Surdeanu, 2023)' (§1). The numerical variation perturbation is specifically designed to test overfitting. Table 3 analyzes GPT-4's potential preference bias."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is motivated by contamination concerns. GSM-PLUS is designed to test whether models truly understand math or have memorized GSM8K solutions. The perturbation approach itself serves as a contamination detection method — if models solve originals but not perturbations, memorization is implicated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human annotation is for dataset construction and baseline performance, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The human involvement is for annotation/evaluation tasks, not a human subjects research study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study; annotators are hired for data quality tasks."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite evaluating 25 models on 10,552+ questions and using GPT-4 for dataset construction."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for either the dataset construction (GPT-4 calls) or the evaluation experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic predictions, producing single-run results. No seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states temperature=0 for deterministic predictions (§4), implying single runs. For COMP+SC, it states 'sampling 5 predictions' (Table 5)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper does not describe how the 8-shot examples were selected or whether any prompt tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of 8-shot demonstrations and 120-question subset for prompting experiments is not justified. No description of how configurations were selected."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 25 models and 8 perturbation types without any correction for multiple comparisons. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The COMP prompting method is proposed and evaluated by the same authors without acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute differences between models or methods. The COMP method requires multiple LLM calls per question (premise extraction + iterative subgoal/calculation/verification) but cost comparison with simpler methods is not provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses what GSM-PLUS measures vs. what is claimed, referencing Polya's principles (§3.1) and acknowledging limitations (e.g., not evaluating solution chain accuracy, focusing only on grade school level). Table 3 validates that GPT-4's dataset creation doesn't introduce bias."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The COMP method is a prompting strategy, not a scaffold, and models are compared using the same prompting approach within each experiment."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is designed to address temporal leakage: GSM8K (2021) may be in training data of later models, so GSM-PLUS creates new perturbations to test whether models memorized solutions. The paper cites Golchin and Surdeanu (2023) on data contamination."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context (e.g., the 8-shot demonstrations could provide structural hints)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM-PLUS variations are derived from GSM8K questions, creating structural dependencies between test items. This non-independence is not discussed — performance on a variation may be correlated with performance on the seed question by construction."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The perturbation approach itself functions as a leakage detection method: if a model solves originals but fails on semantically equivalent perturbations, this suggests memorization rather than understanding. The numerical substitution perturbation specifically tests overfitting to training data."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs show substantial performance drops on GSM-PLUS compared to GSM8K, with PDR up to 40%",
       "evidence": "Table 4 shows PDR ranging from 8.23% (GPT-4) to 40.56% (CodeLlama-7B) across 25 models. All models show lower accuracy on GSM-PLUS than GSM8K.",
       "supported": "strong"
     },
     {
       "claim": "Math SFT improves accuracy but does not significantly improve robustness (PDR)",
       "evidence": "§5.1: LLaMA-2-13B PDR 34.76% vs Abel-13B PDR 31.97%; LLaMA-2-70B PDR 29.40% vs MAmmoTH-70B PDR 29.62%. SFT models show similar or only slightly lower PDR than foundation models.",
       "supported": "strong"
     },
     {
       "claim": "Critical thinking and arithmetic variation cause the largest performance drops",
       "evidence": "Figure 2 and Figure 6 show near-100% PDR for critical thinking on most open-source models, and >40% PDR for adding operation on most models.",
       "supported": "strong"
     },
     {
       "claim": "The compositional prompting method COMP improves robustness over standard COT",
       "evidence": "Table 5: COMP achieves 65.52% on GSM-PLUS vs COT's 62.92%; COMP+SC achieves 69.47% vs COT+SC's 66.88%. Critical thinking improves from 40.83% to 54.17%.",
       "supported": "moderate"
     },
     {
       "claim": "Human performance is unaffected by the perturbations in GSM-PLUS",
       "evidence": "Table 4: Human accuracy is 96.77% on GSM8K and 98.75% on GSM-PLUS, with PDR of -2.05% (slight improvement).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims across 25 models and 8 perturbation types are based solely on point estimate comparisons without any statistical tests, despite making numerous claims about which models are more robust."
     },
     {
       "flag": "No error bars or variance reporting",
       "detail": "Using temperature=0 produces deterministic outputs, but this means results reflect a single decoding path. No assessment of sensitivity to prompt selection, demonstration examples, or other sources of variance."
     },
     {
       "flag": "Small subset for prompting experiments",
       "detail": "The prompting comparison (§5.4, Table 5) uses only 120 seed questions and 960 variations — a small subset that may not be representative. No justification for this sample size."
     },
     {
       "flag": "GPT-4 used for both dataset creation and evaluation",
       "detail": "GPT-4 generated the question variations and is also one of the evaluated models. While Table 3 shows similar performance on self-generated vs human-corrected variants, this creates a potential circular evaluation concern."
     }
   ],
   "cited_papers": [
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K, the foundational benchmark that GSM-PLUS extends for robustness evaluation."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Core prompting technique evaluated for robustness in mathematical reasoning."
     },
     {
       "title": "Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks",
       "authors": ["Wenhu Chen", "Xueguang Ma", "Xinyi Wang"],
       "year": 2022,
       "arxiv_id": "2211.12588",
       "relevance": "Program-of-thought prompting evaluated as a baseline prompting technique for math reasoning."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Self-consistency technique used in COMP+SC ensemble for improving robustness."
     },
     {
       "title": "MetaMath: Bootstrap your own mathematical questions for large language models",
       "authors": ["Longhui Yu", "Weisen Jiang", "Han Shi"],
       "year": 2023,
       "arxiv_id": "2309.12284",
       "relevance": "Math SFT model evaluated for robustness; demonstrates question bootstrapping for math training."
     },
     {
       "title": "Large language models can be easily distracted by irrelevant context",
       "authors": ["Freda Shi", "Xinyun Chen", "Kanishka Misra"],
       "year": 2023,
       "relevance": "Prior work on distractor insertion in math reasoning (GSM-IC), which GSM-PLUS extends with additional perturbation types."
     },
     {
       "title": "Are NLP models really able to solve simple math word problems?",
       "authors": ["Arkil Patel", "Satwik Bhattamishra", "Navin Goyal"],
       "year": 2021,
       "relevance": "SVAMP benchmark for math robustness evaluation; early evidence that models rely on shortcuts."
     },
     {
       "title": "Time travel in LLMs: Tracing data contamination in large language models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "arxiv_id": "2308.08493",
       "relevance": "Directly relevant to benchmark contamination concerns that motivate GSM-PLUS."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Advanced prompting technique for reasoning that is related to the compositional prompting approach explored."
     },
     {
       "title": "MAmmoTH: Building math generalist models through hybrid instruction tuning",
       "authors": ["Xiang Yue", "Xingwei Qu", "Ge Zhang"],
       "year": 2023,
       "arxiv_id": "2309.05653",
       "relevance": "Math SFT model evaluated for robustness; demonstrates hybrid training approach for math reasoning."
     }
   ]
 }