ai-research-survey

scan.json (26238B)

---

 {
   "paper": {
     "title": "Benchmarking Hallucination in Large Language Models based on Unanswerable Math Word Problem",
     "authors": [
       "Yuhong Sun",
       "Zhangyue Yin",
       "Qipeng Guo",
       "Jiawen Wu",
       "Xipeng Qiu",
       "Hui Zhao"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2403.03558"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code and data are available at https://github.com/Yuki-Asuuna/UMWP.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset UMWP is released at the same GitHub repository. Additionally, the answerable portions come from publicly available datasets (GSM8K, SVAMP, MultiArith, ASDiv)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency details are mentioned in the paper. The paper mentions using Spacy v3.6.1 and SimCSE but does not provide a full environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released on GitHub, the paper itself does not include a 'Reproducing Results' section or detailed instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., F1 scores in Figure 3 and Figure 4). No confidence intervals or error bars are shown on any figures or tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'RLHF substantially improves the F1 score', 'ICL and Instruction input forms significantly improve') but no statistical significance tests are reported. Comparisons are based solely on raw F1 score differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw F1 scores but does not provide effect sizes (e.g., Cohen's d) for its comparative claims about model size, RLHF, or input form effects. Cohen's kappa is reported for evaluation method agreement (Table 3) but this measures inter-rater agreement, not effect size for the main claims."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 5,200 questions and the human benchmark uses 200 samples from 5 volunteers, but no justification is provided for why these sample sizes are sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be from single runs with no indication of repeated experiments or result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 31 LLMs across different series (GPT-3, InstructGPT, Claude, LLaMA) and includes a human benchmark (Section 5.2, Figure 4). The comparison between template-only and template+rule evaluation methods (Table 3) also serves as a baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include GPT-4-0613 and Claude-2, which were contemporary at the time of writing (2024). The LLaMA-2-70b-chat was also recent. The paper covers a wide range from older (text-ada-001) to state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares two evaluation method variants: Template-only vs Template+Rule (mathematical expression detection), showing the contribution of the expression detection component. Section 5.1 also reports a threshold ablation for the similarity function."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses F1 score as the primary metric and also reports Cohen's kappa coefficient for evaluation method agreement (Table 3). Accuracy of answerable questions is reported separately in Appendix A.4 (Figure 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A human benchmark is established in Section 5.2 using 5 volunteers who evaluate 200 randomly sampled questions. The human F1 score of 93.16% is compared against LLMs (Figure 4). Additionally, 5 annotators participate in the evaluation method comparison (Section 5.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The entire UMWP dataset of 5,200 questions is used for evaluation. There is no mention of a held-out test set or dev/test split. All models are evaluated on the same full dataset."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides the distribution across five unanswerable question categories (Key Information Missing 32%, Ambiguous Key Information 49%, Unrealistic Conditions 11%, Unrelated Object 4%, Question Missing 5%). Results are broken down by model series (Figure 3) and by input form (Direct, Instruction, ICL)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.6 (Table 6) provides a detailed case study of hallucination examples categorized by failure type (making arbitrary assumptions, ignoring scope condition, ignoring unreasonable condition, etc.). Section 5.5 discusses noise analysis of irrelevant outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that text-babbage-001 breaks the general trend of improvement with model size in the InstructGPT series (Section 5.4). The paper also shows that many models perform poorly, with some achieving very low F1 scores (e.g., near 0 for small LLaMA models in Direct form)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that ICL and RLHF 'significantly enhance the model's ability to avoid hallucination,' which is supported by the results in Figure 3 showing consistent F1 improvements with RLHF and ICL input forms. The claim about testing 31 LLMs is verified in the experiments."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'RLHF substantially improves the F1 score' and 'ICL input forms...significantly improving the LLMs' ability to recognize hallucination.' These are observational comparisons between different model versions, not controlled experiments. The improvement could be due to many factors beyond RLHF alone (different training data, model architecture changes, etc.)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'Hallucination in Large Language Models' broadly, but the paper only tests hallucination in the specific context of unanswerable math word problems in English. The abstract claims 'utilizing MWP is a reliable and effective approach to assess hallucination' without bounding this to the tested domain. The Limitations section partially addresses this but the title and abstract overreach."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, the RLHF improvement could be due to instruction-following ability rather than hallucination reduction. The model size effect could be confounded with training data differences. No threats-to-validity or alternative explanation analysis is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Most model versions are specified with snapshot dates or version identifiers: gpt-4-0613, gpt-4-0314, gpt-3.5-turbo-0613, gpt-3.5-turbo-0301, text-davinci-001/002/003, claude-1, claude-instant-1/1.1/1.2, claude-2, LLaMA-7b/13b/30b/65b, LLaMA-v2-7b/13b/70b and their chat variants. The GPT models include snapshot dates (e.g., 0613, 0314). Claude models use version numbers (claude-1, claude-2) which, while not full snapshot dates, were the standard identifiers at the time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for all three input forms (Direct, Instruction, ICL) are provided in Figures 6, 7, and 8, including the complete ICL examples with 8 few-shot demonstrations. The instruction text and all fill values are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'we set the temperature T = 0.7 for GPT, InstructGPT, LLaMA, and LLaMA-2' and reports the similarity threshold T=0.75, but temperature is not stated for Claude models. No other generation hyperparameters (top-p, max tokens) are reported for any model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper sends prompts directly to LLMs and evaluates their outputs. There is no multi-step agent workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the dataset construction process: source datasets (SVAMP, MultiArith, GSM8K, ASDiv), modification strategies (Table 5 with 8 strategies), annotator process (2 annotators modify, 3 volunteers validate, unanimous agreement required). Table 2 provides statistics of the answerable questions by source."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present (after the Ethics Statement) discussing that the work focuses only on English QA and does not explore other tasks like summarization or code generation, and that it only addresses hallucination from the prompt engineering perspective."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions only generic limitations: focus on English QA, not exploring other tasks, and only addressing prompt engineering perspective. These are not specific threats to the validity of the reported results (e.g., no discussion of evaluation method reliability, sample bias, or confounds in the RLHF comparison)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions focus on English QA but does not explicitly state what the results do NOT show. For example, it does not bound the generalization claims: results are about math word problem hallucination specifically, not hallucination in general. The title and abstract imply broader applicability than demonstrated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The UMWP dataset is released at the GitHub repository (https://github.com/Yuki-Asuuna/UMWP), allowing independent verification of the benchmark questions. The source datasets (GSM8K, SVAMP, MultiArith, ASDiv) are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection: four source MWP datasets were used, two annotators modified answerable questions into unanswerable ones using 8 specific strategies (Table 5), and three volunteers validated each question (unanimous agreement required for acceptance). Statistics are provided in Tables 1 and 2."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '5 volunteers' for the human benchmark and '2 data annotators' and '3 volunteers' for dataset construction, but provides no information about how these individuals were recruited, their qualifications, or potential selection bias. The Ethics Statement mentions compensation but not recruitment channels."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: source datasets selected (Table 2) -> 2 annotators apply 8 modification strategies (Table 5) -> 3 volunteers validate -> questions with unanimous unanswerable annotations accepted -> final dataset of 2,600 answerable + 2,600 unanswerable questions. Algorithm 1 documents the evaluation pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work is supported by National Key Research and Development Program of China(2022YFC3302600).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: East China Normal University, Fudan University, and Shanghai AI Laboratory. None of the authors appear to be affiliated with the companies whose models are evaluated (OpenAI, Meta, Anthropic, Google)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the National Key Research and Development Program of China, a government funding body that has no financial interest in the relative performance of any specific LLM evaluated in this study."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. While the authors appear to be academic researchers with no obvious conflicts, the absence of an explicit declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 31 LLMs on the UMWP benchmark but does not state the training data cutoff dates for any of the models. This is relevant because UMWP is built from publicly available datasets (GSM8K, SVAMP, etc.) that could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The answerable questions in UMWP come directly from publicly available datasets (GSM8K, SVAMP, MultiArith, ASDiv) that are likely in the training data of the evaluated models. The paper does not discuss this potential contamination issue at all."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), SVAMP (2021), MultiArith (2016), and ASDiv (2020) were all published before the training cutoffs of models like GPT-4 and LLaMA-2. The answerable portion of UMWP uses these questions directly, creating contamination risk for the answerable question accuracy. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human involvement (5 volunteers for benchmarking, annotators for dataset creation) is minimal and serves as a reference point, not a human subjects study. This is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The human participants are volunteers performing annotation and benchmark tasks, not subjects of a human study. The Ethics Statement discusses data ethics but this is not a human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper. The volunteers and annotators serve as reference points, not as subjects of a human study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Annotators and volunteers are tools for dataset construction and validation, not study participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with human participants assigned to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with human participants requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study with participant attrition concerns."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs inference on 31 LLMs across 5,200 questions in 3 input forms, which involves substantial API costs. No inference costs, tokens consumed, or wall-clock time are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget information is provided. The paper does not state the hardware used for running open-source models (LLaMA series) or the total API spend for commercial models."
       }
     }
   },
   "claims": [
     {
       "claim": "In-context learning and RLHF training significantly enhance the model's ability to avoid hallucination on unanswerable math word problems.",
       "evidence": "Figure 3 shows F1 score improvements with ICL input form vs Direct for all model series. RLHF chat models (LLaMA-v2-7b-chat, 13b-chat, 70b-chat) consistently outperform their base counterparts across all three input forms (Section 5.4).",
       "supported": "moderate"
     },
     {
       "claim": "Larger model size generally leads to better hallucination avoidance (higher F1 scores).",
       "evidence": "Figure 3 shows continuous F1 improvement as model size increases in the LLaMA and InstructGPT series, with the exception of text-babbage-001 in InstructGPT (Section 5.4, 'Model Size').",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 achieves the best LLM performance with an F1 score of 85.24% under instruction input, but still falls short of the human benchmark of 93.16%.",
       "evidence": "Figure 4 directly shows these numbers. The human benchmark was established with 5 volunteers on 200 randomly sampled questions (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Combining text similarity templates with mathematical expression detection improves consistency with human judgment compared to template-only evaluation.",
       "evidence": "Table 3 shows Cohen's kappa improvement across 5 models when adding rule-based expression detection: improvements range from +0.027 (gpt-4) to +0.072 (text-davinci-003). All kappa values exceed 0.75.",
       "supported": "strong"
     },
     {
       "claim": "LLaMA-v2-13b-chat's performance can compete with LLaMA-65b despite having significantly fewer parameters.",
       "evidence": "Figure 3 shows LLaMA-v2-13b-chat achieving comparable F1 scores to LLaMA-65b across input forms (Section 5.4, 'RLHF').",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces UMWP, a dataset of 5,200 math word problems (half answerable, half unanswerable) for benchmarking LLM hallucination. Testing 31 LLMs across three input forms reveals that model size, RLHF training, and in-context learning all improve hallucination avoidance, with GPT-4 achieving the best F1 score (85.24%) but still trailing the human benchmark (93.16%). The paper also shows that combining template-based text similarity with mathematical expression detection produces evaluation results more consistent with human judgment (Cohen's kappa > 0.75 for all tested models).",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All experimental results are reported as single-point F1 scores with no error bars, confidence intervals, or variance across runs. Given that LLM outputs can vary with temperature=0.7, single-run results may not be stable."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "The answerable half of UMWP comes directly from GSM8K, SVAMP, MultiArith, and ASDiv, all publicly available before the training cutoffs of most evaluated models. Models that memorized these problems would perform differently on answerable vs novel unanswerable variants, but this is never discussed."
     },
     {
       "flag": "Causal claims from observational comparisons",
       "detail": "The paper attributes F1 improvements specifically to RLHF and model size, but these comparisons confound multiple factors (different training data, architectural changes, training procedures). The claim that 'RLHF substantially improves' performance cannot be isolated from these confounds."
     },
     {
       "flag": "Overly broad title and claims",
       "detail": "The title 'Benchmarking Hallucination in Large Language Models' and the abstract claim that 'MWP is a reliable and effective approach to assess hallucination' suggest general applicability, but the paper only tests one specific type of hallucination (failing to recognize unanswerable math problems) in English."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims about the effects of model size, RLHF, and input forms are supported only by visual comparison of F1 scores, without any statistical tests to confirm the differences are meaningful."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske"],
       "year": 2023,
       "relevance": "Comprehensive survey of hallucination in NLG systems, directly relevant to LLM reliability and evaluation methodology."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Benchmark for evaluating LLM truthfulness, a key methodology for assessing LLM reliability."
     },
     {
       "title": "HaluEval: A large-scale hallucination evaluation benchmark for large language models",
       "authors": ["Junyi Li", "Xiaoxue Cheng", "Wayne Xin Zhao"],
       "year": 2023,
       "relevance": "Large-scale hallucination evaluation benchmark for LLMs, directly comparable methodology for assessing hallucination."
     },
     {
       "title": "FActScore: Fine-grained atomic evaluation of factual precision in long form text generation",
       "authors": ["Sewon Min", "Kalpesh Krishna", "Xinxi Lyu"],
       "year": 2023,
       "arxiv_id": "2305.14251",
       "relevance": "Automated factuality evaluation methodology for LLMs, relevant to benchmarking and evaluation design."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique for LLM reasoning, relevant to understanding how prompting affects LLM capability."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT paper describing RLHF training, directly relevant to understanding alignment and instruction-following capability."
     },
     {
       "title": "A survey of large language models",
       "authors": ["Wayne Xin Zhao", "Kun Zhou", "Junyi Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive LLM survey relevant to understanding the landscape of models evaluated in LLM benchmarking studies."
     },
     {
       "title": "Do large language models know what they don't know?",
       "authors": ["Zhangyue Yin", "Qiushi Sun", "Qipeng Guo"],
       "year": 2023,
       "relevance": "Evaluates LLM self-knowledge using unanswerable questions, directly related to hallucination benchmarking methodology."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K dataset used as source for UMWP, relevant to LLM math reasoning evaluation methodology."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Technical report for GPT-4, the top-performing model in this benchmark, relevant to LLM capability evaluation."
     }
   ]
 }