ai-research-survey

scan.json (34290B)

---

 {
   "paper": {
     "title": "HalluLens: LLM Hallucination Benchmark",
     "authors": [
       "Yejin Bang",
       "Ziwei Ji",
       "Alan Schelten",
       "Anthony Hartshorn",
       "Tara Fowler",
       "Cheng Zhang",
       "Nicola Cancedda",
       "Pascale Fung"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2504.17550",
     "doi": "10.48550/arXiv.2504.17550"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HalluLens introduces a taxonomy distinguishing LLM hallucination (inconsistency with training data or input) from factuality (correctness vs. world knowledge), and proposes three dynamically-generated extrinsic hallucination tasks. Across 13 LLMs, GPT-4o achieves the highest correct answer rate (52.59%) on PreciseWikiQA while Llama-3.1-405B has the lowest hallucination-when-not-refused rate (26.84%) but high false refusal (56.77%), revealing a precision-recall trade-off. The paper demonstrates that TruthfulQA has significant flaws including ~25% potentially incorrect MC1 scores and outdated gold answers. Dynamic test set generation achieves low cross-run variance (<1.01% std dev) while mitigating data leakage.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link on the first page: 'Code: https://github.com/facebookresearch/HalluLens'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses publicly available datasets (GoodWiki, ITIS database, medicine list) and the test sets are dynamically generated via the released code pipeline. The data sources are open-source as noted in Section 3 and Appendix B."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed library version listings are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released on GitHub, the paper itself contains no step-by-step reproduction instructions or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 and Figure 7 show error bars from three runs. Section B.1 reports: 'average standard deviation of 0.64%, 1.01%, and 0.56% for false refusal rate, hallucination rate when not refused, and correct answer rate respectively.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Kendall's τ correlations with statistical significance (p ≤ 0.01) are reported for NonExistentRefusal subtask agreement (Section 3.3.3, Figure 8). However, no significance tests are applied to the main model performance comparisons — claims like 'GPT-4o achieves the highest correct answer scores' are based on comparing point estimates without tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute percentages with full context (e.g., false refusal rate, hallucination rate, correct answer rate in Table 2), allowing direct assessment of effect magnitudes across models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (5,000 for PreciseWikiQA, 250 for LongWiki, 8,000+1,950 for NonExistentRefusal) and the PreciseWikiQA size follows from the 10 difficulty bins × 500 pages structure. However, no formal power analysis or statistical justification for why these sizes are sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across three trials are reported: <1.01% for PreciseWikiQA (Section B.1), '1.85%, 0.95%, 1.20%, 0.84%' for LongWiki metrics (Section B.2). Error bars are shown in Figures 5 and 7."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "13 models from multiple families (Llama, Qwen, Gemma, Mistral, Claude, GPT) are evaluated against each other. Existing benchmarks (TruthfulQA, SimpleQA, HHEM, ANAH, FaithEval) are analyzed for comparison in Sections 4-5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o (2024-08-06), Claude-3-sonnet (2024-03-07), Llama-3.3-70B-Instruct, Qwen2.5, and Gemma-2, which were all contemporary at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "There is no formal ablation study of the benchmark design components. The round-robin entity generation comparison (Figure 9) and difficulty-level analysis (Figure 4, 6) are sensitivity analyses, not ablations of specific design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used per task: PreciseWikiQA uses false refusal rate, hallucination rate when not refused, and correct answer rate (Section 3.1.1). LongWiki adds precision, recall@K, and F1@K (Section 3.2.1). NonExistentRefusal uses false acceptance rate (Section 3.3.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Multiple human evaluations validate the pipeline: 250 samples annotated for gold answer quality (97.2% correct, Section B.1), 440 responses for evaluator agreement (94.77%, Section B.3), 500 claims for LongWiki pipeline validation (76.8% agreement, Section B.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Test sets are dynamically generated at evaluation time and not used for any tuning or selection decisions. No model fine-tuning is performed — all models are evaluated in their released form."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task, subtask (MixedEntities vs GeneratedEntities in Table 4/7), domain (animal, plant, bacteria, medicine in Table 7), difficulty level (10 bins in Figure 4/6), and place frequency (Figure 11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure patterns are discussed qualitatively: Claude models expressing uncertainty while still accepting non-existent entities (Section 3.3.3), Gemma blanket-refusing medicine queries (Table 7), retrieval failures in LongWiki pipeline (15.4% failure rate, Section B.2), and specific TruthfulQA failure examples (Section 5.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The LongWiki evaluation pipeline achieves only 76.8% agreement with human annotators (Section B.2). 5% of claims are not verifiable within Wikipedia. The retrieval step fails 15.4% of the time. These limitations of their own approach are reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims three contributions: (1) clear hallucination taxonomy — supported by Section 2; (2) new extrinsic hallucination tasks with dynamic generation — supported by Section 3 with results in Tables 2-4; (3) comprehensive analysis of existing benchmarks — supported by Sections 4-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal interpretations without adequate justification: 'places with middle-level frequency are closer to the knowledge boundary, causing the model to be uncertain' (Section B.3.2), 'RLHF can incur an alignment tax' (Section 2.3). These are stated as causal mechanisms but are observational interpretations without experimental manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM Hallucination Benchmark' implies broad applicability, but the benchmark tests only English-language models, uses Wikipedia as the primary knowledge proxy, and covers limited domains. The paper notes 'extending this work to other languages is important' (Section 6) but does not bound the main claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically consider alternative explanations for model performance differences. For instance, using Llama-3.1-70B as the judge to evaluate Llama-family models could introduce evaluation bias. The effect of different instruct-tuning strategies on refusal patterns is noted (Section 3.3.3) but not explored as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "A central contribution is distinguishing the proxy (benchmark performance) from the construct (hallucination). The paper explicitly separates hallucination from factuality (Section 2.1), acknowledges that Wikipedia is a proxy for training data (not the actual training data), and notes the 5% gap where claims are not verifiable in Wikipedia."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed with snapshot dates for commercial models: 'Claude-3-haiku (2024-02-29), Claude-3-sonnet (2024-03-07), and GPT-4o (2024-08-06)' (Section 3). Open-source models are specified with exact names and sizes (e.g., 'Llama-3.1-8B-Instruct', 'Qwen2.5-7B-Instruct')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix D for all pipeline stages: question generation (D.1), answerability judgment, hallucination judgment, abstention judgment, LongWiki generation (D.2), NonExistentRefusal evaluation (D.3), and the 10 inference prompt templates (D.3.1)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section B states: 'We evaluate all models under the same decoding setup, using a temperature of zero and top-p of one.' Section 3.2.2 states 'maximum length of 1024 tokens' for LongWiki."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The benchmark directly queries models via standard inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: GoodWiki source selection with harmonic centrality binning (Section 3.1.2), question generation with answerability filtering, answer length filtering (>10 words removed), difficulty control (levels 5-9 for LongWiki), ITIS database for MixedEntities, and Brave Search verification for GeneratedEntities (Section 3.3.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Limitations are discussed in subsections throughout (B.2 discussion, B.3 discussion) but not in a consolidated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: Wikipedia covers only a fraction of training data (5% unverifiable claims, Section B.2), reference retrieval fails 15.4% of the time (Section B.2), LLM-evaluator agreement with humans is 76.8% for LongWiki (Section B.2), and Gemma's blanket medicine refusal biases results (Table 7, Section B.3.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the benchmark does NOT measure or which populations/settings are excluded. The English-only limitation is acknowledged only in passing in the Related Work section, and no explicit scope boundaries are drawn for the main results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the code pipeline is released, the raw model outputs and generated test sets from the reported experiments are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is thoroughly described: GoodWiki dataset of 44,754 Wikipedia pages (Section 3.1.2), ITIS database with 145,000+ taxonomic records (Section 3.3.2), medicine list covering 250,000 drugs, harmonic centrality-based difficulty binning, and entity generation via round-robin with three LLMs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited. Data sources are public databases and benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with counts at each stage: 44,754 GoodWiki pages → 5,000 pages sampled (500 per difficulty bin) → 5,000 QA pairs for PreciseWikiQA; 250 prompts for LongWiki (50 per difficulty level 5-9); 8,000 MixedEntities + 1,950 GeneratedEntities. Filtering criteria (answer length ≤10 words, answerability check) are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure appears in the paper. The acknowledgments thank individuals but do not mention grants, corporate sponsorship, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: 'FAIR at Meta, GenAI at Meta, HKUST' with footnote 'Work done during Internship at FAIR.' Meta employees evaluating Llama models (Meta's product) is evident from the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta researchers evaluate Meta's own Llama models alongside competitors. Meta has a financial interest in Llama models performing well on hallucination benchmarks. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper, despite the clear conflict of Meta employees evaluating Meta products."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates for the evaluated models are not stated. The paper notes GoodWiki uses Wikipedia pages from September 2023 and assumes Wikipedia is in training data, but does not state when each model's training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is a central concern of the paper. Section 1 discusses how 'static test sets are especially vulnerable to obsolescence' and Section 2.4 identifies 'robustness against unintentional data leakage' as a key benchmark criterion. The dynamic generation approach is specifically designed to prevent overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is extensively addressed. The paper introduces dynamic test generation to prevent contamination (Section 3), analyzes contamination of existing benchmarks like TruthfulQA (Section 5.1, noting 'it is now saturated due to inclusion in training data'), and cites Deng et al. (2024) on widespread contamination of benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Human annotations are used for pipeline validation, not as a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — the paper evaluates LLM outputs using automated pipelines with human validation of the pipeline itself."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, tokens consumed, or wall-clock time is reported for running the benchmark, despite evaluating 13 models across thousands of test instances with LLM-based evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The benchmark requires generating dynamic test sets, running inference on 13 models, and running LLM-based evaluation, but the total compute cost is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across three trials with standard deviations. Section B.1: 'average standard deviation of 0.64%, 1.01%, and 0.56%'. Section B.2: 'average standard deviation across the models of 1.85%, 0.95%, 1.20%, 0.84%'. Temperature is set to 0, so variance comes from dynamic test generation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'Each experiment consists of three trials' (Section B). Table 2 caption: 'average of three trials of evaluation.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "No hyperparameter search was performed — fixed standard settings (temperature=0, top-p=1) are used uniformly across all models (Section B), making the search budget transparently zero."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "No configuration selection is performed. All models are evaluated with the same fixed settings (temperature=0, top-p=1), eliminating selection bias."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "13 models are compared across multiple tasks and metrics with no correction for multiple comparisons. Kendall's τ correlations report significance (Figure 8) but without family-wise error rate correction across the many comparisons made."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Meta researchers evaluate Meta's Llama models alongside competitors but do not acknowledge or address the potential bias of evaluating their own models. Additionally, using Llama-3.1-70B as the evaluation judge for all models including Llama variants is not discussed as a potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 7B to 405B parameters with vastly different compute requirements, but performance is not reported as a function of compute budget. No cost-normalized comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "This is a central contribution. Section 2 extensively discusses what hallucination means vs factuality, Section 5 analyzes construct validity of TruthfulQA (showing it measures factuality, not hallucination), and Section 5.2 discusses when factuality benchmarks can/cannot serve as hallucination benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used — models are directly queried via standard inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is a central concern. The paper introduces dynamic test generation specifically to prevent it (Section 3): 'To reduce the risk of test sets being memorized or leaked, we dynamically generate new test questions during evaluation.' The GoodWiki source date (September 2023) is disclosed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., the phrasing of prompts, the structure of questions) could leak information about expected answers to the tested models."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether generated test questions share structural patterns that could correlate with training data exposure. Questions are drawn from Wikipedia pages that may have overlapping content or similar structures."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Dynamic test generation serves as a concrete leakage prevention method. For NonExistentRefusal, entities are verified via Brave Search API to confirm non-existence (Section 3.3.2). The paper also demonstrates leakage in TruthfulQA as evidence for the need for dynamic approaches."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing hallucination benchmarks conflate hallucination with factuality, requiring distinct evaluation approaches with separate taxonomies.",
       "evidence": "Section 2 provides a detailed taxonomy distinguishing extrinsic/intrinsic hallucination from factuality. Section 5.1 demonstrates TruthfulQA is primarily a factuality benchmark, with ~200 samples (~25% of test set) scored as incorrect by MC1 that could be factually correct.",
       "supported": "strong"
     },
     {
       "claim": "Dynamic test set generation produces stable evaluation results with low variance across runs.",
       "evidence": "Three trials show average std dev of 0.64%/1.01%/0.56% for PreciseWikiQA metrics (Section B.1), 1.85%/0.95%/1.20%/0.84% for LongWiki metrics (Section B.2), and consistent rankings across NonExistentRefusal trials (Figure 7).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o achieves the highest correct answer rate (52.59%) on PreciseWikiQA among tested models.",
       "evidence": "Table 2 shows GPT-4o at 52.59% correct answer rate, 4.13% false refusal, and 45.15% hallucination when not refused, averaged over three trials.",
       "supported": "moderate"
     },
     {
       "claim": "Larger models within the same family tend to hallucinate less and refuse less frequently.",
       "evidence": "Tables 2-4 show trends: Llama-3.1 8B→70B→405B shows decreasing hallucination rates (48.37%→37.30%→26.84%). Qwen 7B→14B and Gemma 9B→27B show similar patterns. However, Llama-3.3-70B deviates with higher false acceptance than 3.1-405B (40.82% vs 6.88%).",
       "supported": "moderate"
     },
     {
       "claim": "TruthfulQA contains inaccurate gold answers and the MC1 log-probability metric produces misleading results.",
       "evidence": "Section 5.1 provides specific examples: incorrect gold answers (kindergarten importance, cannabis legality in Asia), subjective/ambiguous prompts ('big' oil reserves), and the MC1 tone-matching problem where correct model responses score as incorrect because the false option matches the model's generation tone better.",
       "supported": "strong"
     },
     {
       "claim": "Models' false acceptance rate varies with knowledge boundary proximity — middle-frequency places show the highest false acceptance rates.",
       "evidence": "Figure 11 shows the pattern across N-gram frequency groups (low, middle, high). The explanation is that middle-frequency places are closer to the model's knowledge boundary, making the model uncertain (Section B.3.2).",
       "supported": "moderate"
     },
     {
       "claim": "The automatic LLM evaluator achieves high agreement with human judgments across all tasks.",
       "evidence": "Evaluator accuracy: 96.67% for abstention judgment and 95.56% for correctness judgment in PreciseWikiQA (Section B.1), 94.77% agreement for NonExistentRefusal (Section B.3), but only 76.8% for LongWiki claim verification (Section B.2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest — Meta evaluating Meta models",
       "detail": "Meta (FAIR/GenAI) researchers evaluate Meta's Llama models alongside competitors. Llama-3.1-405B achieves the best false acceptance rates on NonExistentRefusal (6.88% average), but the conflict is never acknowledged."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Llama-3.1-70B-Instruct is used as the evaluation judge across all tasks (Table 6). This same-family model judges other Llama variants, creating potential bias from shared training data or output style preferences. The paper validates judge accuracy on human annotations but does not test for family-specific bias."
     },
     {
       "flag": "Selective model coverage",
       "detail": "Only Claude-3-haiku and Claude-3-sonnet are included — not Claude-3-Opus or Claude-3.5-Sonnet (available by April 2025). No Gemini models are tested despite being a major competitor. This selection could favor Meta models by excluding stronger competitors."
     },
     {
       "flag": "No significance tests for main model comparisons",
       "detail": "Performance differences between 13 models are presented as rankings based on point estimates (averaged over 3 runs) without any statistical significance tests. Small differences may not be meaningful."
     },
     {
       "flag": "LongWiki evaluation pipeline has low agreement",
       "detail": "The LongWiki verification pipeline agrees with human annotators only 76.8% of the time (Section B.2), and the retrieval step fails to find relevant evidence 15.4% of the time. Results from this task should be interpreted with caution."
     }
   ],
   "cited_papers": [
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske"],
       "year": 2023,
       "relevance": "Foundational hallucination taxonomy (intrinsic/extrinsic) that HalluLens builds upon and extends to the LLM context."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["Lei Huang", "Weijiang Yu"],
       "year": 2023,
       "arxiv_id": "2311.05232",
       "relevance": "Major LLM hallucination survey that expanded the definition to include factual errors — HalluLens argues this conflation is problematic."
     },
     {
       "title": "Siren's song in the AI ocean: a survey on hallucination in large language models",
       "authors": ["Yue Zhang", "Yafu Li"],
       "year": 2023,
       "arxiv_id": "2309.01219",
       "relevance": "Another LLM hallucination survey with a different taxonomy that HalluLens critiques for conflating hallucination with factuality."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Widely-used benchmark that HalluLens critically analyzes, revealing flawed gold answers and metric limitations."
     },
     {
       "title": "Measuring short-form factuality in large language models",
       "authors": ["Jason Wei"],
       "year": 2024,
       "arxiv_id": "2411.04368",
       "relevance": "SimpleQA benchmark for factuality that HalluLens compares against and discusses as a potential hallucination proxy with metric modifications."
     },
     {
       "title": "The dawn after the dark: An empirical study on factuality hallucination in large language models",
       "authors": ["Junyi Li", "Jie Chen"],
       "year": 2024,
       "relevance": "HaluEval 2.0 benchmark that HalluLens analyzes as a 'factuality hallucination' benchmark overlapping factuality and extrinsic hallucination."
     },
     {
       "title": "FActScore: Fine-grained atomic evaluation of factual precision in long form text generation",
       "authors": ["Sewon Min", "Kalpesh Krishna"],
       "year": 2023,
       "relevance": "Foundational framework for claim-level factuality evaluation that HalluLens adapts for its LongWiki evaluation pipeline."
     },
     {
       "title": "VeriScore: Evaluating the factuality of verifiable claims in long-form text generation",
       "authors": ["Yixiao Song", "Yekyung Kim", "Mohit Iyyer"],
       "year": 2024,
       "relevance": "Improved claim extraction and verification method that HalluLens adopts for its LongWiki evaluation pipeline."
     },
     {
       "title": "Long-form factuality in large language models",
       "authors": ["Jerry Wei"],
       "year": 2024,
       "arxiv_id": "2403.18802",
       "relevance": "SAFE evaluator and LongFact benchmark for long-form factuality that HalluLens compares against, noting its reliance on internet search makes it factuality-focused."
     },
     {
       "title": "FaithEval: Can your language model stay faithful to context",
       "authors": ["Yifei Ming"],
       "year": 2024,
       "relevance": "Intrinsic hallucination benchmark testing faithfulness to noisy or contradictory contexts, included in HalluLens as an existing intrinsic evaluation task."
     },
     {
       "title": "ANAH-v2: Scaling analytical hallucination annotation of large language models",
       "authors": ["Yuzhe Gu", "Ziwei Ji"],
       "year": 2024,
       "relevance": "Intrinsic hallucination benchmark with reference-based QA evaluation, included as an existing task in HalluLens."
     },
     {
       "title": "Investigating data contamination in modern benchmarks for large language models",
       "authors": ["Chunyuan Deng", "Yilun Zhao"],
       "year": 2024,
       "relevance": "Demonstrates widespread benchmark contamination in LLM training data, motivating HalluLens's dynamic test generation approach."
     },
     {
       "title": "Survey on factuality in large language models: Knowledge, retrieval and domain-specificity",
       "authors": ["Cunxiang Wang"],
       "year": 2023,
       "arxiv_id": "2310.07521",
       "relevance": "Factuality survey that highlights the importance of distinguishing factuality from hallucination, supporting HalluLens's taxonomic contribution."
     },
     {
       "title": "A multitask, multilingual, multimodal evaluation of ChatGPT on reasoning, hallucination, and interactivity",
       "authors": ["Yejin Bang"],
       "year": 2023,
       "relevance": "Early systematic evaluation of ChatGPT hallucination across tasks, by the same first author, establishing that LLMs generate content from internal knowledge."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners can use the released benchmark suite to evaluate hallucination in their LLM deployments, and the dynamic generation prevents gaming."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the widely-held conflation of hallucination with factuality and demonstrates TruthfulQA has ~25% problematic scores — contrarian to common benchmark usage."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Hallucination undermines AI system trust, but this paper measures the problem rather than demonstrating a new attack or existential risk."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The TruthfulQA critique has mild drama ('contains incorrect gold answers'), but the paper maintains a measured academic tone."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released at github.com/facebookresearch/HalluLens — practitioners can run the benchmark, though setup effort is needed."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Meta FAIR, a well-known AI lab. Published at ACL, a top NLP venue. Evaluates popular models (GPT-4o, Claude, Llama)."
     }
   }
 }