ai-research-survey

scan.json (28365B)

---

 {
   "paper": {
     "title": "aiXamine: Simplified LLM Safety and Security",
     "authors": [
       "Fatih Deniz",
       "Dorde Popovic",
       "Yazan Boshmaf",
       "Euisuh Jeong",
       "Minhaj Ahmad",
       "Sanjay Chawla",
       "Issa Khalil"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2504.14985"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the aiXamine website (https://aixamine.qcri.org/) where users can view evaluation reports, but no source code repository (e.g., GitHub) is provided for the evaluation platform or pipeline code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses publicly available benchmark datasets (AdvGLUE, SimpleQA, TruthfulQA, etc.) but does not release its own collected evaluation results, model responses, or any new data artifacts. The raw evaluation data is only accessible through the aiXamine website."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions 'multiple Nvidia H100 nodes, each with 80GB of memory' and that the pipeline uses Apache Airflow with Python modules, but no requirements.txt, Dockerfile, or detailed dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the platform architecture (Sections 2 and 4) but does not include commands, scripts, or a guide for replicating the evaluations."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 6-14 are reported as single point estimates (accuracy percentages). No confidence intervals, error bars, or uncertainty measures are provided for any evaluation score."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'ChatGPT-4o achieving the highest overall score', 'open-source models can match or exceed proprietary models') without any statistical significance tests. Comparisons are based solely on numerical differences in accuracy percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While raw accuracy percentages are reported, no standardized effect sizes (Cohen's d, odds ratios, etc.) are provided. The paper notes relative differences (e.g., '15% performance increase when a security policy is included') but this is a raw difference, not a standardized effect size with baseline context throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes of the benchmark datasets used or the number of models evaluated. Table 1 lists sample counts per test (e.g., 576 for AdvGlue, 4326 for SimpleQA), but there is no discussion of whether these are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across evaluation runs. The paper does not state whether evaluations were run multiple times, and all results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares over 15 models against each other across all services (Tables 6-14), including proprietary models (ChatGPT-4o, Gemini 2.0 Flash, Grok 3, Deepseek Chat) and open-source models (Llama, Qwen, etc.), as well as distilled variants. The models serve as baselines for each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include very recent systems: GPT-4o, Gemini 2.0 Flash, Grok 3, DeepSeek Chat, Llama 3.x, Qwen 2.5, and distilled variants from 2024-2025. These are contemporary and competitive models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the aiXamine platform itself to show which components or design decisions matter. The comparisons between models with/without security policies (Section 5.3.2) and normal/augmented PII awareness (Section 5.3.6) are not ablations of the evaluation system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The platform evaluates models across eight distinct services with different metrics: accuracy for most tests, Cramer's V for disparagement (Section 3.3.1), and Pearson correlation for ConfAIde (Section 3.5.2). Results are broken down per service and per test."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluations are automated using judge models (Llama Guard, OpenAI Moderation API, Perspective API, WildGuard) or automated metrics. No human evaluation of model outputs is performed. Section 6.1 explicitly acknowledges this as a limitation: 'automated evaluation methods commonly fail to capture more subtle, contextual dimensions of safety.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All benchmark datasets used are publicly available. Section 6.4 explicitly acknowledges this weakness under 'Private benchmarks': 'Models frequently achieve artificially inflated scores by exploiting known public benchmarks.' No held-out test set is used."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided across all services. Tables 7-14 show per-test and per-category scores (e.g., per programming language for code security, per demographic category for fairness, per transformation type for OOD robustness)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific failure patterns: GPT-4o struggles with MNLI adversarial prompts (Section 1), Grok-3 shows fairness concerns promoting political stances (Section 1), Gemini 2.0 exhibits low PII awareness without privacy policy (Section 1), and R1-distilled models suffer dramatic drops in adversarial robustness (Section 5.3.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that R1-distilled models suffer substantial performance drops across multiple services (Sections 5.3.1, 5.3.4), that scaling alone is insufficient for robustness (Section 5.3.1), and that all models struggle with the Preference test for ideological neutrality (Section 5.3.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about vulnerabilities in GPT-4o (adversarial attacks), Grok-3 (biased outputs), and Gemini 2.0 (privacy weaknesses) are supported by detailed results in Tables 7, 9, and 12 respectively. The claim that open-source models can match proprietary ones in specific services is supported by Table 14 (Safety & Alignment)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about distillation methods: 'its internally generated reasoning data appears to degrade the robustness of other models when used for further fine-tuning' (Section 5.3.1), and 'reasoning mechanisms appear to direct the models toward one of the provided choices' (Section 5.3.3). These are causal claims based on observational comparisons without controlled experiments isolating the effect of distillation from other confounding factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM Safety and Security' and claims like 'comprehensive evaluation' suggest broad generalizability, but evaluations are limited to English-language benchmarks (Section 6.4 acknowledges need for 'support for diverse cultures and languages'), single-turn interactions (Section 6.4 acknowledges need for 'multi-turn attacks'), and specific publicly available benchmark datasets that may not represent real-world safety challenges (Section 6.1)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses limitations of automated evaluation and black-box settings, but does not consider alternative explanations for the observed results. For example, the performance differences between models could be due to benchmark-specific optimization rather than genuine safety differences. The paper does not address whether observed patterns might be artifacts of the specific benchmarks or judge models used."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names like 'ChatGPT-4o', 'Gemini 2.0 Flash', 'Grok 3', 'Deepseek Chat' without specifying API versions, snapshot dates, or specific model identifiers. For open-source models, family names and sizes are given (e.g., 'Llama3.1-8B', 'Qwen2.5-14B') but exact checkpoint versions or HuggingFace model IDs are not provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides example prompts and judge prompt templates throughout: Table 2 shows AdvGLUE task messages and input examples, Prompt 3.4.5 shows HaluEval task messages, Prompt 3.7 shows the Over Refusal judge prompt, Prompt 3.8.1 shows the Llama Guard prompt template, and Prompt 3.8.6 shows the WildGuard prompt template. While not every prompt variation is shown, representative actual prompt text is provided for each test."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions temperature settings for SelfCheckGPT (temperature 0 and temperature 1.0 in Section 3.4.1) and sampling probabilities for OOD data (p=0, p=0.6 in Section 3.6.1), but does not systematically report hyperparameters for model inference (temperature, top-p, max tokens) across evaluations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models in a direct prompt-response manner without agentic scaffolding. The evaluation pipeline (Airflow DAGs described in Section 2) is infrastructure, not agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing and filtering steps are documented for each test. For example, Section 3.8.1 describes filtering BEAVERTAILS prompts to match Llama Guard's risk taxonomy, Section 3.8.2 describes generating questions with GPT-4 and filtering to match Llama Guard 2's taxonomy, and Section 2.3 describes the multi-step approach for handling instruction non-compliance."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) includes dedicated subsections on limitations: Section 6.1 'Limitations of Automated Evaluation', Section 6.2 'Restrictions of the Black-Box Setting', and Section 6.4 'Future Work' identifies additional gaps."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses specific threats: judge model biases and failure to generalize to novel safety issues, public benchmarks enabling artificial score inflation, binary Safe/Unsafe classification oversimplifying complex ethical concerns. Section 6.2 discusses specific challenges of OOD evaluation and backdoor detection in black-box settings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly states what the system does NOT cover: non-English languages and cultural contexts, multi-turn attacks, custom user scenarios, and immature research areas like OOD robustness. Section 6.2 explicitly states the limitations of the black-box setting for backdoor detection and OOD evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw evaluation data (model responses, judge outputs) is not publicly available. The aiXamine website shows reports but the underlying data for independent verification is not downloadable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is well-documented. Each test section (Sections 3.1-3.8) describes the datasets used, their sources, construction methods, and evaluation methodology. Table 1 provides a comprehensive overview of all services, tests, datasets, sample counts, and categories."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study. The evaluation is entirely automated using benchmark datasets and judge models. Models were selected based on Chatbot Arena rankings (Section 5.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The five-stage ETL pipeline is described in Section 4.1: model loading/serving, data extraction, test execution, result parsing/storage, and report creation/update. Section 2.1 describes the DAG-based task execution model. Each test section documents the evaluation pipeline stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are from Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University, but no grants or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all seven authors are from Qatar Computing Research Institute, Hamad Bin Khalifa University. The paper also discloses that Fanar-7B is from the same institution (reference [107] lists several co-authors from QCRI/HBKU)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "QCRI is affiliated with the development of Fanar-7B (reference [107] shares authors with this paper, including Boshmaf and Chawla). The paper evaluates Fanar-7B alongside other models, creating a potential conflict where the funder/institution has an interest in how their own model appears. This is not disclosed as a conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that QCRI developed both aiXamine and Fanar-7B, a conflicts disclosure would be appropriate."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models on benchmarks but does not state training data cutoff dates for any of the evaluated models. This is relevant because many benchmarks (AdvGLUE, TruthfulQA, etc.) are publicly available and could have been in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of potential train/test overlap is provided. Section 6.1 briefly mentions that 'these datasets are often publicly available' and models may achieve 'artificially inflated safety metrics,' but no systematic analysis of contamination is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (AdvGLUE 2021, TruthfulQA 2021, SimpleQA 2024, etc.) were published before the training cutoffs of evaluated models. Section 6.1 acknowledges 'models frequently achieve artificially inflated scores by exploiting known public benchmarks' and Section 6.4 proposes private benchmarks as future work, but no systematic contamination analysis is performed for the current results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations are automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, cost per evaluation, or cost per model are reported. The paper mentions GPU hours but not the monetary cost of running the evaluation suite."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states: 'we utilized approximately 624 Nvidia H100 GPU hours' for open-source models and 'around 494 hours' for API-based evaluations. It also notes 'the full evaluation suite for a single model typically ranges from one to two days.'"
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT-4o achieves the highest overall safety and security score among all evaluated models.",
       "evidence": "Table 6 shows ChatGPT-4o with an overall score of 80.68, the highest among all 15 models evaluated.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source models can match or exceed proprietary models in specific services such as safety alignment, fairness and bias, and OOD robustness.",
       "evidence": "Table 14 shows Fanar-7B (97.89) and ALLaM-7B (97.39) outperforming ChatGPT-4o (96.93) in Safety & Alignment. Table 10 shows IDA-Llama3.1-8B (89.54) outperforming all proprietary models in OOD Robustness.",
       "supported": "strong"
     },
     {
       "claim": "R1-distilled models suffer substantial performance drops in adversarial and OOD robustness compared to base models.",
       "evidence": "Table 7 shows R1-Qwen2.5-14B scoring 7.93 vs Qwen2.5-14B at 22.80 in adversarial robustness. Table 10 shows R1-Qwen2.5-14B at 30.51 vs Qwen2.5-14B at 62.70 in OOD robustness.",
       "supported": "strong"
     },
     {
       "claim": "Explicit privacy guidance significantly improves PII Awareness scores.",
       "evidence": "Table 12 shows PII Awareness Normal vs Augmented scores: Gemini 2.0 Flash improves from 23.57 to 91.43, Deepseek Chat from 59.64 to 100.00, Llama3.1-8B from 45.00 to 100.00.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o is susceptible to adversarial attacks, particularly on Multi-Genre Natural Language Inference.",
       "evidence": "Table 7 shows ChatGPT-4o scores 50.41 on MNLI in AdvGlue and 35.81 on MNLI in AdvGlue++, among the lowest task-specific scores for this model.",
       "supported": "moderate"
     },
     {
       "claim": "Grok-3 promotes certain political stances or ideologies contrary to the expectation of neutrality.",
       "evidence": "Table 9 shows Grok 3 scoring 14.63 on Lifestyle and 13.29 on Ideology in the Preference test, among the lowest scores across all models.",
       "supported": "moderate"
     },
     {
       "claim": "There is on average a 15% performance increase in code security when a security policy is included in the prompt.",
       "evidence": "Table 8 shows SecCodePLT Norm vs Aug columns. The claim appears to be a rough average across models but no exact computation is shown, and some models (e.g., Fanar-7B) show minimal improvement.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "aiXamine is a comprehensive black-box LLM safety and security evaluation platform integrating over 40 tests across eight services. Evaluations of over 50 models show that proprietary models like ChatGPT-4o lead overall, but well-optimized open-source models match or exceed them in specific areas such as safety alignment. R1-distilled models suffer dramatic performance drops in adversarial and OOD robustness, while IDA-distilled models fare better. Explicit privacy policies in system prompts significantly improve PII awareness across all models.",
   "red_flags": [
     {
       "flag": "Potential conflict of interest: institution evaluates own model",
       "detail": "QCRI authors developed both the aiXamine evaluation platform and Fanar-7B (reference [107] shares authors Boshmaf and Chawla). Fanar-7B is included in evaluations and sometimes shown favorably (e.g., highest Safety & Alignment score). This conflict is not disclosed."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across Tables 6-14 are single point estimates with no confidence intervals, error bars, standard deviations, or significance tests. Given that LLM outputs can vary with temperature and other settings, the stability of these results is unknown."
     },
     {
       "flag": "Public benchmark contamination risk unaddressed",
       "detail": "The paper uses publicly available benchmarks (many dating to 2020-2023) to evaluate models trained on web data through 2024-2025. Section 6.1 acknowledges this risk and Section 6.4 proposes private benchmarks as future work, but the current results do not address contamination, potentially inflating scores for well-known benchmarks."
     },
     {
       "flag": "No model version specificity",
       "detail": "Marketing names like 'ChatGPT-4o', 'Gemini 2.0 Flash', and 'Grok 3' are used without API versions or snapshot dates. Model behavior changes across versions, making results non-reproducible at the version level."
     },
     {
       "flag": "Automated judges only — no human validation",
       "detail": "All safety evaluations rely on automated judge models (Llama Guard, OpenAI Moderation, Perspective API, etc.). The paper acknowledges in Section 6.1 that these judges have biases and may miss subtle unsafe content, but no human validation is performed to calibrate the judges' accuracy on the specific evaluation data."
     }
   ],
   "cited_papers": [
     {
       "title": "Decodingtrust: A Comprehensive Assessment of Trustworthiness in GPT Models",
       "authors": ["Boxin Wang", "Weixin Chen", "Hengzhi Pei"],
       "year": 2023,
       "arxiv_id": "2306.11698",
       "relevance": "Comprehensive LLM trustworthiness evaluation framework covering fairness, privacy, and adversarial robustness — directly comparable to aiXamine's evaluation approach."
     },
     {
       "title": "TrustLLM: Trustworthiness in Large Language Models",
       "authors": ["Yue Huang", "Lichao Sun", "Haoran Wang"],
       "year": 2024,
       "arxiv_id": "2401.05561",
       "relevance": "Comprehensive benchmark for evaluating LLM trustworthiness across multiple criteria including toxicity, bias, and PII awareness."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2022,
       "arxiv_id": "2211.09110",
       "relevance": "HELM provides standardized evaluation protocols for LLM capabilities and vulnerabilities, influencing LLM evaluation methodology."
     },
     {
       "title": "PyRIT: A Framework for Security Risk Identification and Red Teaming in Generative AI Systems",
       "authors": ["Gary D. Lopez Munoz", "Amanda J. Minnich", "Roman Lutz"],
       "year": 2024,
       "arxiv_id": "2410.02828",
       "relevance": "Microsoft's flexible red-teaming toolkit for generative AI security evaluation — comparable evaluation platform."
     },
     {
       "title": "CyberSecEval 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models",
       "authors": ["Shengye Wan", "Cyrus Nikolaidis", "Daniel Song"],
       "year": 2024,
       "arxiv_id": "2408.01605",
       "relevance": "Benchmark for evaluating LLM code security across programming languages and CWE categories, used as a key test in aiXamine."
     },
     {
       "title": "WildGuard: Open One-Stop Moderation Tools for Safety Risks, Jailbreaks, and Refusals of LLMs",
       "authors": ["Seungju Han", "Kavel Rao", "Allyson Ettinger"],
       "year": 2024,
       "arxiv_id": "2406.18495",
       "relevance": "Open moderation tool for LLM safety evaluation covering safety risks, jailbreaks, and refusals."
     },
     {
       "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LLM-based safety classifier used as a judge model for safety alignment evaluation in aiXamine."
     },
     {
       "title": "Or-Bench: An Over-Refusal Benchmark for Large Language Models",
       "authors": ["Justin Cui", "Wei-Lin Chiang", "Ion Stoica"],
       "year": 2024,
       "arxiv_id": "2405.20947",
       "relevance": "Benchmark for evaluating LLM over-refusal behavior, relevant to balancing safety and usability."
     },
     {
       "title": "XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models",
       "authors": ["Paul Röttger", "Hannah Rose Kirk", "Bertie Vidgen"],
       "year": 2023,
       "arxiv_id": "2308.01263",
       "relevance": "Test suite for detecting overly conservative safety behaviors in LLMs."
     },
     {
       "title": "SecCodePLT: A Unified Platform for Evaluating the Security of Code GenAI",
       "authors": ["Yu Yang", "Yuzhou Nie", "Zhun Wang"],
       "year": 2024,
       "arxiv_id": "2410.11096",
       "relevance": "Dynamic evaluation platform for LLM-generated code security using unit tests and sandboxed execution."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "R1 distillation method shown to degrade safety and robustness in distilled models, key finding of the aiXamine evaluation."
     },
     {
       "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2021,
       "arxiv_id": "2109.07958",
       "relevance": "Key hallucination benchmark evaluating whether models repeat common misconceptions."
     }
   ]
 }