ai-research-survey

scan.json (30339B)

---

 {
   "paper": {
     "title": "AEGIS2.0: A Diverse AI Safety Dataset and Risks Taxonomy for Alignment of LLM Guardrails",
     "authors": [
       "Shaona Ghosh",
       "Prasoon Varshney",
       "Makesh Narsimhan Sreedhar",
       "Aishwarya Padmakumar",
       "Traian Rebedea",
       "Jibin Rajan Varghese",
       "Christopher Parisien"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2501.09004"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states in the Ethics section: 'We will soon release our dataset and models under a commercial-permissive license.' This is a promise of future release, not an actual release. The paper references the llama-recipes repository for training but no code for the AEGIS2.0 pipeline itself is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'We plan to open-source AEGIS2.0 data and models to the research community,' and the Ethics section confirms access 'will be carefully monitored.' At the time of the paper, the dataset was not yet released — this is a stated intention, not an actual release."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4.4 specifies the training environment: 8 x A100 GPUs with PyTorch FSDP, the llama-recipes repository, LoRA r=16, alpha=32, batch size=4, learning rate 1e-4 with CosineAnnealingWarmRestarts scheduler. This is sufficient to recreate the training environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While hyperparameters and hardware are described, no step-by-step reproduction instructions are provided. The dataset and code are not yet released, making reproduction impossible regardless of instructions. No README or script-level guidance is given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports standard deviations in parentheses for all results across the ablation study (e.g., 'catdesc prompt: 0.761(0.005)'). Table 3 mentions results are averaged over three runs, and Table 7 explicitly confirms standard deviations over 3 random seeds."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used when claiming that LLAMA3.1-AEGISGUARD outperforms baselines like LLAMAGUARD3-8B. Comparisons are made based on F1 score differences with no p-values, t-tests, or other statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw F1 scores for each model and condition but does not compute formal effect sizes (Cohen's d, odds ratios, percentage improvement). The reader can calculate differences from Table 3, but the paper itself does not explicitly report effect sizes or contextualize the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test split of 1,984 samples is described as selected via stratified sampling (Section 4) but no power analysis or justification for this specific sample size is provided. The choice of 3 random seeds for averaging is also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports standard deviations across 3 random seed trials for all ablation results (e.g., 'catlist+ prompt: 0.759(0.009)'). The paper explicitly states 'Mean harmfulness F1 scores reported over 3 random seeds with standard deviation in parenthesis.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared including LLAMAGUARD3-8B, LLAMAGUARD3-1B, LLAMAGUARD2-8B, OPENAI MOD API, BEAVERDAM, and WILDGUARD in Tables 3 and 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2023-2024, contemporary with the paper (2025). LLAMAGUARD3-8B (Llama 3.1 family, 2024), WILDGUARD (2024), and ShieldGemma (2024) are among the most recent models in this space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "A thorough ablation study in Table 7 (Appendix A.5) isolates the effect of prompt formats (catdesc/catlist/catlist+), refusal data, LLM jury labels vs. conversation labels, and the source of response labels. Section 5 and Appendix A.5 describe these ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports harmfulness F1 scores across multiple benchmarks (WILDGUARDTEST, XSTest, OpenAI Moderation Dataset) and also category prediction accuracy (94% reported in Section 5.1). Both binary safety and multi-class category prediction are evaluated."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates entirely via automated benchmarks (F1 against ground truth labels on WGTEST, XSTEST, OAI Mod). Human evaluation of the model's safety classifications on novel inputs is not performed. For a content moderation model, human evaluation of outputs is relevant — the model's safety judgments could be assessed by human reviewers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly describes: 'we provide a train-test split of the dataset, by selecting 1,984 samples for testing via stratified sampling.' Models are evaluated on the held-out AEGIS2.0 test set (Table 4) and on external out-of-domain benchmarks (Table 3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix A.6 provide per-category breakdowns including heatmaps (Figure 1) showing ground truth vs. predictions across all OpenAI taxonomy categories, and Figure 4 showing WILDGUARDTEST category distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative examples of model misclassifications are shown. No error analysis of specific cases where AEGISGUARD fails. The Limitations section discusses dataset-level weaknesses (single response model, category imbalance) but not specific failure patterns of the trained model. The heatmap in Figure 1 shows some misclassification patterns at the category level but no qualitative failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) shows that the catdesc prompt style 'is not consistently beneficial over the catlist and catlist+ style prompts' (Section A.5.1), which is an unexpected and negative result explicitly acknowledged. The paper also reports that adding refusal data does not consistently improve all metrics."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that lightweight models trained on AEGIS2.0 achieve performance competitive with WILDGUARD. Tables 3 and 4 support this: LLAMA3.1-AEGISGUARD achieves 0.808 vs. WILDGUARD's 0.828 unweighted average. The abstract appropriately uses 'competitive with' rather than 'surpasses.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'adding fine-grained categories improves performance') which are justified by controlled single-variable ablations in Table 7. The ablation design manipulates one variable at a time (prompt format, refusal data, jury labels), allowing causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is appropriately bounded: it tests LLAMA3.1-8B-INSTRUCT as the base model and evaluates on English-language safety data. The Limitations section explicitly notes the English-only focus and single response model (Mistral-7B-v0.1). The paper does not claim universal coverage."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss substantive alternative explanations for its main results. The competitive performance with WILDGUARD could be partly explained by base model knowledge (zero-shot baseline is already 0.738), benchmark contamination, or overlap between AEGIS2.0 training sources and test benchmarks. The catdesc discussion is about an internal design choice, not an alternative explanation for the main findings. The Limitations section mentions LLM jury bias but does not connect it to explaining experimental outcomes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model versions are specified throughout: LLAMA3.1-8B-INSTRUCT, LLAMAGUARD3-8B, LLAMAGUARD3-1B, LLAMAGUARD2-8B, Mixtral-8x22B-v0.1, Mistral-NeMo, Gemma-2-27B-it, Mistral-7B-v0.1. These are specific named versions, not just generic model families."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.12 provides the full system prompt templates (catlist, catlist+, catdesc styles) with complete text including taxonomy definitions and output format. The {prompt} and {response} placeholders represent input data to classify (user messages and LLM responses), not prompt engineering decisions. The actual prompt engineering — system instructions, taxonomy, output format — is fully specified. The annotator prompt templates for the LLM jury are also shown in Figure 8."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4.4 reports: AnyPrecisionAdamW optimizer, initial learning rate 1e-4, CosineAnnealingWarmRestarts with T_0=0.2*training_length and T_mult=2, LoRA r=16 alpha=32, 3-4 epochs, batch size 4, 8 x A100 GPUs with PyTorch FSDP."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this work. The trained AEGISGUARD model is a single-pass classifier, not an agent with tools, retry logic, or memory. The jury-of-LLMs is a data labeling pipeline (ensemble voting), not agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the data pipeline: prompt sources (Anthropic/hh-rlhf, DAN, AART, Do-Not-Answer), response generation with Mistral-7B-v0.1, human annotation at dialogue level, jury-of-LLMs for response labels, and refusal data generation. Section 4.1 describes the annotation QA process with 10-15% auditing per chunk."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is explicitly titled 'Limitations' and provides substantive discussion covering: single response model limitation, category imbalance, LLM jury bias, English-only coverage, and annotator bias. This is a dedicated section with multi-paragraph discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: 'use of the LLM-Jury annotations introduces potential biases inherent to the models themselves' with specific reference to gender, race, and cultural biases; underrepresentation of Sexual (minor) and Threat categories shown by Figure 7 distribution; 12 US-based annotators may not represent global cultural norms."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states 'AEGIS2.0 also lacks robust multilingual support' and 'the dataset primarily focuses on English-language data.' Section A.1 states 'We do not claim that our taxonomy and safety policy are comprehensive, and that the model trained with this mitigates all potential risks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is not yet released at the time of the paper. The Ethics section states 'We will soon release our dataset and models' with monitored access. No current download link or archive is provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes data collection thoroughly: prompts sourced from Anthropic/hh-rlhf, DAN (Shen et al., 2024), AART (Radharapu et al., 2023), and Do-Not-Answer (Wang et al., 2023). Responses generated using Mistral-7B-v0.1 and Gemma-2-27B for refusals. Section 4.1 details the annotation team of 12 annotators."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The Ethics section describes annotator selection: 12 annotators in the US from various ethnic/religious backgrounds, 4 from engineering backgrounds and 8 from creative writing. Annotators 'were asked to join on a volunteer basis based on their skill level, availability, and willingness to expose themselves to potentially toxic content.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: prompt selection from source datasets → Mistral-7B-v0.1 response generation → dialogue-level human annotation (3 annotations per sample) → QA auditing (10-15% per chunk) → jury-of-LLMs for response-level labels → majority voting → ternary-to-binary label conversion. Final dataset: 34,248 samples including 16,880 standalone prompts and 17,368 prompt-response pairs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grants are mentioned in the paper. All authors are affiliated with NVIDIA, but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All seven authors list NVIDIA as their affiliation on the title page. The paper evaluates LLM safety models, not NVIDIA's own commercial products specifically, though NVIDIA has commercial interest in LLM safety tooling."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are NVIDIA employees, and NVIDIA has commercial interest in LLM safety products (including NeMo Guardrails, referenced in the paper). The work directly develops tools that NVIDIA may commercialize, creating a non-independent relationship between the organization and the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Given all authors are NVIDIA employees working on a product NVIDIA intends to release commercially, the absence of any financial interests disclosure is a gap."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates models including LLAMA3.1-8B-INSTRUCT and other LLMs on safety benchmarks, but does not state the training data cutoff for any of these models. The evaluation benchmarks (WILDGUARDTEST, XSTest, OpenAI Moderation Dataset) may have been in the training data of base models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of potential train/test overlap is provided. LLAMA3.1-8B-INSTRUCT is used as a base model, and benchmarks like XSTest (2023) and the OpenAI Moderation Dataset (2023) may have been in the pre-training corpus of Llama 3.1, but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not address whether XSTest or the OpenAI Moderation Dataset were available before the training cutoff of LLAMA3.1-8B-INSTRUCT. The base model's training data composition is not discussed in relation to the evaluation benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are professional data labelers performing a commercial annotation task, not research participants in a study. The paper studies the resulting dataset and models, not annotator behavior. Pre-registration applies to studies where hypotheses concern human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are professional workers performing data labeling, not research subjects. The ethical safeguards described (Adult Content Acknowledgement, wellbeing checks) are occupational health measures, not IRB-governed research protocols. IRB applies to research involving human subjects."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Ethics section reports: 12 annotators, all US-based, various ethnic and religious backgrounds, 4 from engineering backgrounds and 8 from creative writing/linguistics. Gender distribution is not explicitly reported, but age and social status diversity are mentioned."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "The Ethics section states annotators 'were asked to join on a volunteer basis based on their skill level, availability, and willingness to expose themselves to potentially toxic content.' This describes inclusion criteria. All must have 'been extensively trained in working with Large Language Models.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human study is an annotation task, not an experimental study with condition assignment. Randomization of condition assignment is not applicable as all annotators perform the same annotation task."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Ethics section states: 'This tooling technology allows for large sets of data to be analyzed by individual annotators without seeing the work of their peers. This is essential in preventing bias between annotators.' Each annotator is blinded to others' annotations, with Label Studio used to enforce this."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states they 'averaged twelve annotators at any given time' over six months but does not report how many annotators started, whether any dropped out, or attrition rates. The 'at any given time' phrasing implies there may have been turnover that is not explicitly reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost (API cost, tokens consumed, latency) is reported for either the jury-of-LLMs labeling system or the trained LLAMA3.1-AEGISGUARD model. Training time is mentioned (15 minutes per epoch on 8 x A100s) but inference cost is absent."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4.4 states: 'we used 8 x A100 GPUs with PyTorch FSDP enabled, with a batch size of 4 and packing enabled. The training time is about 15 minutes per epoch on this setup.' This gives a compute budget estimate for reproduction."
       }
     }
   },
   "claims": [
     {
       "claim": "Parameter-efficient fine-tuning on AEGIS2.0 using LLAMA3.1-8B-INSTRUCT surpasses LLAMAGUARD3-8B and performs at par with the state-of-the-art WILDGUARD model.",
       "evidence": "Table 3 shows LLAMA3.1-AEGISGUARD achieves 0.808 unweighted average F1 vs. LLAMAGUARD3-8B (0.764) and WILDGUARD (0.828). Mean over 3 random seeds.",
       "supported": "moderate"
     },
     {
       "claim": "Including fine-grained risk categories in the prompt template improves binary safe/unsafe classification performance compared to core categories alone.",
       "evidence": "Table 7 (Appendix A.5.1): catlist+ (0.803) outperforms catlist (0.787) on unweighted average F1. The paper attributes this to fine-grained risks like phishing and malware that appear in WILDGUARDMIX but not in core categories.",
       "supported": "strong"
     },
     {
       "claim": "Using jury-of-LLM annotations for response labels substantially boosts response classification performance over using conversation-level annotations.",
       "evidence": "Table 7: removing LLM Jury labels drops response classification F1 on WGTEST from 0.771 to 0.511 and XSTest from 0.847 to 0.521 — a dramatic degradation.",
       "supported": "strong"
     },
     {
       "claim": "Training on a combined blend of topic-following and safety data improves adaptability to new risk categories not seen during training.",
       "evidence": "Table 5: LLAMA3.1-AEGISGUARD + TF improves on all four new categories (Financial 0.748 vs. 0.722, Legal 0.890 vs. 0.875, Medical 0.941 vs. 0.895, NSFW 0.772 vs. 0.699) compared to the base model.",
       "supported": "moderate"
     },
     {
       "claim": "AEGIS2.0 is the first content moderation training dataset fully suitable for commercial use.",
       "evidence": "Table 1 shows AEGIS2.0 is the only dataset among XSTest, OpenAI Mod, HarmBench, ToxicChat, WILDGUARDMIX, and BeaverTails to have commercial use for training. The reasoning for others' ineligibility is provided in Section 2.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "AEGIS2.0 is a 34,248-sample LLM safety dataset with human annotations across a 12-category taxonomy (plus 9 fine-grained subcategories), designed for commercial use. A lightweight model (LLAMA3.1-AEGISGUARD) fine-tuned via PEFT on this dataset achieves F1 scores competitive with WILDGUARD on standard safety benchmarks while using only open-source, commercially licensed models for weak supervision. Adding topic-following training data improves robustness and zero-shot generalization to novel safety categories not seen during training. Ablation studies show that including fine-grained risk categories in the prompt taxonomy and using LLM jury labels for response annotations are both important for strong performance.",
   "red_flags": [
     {
       "flag": "self-evaluation by product developer",
       "detail": "All seven authors are NVIDIA employees, and the paper develops safety tools that NVIDIA plans to release commercially (under monitored access). There is no competing interests statement. The evaluation shows NVIDIA's model favorably against competitors, but this conflict is not disclosed."
     },
     {
       "flag": "dataset not yet released at publication",
       "detail": "The abstract states 'We plan to open-source AEGIS2.0 data and models' and the Ethics section says 'We will soon release our dataset and models.' Independent verification of the dataset quality, annotation accuracy, and category distribution is impossible until the data is actually released."
     },
     {
       "flag": "no significance testing for competitive comparisons",
       "detail": "The key claim that LLAMA3.1-AEGISGUARD 'surpasses LLAMAGUARD3-8B' is based on F1 score differences (0.808 vs. 0.764) without statistical significance tests. Given the variance reported in Table 7, some of these differences may not be statistically robust."
     },
     {
       "flag": "benchmark contamination not addressed",
       "detail": "The paper uses LLAMA3.1-8B-INSTRUCT as a base model and evaluates on XSTest (2023) and OpenAI Moderation Dataset (2023). Both benchmarks were publicly available before Llama 3.1's training cutoff, but no contamination analysis is provided. The zero-shot baseline in Table 7 (0.738 unweighted average) suggests the base model has substantial prior knowledge of these benchmarks."
     },
     {
       "flag": "single response model limits dataset diversity",
       "detail": "All non-refusal responses in the dataset are generated by a single model (Mistral-7B-v0.1). The paper acknowledges this in Section 8 but does not test how models trained on this data perform when the deployment LLM differs significantly from Mistral-7B-v0.1 in its response characteristics."
     }
   ],
   "cited_papers": [
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": [
         "Hakan Inan",
         "Kartikeya Upasani",
         "Jianfeng Chi"
       ],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Foundational content moderation LLM that AEGIS2.0 directly competes with and improves upon."
     },
     {
       "title": "WildGuard: Open one-stop moderation tools for safety risks, jailbreaks, and refusals of LLMs",
       "authors": [
         "Seungju Han",
         "Kavel Rao",
         "Allyson Ettinger"
       ],
       "year": 2024,
       "arxiv_id": "2406.18495",
       "relevance": "Primary baseline dataset and model that AEGIS2.0 is compared against throughout the paper."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": [
         "Jiaming Ji",
         "Mickel Liu",
         "Josef Dai"
       ],
       "year": 2024,
       "relevance": "Comparable human-annotated safety dataset with 14-category taxonomy that AEGIS2.0 improves upon in commercial usability."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": [
         "Mantas Mazeika",
         "Long Phan",
         "Xuwang Yin"
       ],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Safety evaluation benchmark used for comparison context in the related work section."
     },
     {
       "title": "ToxicChat: Unveiling hidden challenges of toxicity detection in real-world user-AI conversation",
       "authors": [
         "Zi Lin",
         "Zihan Wang",
         "Yongqi Tong"
       ],
       "year": 2023,
       "arxiv_id": "2310.17389",
       "relevance": "Earlier content moderation training dataset that AEGIS2.0 addresses limitations of regarding commercial licensing."
     },
     {
       "title": "ShieldGemma: Generative AI content moderation based on Gemma",
       "authors": [
         "Wenjun Zeng",
         "Yuchi Liu",
         "Ryan Mullins"
       ],
       "year": 2024,
       "arxiv_id": "2407.21772",
       "relevance": "Contemporary content moderation model trained on closed dataset, representing the state of the art AEGIS2.0 competes with."
     },
     {
       "title": "XSTest: A test suite for identifying exaggerated safety behaviours in large language models",
       "authors": [
         "Paul Röttger",
         "Hannah Rose Kirk",
         "Bertie Vidgen"
       ],
       "year": 2023,
       "arxiv_id": "2308.01263",
       "relevance": "Evaluation benchmark used to test for over-refusal behavior in safety models."
     },
     {
       "title": "NeMo Guardrails: A toolkit for controllable and safe LLM applications with programmable rails",
       "authors": [
         "Traian Rebedea",
         "Razvan Dinu",
         "Makesh Sreedhar"
       ],
       "year": 2023,
       "arxiv_id": "2310.10501",
       "relevance": "Production guardrails framework that AEGIS2.0 is designed to support, motivating the need for categorical predictions."
     },
     {
       "title": "AEGIS: Online adaptive AI content safety moderation with ensemble of LLM experts",
       "authors": [
         "Shaona Ghosh",
         "Prasoon Varshney",
         "Erick Galinkin",
         "Christopher Parisien"
       ],
       "year": 2024,
       "arxiv_id": "2404.05993",
       "relevance": "Direct predecessor paper (AEGIS 1.0) that AEGIS2.0 extends with larger dataset, improved taxonomy, and better models."
     },
     {
       "title": "Introducing v0.5 of the AI safety benchmark from MLCommons",
       "authors": [
         "Bertie Vidgen",
         "Adarsh Agrawal"
       ],
       "year": 2024,
       "arxiv_id": "2404.12241",
       "relevance": "Industry safety benchmark that informed the AEGIS2.0 taxonomy design."
     },
     {
       "title": "Do-Not-Answer: A dataset for evaluating safeguards in LLMs",
       "authors": [
         "Yuxia Wang",
         "Haonan Li",
         "Xudong Han"
       ],
       "year": 2023,
       "arxiv_id": "2308.13387",
       "relevance": "One of the source datasets from which prompts were sampled to build AEGIS2.0."
     },
     {
       "title": "AART: AI-assisted red-teaming with diverse data generation for new LLM-powered applications",
       "authors": [
         "Bhaktipriya Radharapu",
         "Kevin Robinson",
         "Lora Aroyo"
       ],
       "year": 2023,
       "arxiv_id": "2311.08592",
       "relevance": "Source of adversarial prompts incorporated into the AEGIS2.0 dataset."
     }
   ]
 }