ai-research-survey

scan.json (32281B)

---

 {
   "paper": {
     "title": "Refining Input Guardrails: Enhancing LLM-as-a-Judge Efficiency Through Chain-of-Thought Fine-Tuning and Alignment",
     "authors": [
       "Melissa Kazemi Rad",
       "Huy Nghiem",
       "Andy Luo",
       "Sahil Wadhwa",
       "Mohammad Sorower",
       "Stephen Rawls"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.13080",
     "doi": "10.48550/arXiv.2501.13080"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Supervised fine-tuning (SFT) with only 400 training examples significantly improves LLM performance as input moderation guardrails, with maximum F1 lift of 227% and ADR lift of 344% across four LLMs. DPO and KTO alignment provide marginal additional gains over SFT alone. The best model (Llama3-DPO, F1=96.1) substantially outperforms LlamaGuard-2 (F1=69.2) and DeBERTaV3 (F1=81.4) on the authors' curated test set. Qualitative analysis reveals base LLMs fail to recognize adversarial patterns like token-level jailbreaks, while fine-tuned versions detect them at the cost of slightly increased false positive rates (0.1% → 1.6%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. No mention of code release plans."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The curated training/test dataset is not released. While the source datasets (AdvBench, MaliciousInstruct, Forbidden Question Set, Jailbreak Prompt Set) are public, the authors' specific selections, splits, annotations, and synthetically generated negative examples are not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D mentions AWS P4D.24XLARGE with 8 A100 40GB GPUs, 96 vCPUs, and 1152 GiB memory, but no software environment details (Python version, library versions, requirements.txt, Dockerfile) are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While training hyperparameters are listed in Table 3, the overall pipeline cannot be reproduced without code, data, and instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (F1, ADR, FPR, Invalid Response Ratio) are reported as point estimates in Figure 3 and Table 1 with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'SFT leads to the most significant lift' and compares multiple methods, but no statistical significance tests (p-values, t-tests, bootstrap) are used. All comparisons are based on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context, e.g., 'maximum lift of 227% and 344% in F1 and ADR, respectively' and shows baseline vs. tuned values. Table 1 shows Llama3-DPO (ADR 93.3) vs LlamaGuard-2 (ADR 54.2), giving readers context for the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training set of 400 and test set of ~6800 examples are stated but never justified. No power analysis or reasoning for why these sizes are adequate for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple experimental runs, standard deviations, or variance measures. Results appear to be from single runs for each configuration."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against base (untuned) versions of all four LLMs, and additionally compares the best model (Llama3-DPO) against LlamaGuard-2, ProtectAI DeBERTaV3, and Meta PromptGuard in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include LlamaGuard-2 (2024), PromptGuard (2024), and DeBERTaV3 — all contemporary guardrail models. The base LLMs themselves (Llama3 8B Instruct, Mistral 7B v0.2) are also recent."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares Base → SFT → SFT+DPO and SFT+KTO across all four LLMs, effectively ablating the contribution of each training stage. Figure 3 shows these comparisons across all metrics."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are reported: F1 Score, Attack Detection Ratio (ADR/recall), False Positive Rate (FPR), and Invalid Response Ratio (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the guardrail system's outputs is conducted. Human annotation was used for creating training data (Table 4, Appendix F), but humans did not evaluate the system's test-time outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly separates 400 training and ~6800 test examples: 'any type of query used in the training dataset is excluded from the test set' (Appendix C)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides ADR breakdowns for three query types: standalone jailbreak, malicious queries with jailbreak prompts, and standalone malicious queries, across all LLMs and tuning strategies."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 5-7 and Appendix H provide qualitative analysis of failure cases: Table 5 shows base model failures on adversarial queries, Table 7 shows false positives from the KTO-aligned model, including users with disabilities being incorrectly flagged."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports FPR increases as a negative outcome (0.1% to 1.6% for Llama3-KTO), and Table 7 explicitly shows false positive examples where the KTO model is 'hyper-sensitive.' They note 'surprisingly a relatively small increase in valid queries incorrectly classified as fraudulent.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significantly enhance the safety of conversational AI systems' and 'potential of alignment processes tailored to a varied range of harmful input queries, even with constrained data resources.' These are supported by the experimental results showing large F1/ADR improvements (Figure 3) with only 400 training examples."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims like 'SFT leads to the most significant lift' through a controlled sequential design: Base → SFT → SFT+DPO/KTO. Each step adds one component while holding others fixed, constituting adequate single-variable manipulation for causal inference about training method effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims these techniques 'significantly enhance the safety of conversational AI systems' and provide 'a feasible framework for deploying more secure and trustworthy AI-driven interactions,' but testing is limited to one violation category (malicious/jailbreak) on one curated test distribution. The paper acknowledges 'covering all existing attack vectors... can be a limiting factor' but the title and framing are broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements, such as whether SFT gains come from format compliance vs. genuine reasoning improvement, whether the test set distribution particularly favors fine-tuned models, or whether memorization of training patterns explains performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures F1/ADR/FPR on a curated binary classification test set and frames this as 'enhancing the safety of conversational AI systems' and 'a feasible framework for deploying more secure AI-driven interactions.' No discussion of the gap between test set performance and real-world deployment safety is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: Mistral-7B-Instruct-v0.2, Mixtral-8x7B-Instruct-v0.1, Llama2-13B-Chat, and Llama3-8B-Instruct (Table 3 and throughout)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows the prompt structure with color-coded sections (system prompt, CoT prompt, user query + policy definition). Figure 7 shows the custom safety prompt added to LlamaGuard-2. Table 4 shows example accepted/rejected responses with their formatting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports LoRA rank (256), alpha, learning rates, batch sizes, beta (0.1), and epochs for all three tuning strategies. Evaluation uses 'top p 1 and temperature 0.' A #END trigger token is used to prevent repetitive generation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is standard LLM fine-tuning and single-pass inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the full data construction: selection of 128 distinct AdvBench examples, all 100 MaliciousInstruct, 180 from Forbidden Question Set, 240 jailbreak prompts, 3600 synthetic negatives. Training/test split proportions and exclusion criteria are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I is titled 'Discussions and Limitations' and provides substantive discussion of resource requirements, training time tradeoffs, and limitations of attack vector coverage."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: 'covering all existing attack vectors in both fine-tuning and evaluation datasets can be a limiting factor,' larger LLMs 'may require a larger training set or additional training epochs,' and alignment techniques may need 'a larger and/or more diverse set of rejected responses.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper states its scope is 'the malicious/jailbreak category' and acknowledges attack coverage limitations, it does not systematically enumerate what was not tested (other attack types, other domains, production conditions, adaptive adversaries, etc.)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The curated dataset (training annotations, test splits, rejected responses) is not released. Only the source datasets are publicly available, but not the authors' specific selections and annotations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes in detail which datasets were used, how many examples from each, the selection criteria (e.g., '128 distinct ones that fall under the category of fraudulent requests'), and how negative examples were synthetically generated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (AdvBench, MaliciousInstruct, Forbidden Question Set, Jailbreak Prompt Set) with synthetic negative examples."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the pipeline: source dataset selection → example filtering → proportional sampling for training → test set construction from remaining examples → accepted response generation via Mixtral → manual annotation → rejected response synthesis (3 per query, 1200 total)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present. Five of six authors are affiliated with Capital One, but no explicit funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: five authors from Capital One and one from University of Maryland, College Park."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Five of six authors are Capital One employees. Capital One, as a financial services company deploying conversational AI, has a direct commercial interest in demonstrating that fine-tuned LLM guardrails are effective for their products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four base models (Mistral 7B, Mixtral 8x7B, Llama2 13B, Llama3 8B), despite evaluating their base (untuned) performance on the test set."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base models' pre-training data included examples from the public test set sources (AdvBench, MaliciousInstruct, etc.), all of which were published before these models were trained."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The test data comes from public datasets (AdvBench 2023, MaliciousInstruct 2023, Jailbreak Prompt Set 2024) that predate or overlap with Llama3's training period. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves LLM fine-tuning and evaluation on curated datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available datasets and synthetic data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite the paper noting 'low response latency is critical.' Figure 9 compares parameter counts as 'a proxy for inference latency' but does not measure actual latency or cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D specifies AWS P4D.24XLARGE instances with 8 A100 40GB GPUs. Table 2 reports training hours for each experiment (e.g., Llama3 SFT: 2.5h, +DPO: 0.5h, +KTO: 2h). GPU counts per task are also stated (3, 6, 7 GPUs for SFT, DPO, KTO)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating whether they come from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports the hyperparameter search space (LoRA rank, alpha, learning rate, batch size, beta, epochs) and the paper states 'We have not conducted an exhaustive hyperparameter search... due to resource constraints. Instead, we considered a small hyperparameter space using Mistral-7B-Instruct-v0.2.'"
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'The best hyperparameters discovered were then adopted in the other three LLMs' but does not explain the selection criterion — no mention of a validation set or what metric determined 'best.' It is unclear whether selection was done on test data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their fine-tuned models against external baselines (LlamaGuard-2, DeBERTaV3, PromptGuard) on their own curated test set without acknowledging the bias of evaluating on a dataset they designed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 explicitly plots training hours vs F1 Score vs number of parameters for all experiments and LLMs, directly addressing the compute-performance tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the curated test set (from AdvBench, MaliciousInstruct, etc.) is representative of real-world attack distributions or whether performance on this benchmark generalizes to production guardrail effectiveness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The approach is standard fine-tuning with direct inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The source datasets (AdvBench, MaliciousInstruct, Jailbreak Prompt Set — all published 2023-2024) predate some base models (Llama3, 2024). No discussion of whether these test examples appeared in pre-training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The binary classification prompt explicitly instructs the model about the policy violation category, which could serve as a strong feature."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training and test examples are drawn from the same source datasets (AdvBench, MaliciousInstruct, etc.) and while specific examples don't overlap, no analysis of distributional similarity or near-duplicate patterns between train and test is provided."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method (canary strings, membership inference, n-gram overlap) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SFT leads to the most significant performance lift across all LLMs, with maximum improvements of 227% in F1 and 344% in ADR.",
       "evidence": "Figure 3 shows F1, ADR, FPR, and Invalid Response Ratio for base vs. SFT vs. SFT+DPO vs. SFT+KTO across four LLMs. The maximum lifts are stated in the Results section.",
       "supported": "moderate"
     },
     {
       "claim": "DPO and KTO alignment provide additional improvements over SFT, with KTO slightly outperforming DPO in F1 and ADR in most LLMs.",
       "evidence": "Figure 3 shows marginal gains from DPO/KTO over SFT. 'KTO slightly outperforming DPO in F1 and ADR in all LLMs, with the exception of Llama3 8B Instruct.'",
       "supported": "weak"
     },
     {
       "claim": "Llama3-DPO outperforms LlamaGuard-2 in ADR by 172% and reduces FPR by 275%.",
       "evidence": "Table 1: Llama3-DPO achieves ADR 93.3%, FPR 0.8% vs. LlamaGuard-2 ADR 54.2%, FPR 2.2% on the full test set.",
       "supported": "moderate"
     },
     {
       "claim": "Base LLMs are poorly equipped against standalone jailbreaks, with ADR values as much as 7.5x smaller than for jailbreaks with malicious queries.",
       "evidence": "Figure 4 shows baseline ADR broken down by query type. Standalone jailbreak ADR is substantially lower than combined jailbreak+malicious across all four base LLMs.",
       "supported": "strong"
     },
     {
       "claim": "These performance improvements can be obtained with small training datasets (400 examples) and minimal hyperparameter tuning.",
       "evidence": "Training set is 400 examples (200 positive, 200 negative). Hyperparameter search was limited (Table 3, Appendix E). Results on ~6800 test examples show significant improvements.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning reduces the invalid response ratio, with Llama3 8B Instruct dropping from 16.8% to 0.3% with DPO.",
       "evidence": "Figure 3, lower right panel. 'The invalid response ratio categorically drops with any tuning technique, by a wider margin for Llama3 8B Instruct.'",
       "supported": "moderate"
     },
     {
       "claim": "Smaller tuned LLMs can achieve better results as input guardrails than larger counterparts with relatively smaller resources.",
       "evidence": "Appendix I discusses this: Llama3-DPO (8B) outperforms Mixtral 8x7B and Llama2 13B variants with less training time. Figure 9 visualizes this tradeoff.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results are single-run point estimates with no uncertainty quantification. Given the small training set (400 examples), results could vary substantially across runs or seeds. The marginal DPO/KTO gains over SFT may not be statistically significant."
     },
     {
       "flag": "Evaluation on authors' own curated test set",
       "detail": "The comparison against external baselines (LlamaGuard-2, DeBERTaV3, PromptGuard) is conducted on the authors' own curated test set, which was designed for their specific policy definition. PromptGuard's 99.8% FPR suggests a severe distribution mismatch rather than genuine model failure, as PromptGuard was designed for prompt injection detection, not their malicious content taxonomy."
     },
     {
       "flag": "Corporate conflict of interest",
       "detail": "Five of six authors are Capital One employees evaluating a guardrail approach relevant to Capital One's products. No conflict of interest or funding disclosure is provided. The authors have a commercial motivation for positive results."
     },
     {
       "flag": "Hyperparameter selection methodology unclear",
       "detail": "Hyperparameters were tuned on Mistral 7B and transferred to other models. The selection criterion for 'best' is not stated — no validation set is mentioned, raising the possibility of test set selection."
     },
     {
       "flag": "Unfair baseline comparison with PromptGuard",
       "detail": "PromptGuard achieves 99.8% FPR on their dataset, indicating it was evaluated outside its intended use case (it detects prompt injection, not malicious content broadly). Reporting that Llama3-DPO achieves '99% FPR reduction over PromptGuard' is misleading when the tool was not designed for their task."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": ["A. Zou", "Z. Wang", "N. Carlini", "M. Nasr", "J. Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational work on adversarial suffix attacks against LLMs showing transferability to black-box models; source of the AdvBench dataset used in this study."
     },
     {
       "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations",
       "authors": ["H. Inan", "K. Upasani", "J. Chi", "R. Rungta", "K. Iyer", "Y. Mao"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LLM-based safety guardrail for human-AI conversations; LlamaGuard-2 is a key baseline in this study."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["R. Rafailov", "A. Sharma", "E. Mitchell", "S. Ermon", "C. D. Manning", "C. Finn"],
       "year": 2024,
       "arxiv_id": "2305.18290",
       "relevance": "Core alignment technique used in this study; DPO is one of two alignment methods evaluated for improving guardrail CoT reasoning."
     },
     {
       "title": "KTO: Model Alignment as Prospect Theoretic Optimization",
       "authors": ["K. Ethayarajh", "W. Xu", "N. Muennighoff", "D. Jurafsky", "D. Kiela"],
       "year": 2024,
       "arxiv_id": "2402.01306",
       "relevance": "Alternative alignment technique evaluated in this study; KTO uses prospect theory loss and only requires binary signals rather than preference pairs."
     },
     {
       "title": "Jailbreaking Black Box Large Language Models in Twenty Queries",
       "authors": ["P. Chao", "A. Robey", "E. Dobriban", "H. Hassani", "G. J. Pappas", "E. Wong"],
       "year": 2024,
       "arxiv_id": "2310.08419",
       "relevance": "Automated black-box jailbreak attack method; motivates the need for robust input guardrails studied in this paper."
     },
     {
       "title": "NeMo Guardrails: A Toolkit for Controllable and Safe LLM Applications with Programmable Rails",
       "authors": ["T. Rebedea", "R. Dinu", "M. Sreedhar", "C. Parisien", "J. Cohen"],
       "year": 2023,
       "arxiv_id": "2310.10501",
       "relevance": "Programmable guardrails toolkit for LLM applications; represents an alternative approach to the LLM-as-a-Judge guardrail paradigm studied here."
     },
     {
       "title": "Jailbroken: How Does LLM Safety Training Fail?",
       "authors": ["A. Wei", "N. Haghtalab", "J. Steinhardt"],
       "year": 2024,
       "relevance": "Analysis of how safety training fails against adversarial attacks; contextualizes the vulnerability of base LLMs that this paper's guardrails address."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis", "Z. Allen-Zhu", "Y. Li", "S. Wang", "L. Wang", "W. Chen"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method used in all experiments to mitigate catastrophic forgetting during guardrail adaptation."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang", "D. Almeida", "C. L. Wainwright"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "RLHF methodology for aligning LLMs with human preferences; the study's DPO/KTO approach is positioned as a more efficient alternative."
     },
     {
       "title": "Do Anything Now: Characterizing and Evaluating In-The-Wild Jailbreak Prompts on Large Language Models",
       "authors": ["X. Shen", "Z. Chen", "M. Backes", "Y. Shen", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2308.03825",
       "relevance": "Comprehensive study of jailbreak prompts in the wild; source of the Forbidden Question Set and Jailbreak Prompt Set used for training and evaluation."
     },
     {
       "title": "Tree of Attacks: Jailbreaking Black-Box LLMs Automatically",
       "authors": ["A. Mehrotra", "M. Zampetakis", "P. Kassianik", "B. Nelson", "H. Anderson", "Y. Singer", "A. Karbasi"],
       "year": 2024,
       "arxiv_id": "2312.02119",
       "relevance": "Automated multi-turn jailbreak attack method; represents the evolving threat landscape that motivates input guardrail research."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans", "M. Bosma", "E. H. Chi", "Q. Le", "D. Zhou"],
       "year": 2022,
       "relevance": "Foundational CoT prompting work that motivates the study's use of CoT reasoning to improve guardrail accuracy and explanation quality."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly applicable to companies deploying conversational AI with safety requirements; the SFT approach with small data is practical but no code is released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that fine-tuning improves task-specific LLM performance; no surprising or contrarian results."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses LLM safety vulnerabilities and demonstrates that base models are poorly equipped against jailbreaks, raising security concerns for deployed systems."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical comparison of fine-tuning strategies."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or model weights released; results cannot be reproduced or tried by practitioners."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Capital One is a recognizable brand but not a prominent AI research lab; the work is not about a widely known AI product."
     }
   }
 }