ai-research-survey

scan.json (32635B)

---

 {
   "paper": {
     "title": "ShieldLearner: A New Paradigm for Jailbreak Attack Defense in LLMs",
     "authors": [
       "Ziyi Ni",
       "Hao Wang",
       "Huacan Wang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.13162",
     "doi": "10.48550/arXiv.2502.13162"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ShieldLearner proposes a parameter-free jailbreak defense paradigm that distills attack patterns into a Pattern Atlas and defense heuristics into a Meta-analysis Framework through self-learning. On standard jailbreak benchmarks (HarmBench, AdvBench), ShieldLearner achieves 0% ASR across five attack types, though the authors acknowledge these benchmarks may be 'outdated.' On a harder curated test set with concealed intent, ShieldLearner achieves 11.81% ASR on GPT-4o compared to the next best baseline G4D at 39.75%, while maintaining lower time cost (2.96s vs 8.06s). Ablation studies show all three components (Pattern Atlas, Framework, Self Attack) contribute to performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper. The Limitations section mentions 'our soon-to-be-released, self-learned Pattern Atlas and Analysis Framework,' indicating artifacts are not yet available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While the base datasets used (DAN, JailbreakV, WildJailbreak, HarmBench, AdvBench) are publicly available, the paper's novel contributions — the curated 858 training instances, the 483-prompt hard test set, the learned Pattern Atlas, and the Meta-analysis Framework — are not released. The paper explicitly states these are 'soon-to-be-released.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or environment setup details are provided. The paper does not specify library versions or dependencies beyond the model API names."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. While Algorithm 1 and Algorithm 2 describe the method at a high level, these are not executable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., 11.81% ASR, 20.95% FPR). No confidence intervals, error bars, or ± notation are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ShieldLearner 'outperforms all baselines' and achieves 'significantly higher defense success rate,' but no statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are reported. All comparisons are based on raw numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw ASR and FPR percentages in tables but does not explicitly compute or report effect sizes (e.g., Cohen's d, relative improvement). The reader must manually compute differences between methods from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes used — 858 training instances, 483 hard test prompts, 210 benign prompts. No power analysis or discussion of whether these sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported for any experimental results. Results appear to be from single runs with no indication of repeated trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against six baselines: Vanilla (no defense), Paraphrase, Self-Reminder, ICD (In-Context Demonstrations), IA (Intent Analysis), and G4D, as shown in Tables 1 and 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: G4D (Cao et al., 2024), IA (Zhang et al., 2024a), and ICD (Wei et al., 2023). These represent contemporary competitive approaches to jailbreak defense."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents ablation results on GPT-4o removing three components individually: Self Attack (w/o Self Attack), Pattern Retrieval (w/o Pattern Retrieval), and Framework (w/o Framework). Figure 5 shows performance as a function of training data size."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: Attack Success Rate (ASR), False Positive Rate (FPR), and Time Cost (seconds per prompt), as described in Section 4.3 and shown in Tables 1-3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated using Llama-Guard-3 for ASR assessment and keyword matching for FPR, as described in Appendix C."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses data curated from DAN and JailbreakV (858 instances + 300 benign + 100 concealed-intent). Testing uses separate datasets: HarmBench and AdvBench for Easy Mode, and a separately curated set from WildJailbreak/JailbreakV for Hard Mode (Section 4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-attack-method breakdowns across DAN, SAA, DeepInception, GCG, and PAIR. Table 3 provides per-component ablation breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case studies in Appendix D (Tables 4-5) show successful detection examples (with vs. without patterns), not failure cases. On the hard test set, ShieldLearner still has 11.81% ASR (GPT-4o) meaning some attacks succeed, but these failures are not analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows that FPR increases with training data size, especially for GPT-3.5-turbo (20.48% → 30.48%), which the paper acknowledges as 'potential overfitting as the model becomes overly sensitive to harmful patterns.' The easy-mode results also admit standard benchmarks are 'outdated.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significantly higher defense success rate than existing baselines' (supported by Tables 1-2), 'lower computational overhead' (supported by Time Cost columns showing 2.14-2.96s vs G4D's 6.53-8.06s), and 'practical and efficient solution' (partially supported by the time cost data)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims come from ablation studies (Table 3): removing individual components degrades performance. The ablation design uses controlled single-variable manipulation (removing one component at a time while keeping others), which is adequate for these component-level causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'a revolutionary breakthrough in security' (Section 1) and the title calls it 'A New Paradigm,' but experiments are limited to two OpenAI models (GPT-3.5-turbo, GPT-4o), English-language prompts, and specific jailbreak attack types. No testing on open-source models, other languages, or other LLM providers."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for its results. For example, the 0% ASR on easy mode could be due to model memorization of the attacks (the paper hints at this but doesn't analyze it). No confounds are discussed, such as whether the improvement comes from additional inference-time compute rather than the learned patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures ASR (whether jailbreak attacks succeed) and FPR (whether benign inputs are incorrectly blocked) as proxies for defense effectiveness. These metrics directly measure what is claimed — the gap between measurement and framing is small. The paper appropriately frames results in terms of these specific metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 specifies exact model versions: 'GPT-4o-2024-08-06' and 'GPT-3.5-turbo-1106' with specific date/version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix E: pattern extraction prompt (E.1), self-attack prompt (E.2), and meta-analysis framework prompt (E.3). These include the complete system and user prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 reports method-specific hyperparameters (3 rounds of optimization, 3 adversarial iterations, 0.7/0.3 retrieval weights, top-5 results, 0.5 similarity threshold), but LLM API parameters such as temperature, top-p, and max tokens are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent pipeline is described in detail: Section 3.2 covers the self-learning phase with pattern extraction agent, critic agent, and risk analyzer. Algorithm 1 and Algorithm 2 formalize the workflows. Figure 1 provides an overview diagram. Figure 4 illustrates the test phase architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 documents preprocessing: 'we first removed duplicate samples (i.e., those with identical first and last 20 characters) and eliminated overly similar expressions' from 1,405 + 5,000 samples to yield 858 training instances. Hard test set curation criteria are described in Appendix B."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section (Section 8) is present, titled 'Training Datasets: The More Diverse, the Better,' which provides substantive discussion of how dataset quality affects performance."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses general observations about dataset diversity ('current jailbreak attack datasets vary widely') rather than specific threats to the validity of this study's results. No mention of threats like model-specific results, language limitations, potential train-test contamination, or the lack of statistical testing."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Missing boundaries: no statement about results being limited to two specific OpenAI models, English only, specific attack types only, or that the 'hard test set' was manually curated with potential subjective bias."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data, intermediate outputs, or detailed results are available for verification. The curated datasets, Pattern Atlas, and Meta-analysis Framework are described as 'soon-to-be-released' but not currently available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: 1,405 templates from DAN, 5,000 prompts from JailbreakV, deduplication to 858 training instances, 300 benign from WildJailbreak, 100 concealed-intent prompts manually selected for framework training. Test sets from HarmBench, AdvBench, and curated WildJailbreak/JailbreakV."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public jailbreak/safety benchmarks (DAN, JailbreakV, WildJailbreak, HarmBench, AdvBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 documents the pipeline: DAN (1,405) + JailbreakV (5,000) → deduplication by first/last 20 characters and similarity → 858 training instances. For framework refinement: 100 manually selected prompts from WildJailbreak. Hard test set: 483 selected from WildJailbreak/JailbreakV + 210 benign prompts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments are mentioned anywhere in the paper. Authors are from Chinese Academy of Sciences and Beijing University of Aeronautics and Astronautics, which presumably receive institutional funding, but nothing is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese Academy of Sciences (Institute of Automation) and Beijing University of Aeronautics and Astronautics (Institute of AI). The evaluated products are OpenAI's models, not the authors' own."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The authors evaluate OpenAI's products, which they are not affiliated with, but since no funding source is stated, independence cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a defense method against jailbreak attacks, not a pre-trained model's capability on a knowledge benchmark. The evaluation measures whether the defense pipeline detects attacks, not whether the model has memorized answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defenses rather than model knowledge on benchmarks. Contamination in the traditional sense (model training data containing benchmark answers) is not the relevant concern here."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a defense system's ability to detect jailbreak attacks, not a model's knowledge capabilities on a benchmark. Standard benchmark contamination concerns do not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated using LLM-based and keyword-based methods."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves computational experiments with LLM APIs and public jailbreak datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report 'Time Cost' as average seconds per prompt for all methods: ShieldLearner at 2.14s (GPT-3.5-turbo), 2.19s (GPT-4o) for easy mode, and 2.61s/2.96s for hard mode, compared to baselines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API costs, total number of API calls for training/testing, GPU hours, or overall compute expenditure for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds, seed sensitivity analysis, or results across multiple runs. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results in Tables 1-3 are from single runs or averaged over multiple trials."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The hyperparameters (3 optimization rounds, 0.7/0.3 retrieval weights, top-5, 0.5 threshold) appear to be fixed choices with no justification of how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses fixed hyperparameters without explaining how the configuration was chosen or whether alternative configurations were evaluated. No validation-based selection process is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in this paper, so the question of correction for multiple comparisons is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (ShieldLearner) against baselines without acknowledging author-evaluation bias. No independent evaluation or discussion of potential bias from implementing and tuning their own system while using others' published baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While per-prompt time cost is reported, performance is not systematically analyzed as a function of compute budget. ShieldLearner uses additional inference steps (pattern retrieval, framework analysis) compared to simpler baselines, but compute-matched comparisons are not provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses that standard benchmarks are 'somewhat outdated' and that 'the considerable efforts previously invested to achieve improvements on less challenging datasets are relatively cost-ineffective' (Section 5). This motivates their creation of a harder test set with concealed intent."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the same base models (GPT-3.5-turbo-1106, GPT-4o-2024-08-06) across all defense methods, holding the model constant while varying only the defense scaffold. Section 4.3 states 'For each model, we ensure consistent use across all phases.'"
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3.5/GPT-4o may have been trained on the jailbreak prompts from DAN, JailbreakV, or WildJailbreak datasets, which were all published before the models' training cutoffs. The paper's own easy-mode results hint at this issue ('may even have been encountered during training') but no formal analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides the defense system with information not available in real deployment scenarios, or whether the structured prompt format leaks information about attack type."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training data includes samples from WildJailbreak (300 benign + 100 concealed-intent) and the hard test set also draws from WildJailbreak (483 prompts + 210 benign). The potential overlap between training and test distributions from the same source dataset is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No overlap analysis, decontamination pipeline, or membership inference tests are used to verify independence of training and test data."
       }
     }
   },
   "claims": [
     {
       "claim": "ShieldLearner achieves 0% attack success rate across all five standard jailbreak attack types on both GPT-3.5-turbo and GPT-4o",
       "evidence": "Table 1 shows 0.00% ASR for ShieldLearner across DAN, SAA, DeepInception, GCG, and PAIR attacks on both models, compared to the next best G4D at 0.14% (GPT-3.5) and 0.04% (GPT-4o).",
       "supported": "moderate"
     },
     {
       "claim": "ShieldLearner substantially outperforms all baselines on the hard test set with concealed harmful intent",
       "evidence": "Table 2 shows ShieldLearner at 11.81% ASR (GPT-4o) vs. next best G4D at 39.75%, and 28.16% ASR (GPT-3.5) vs. G4D at 49.48%. FPR is also lowest at 11.62% (GPT-4o).",
       "supported": "moderate"
     },
     {
       "claim": "ShieldLearner achieves strong defense with lower computational overhead than comparable methods",
       "evidence": "Tables 1-2 show time costs of 2.14-2.96s per prompt for ShieldLearner vs. 6.53-8.06s for G4D. However, simpler methods like Vanilla (1.54-1.88s) and Paraphrase (3.18-3.90s) have competitive or lower times.",
       "supported": "moderate"
     },
     {
       "claim": "All three components (Pattern Atlas, Meta-analysis Framework, and Adaptive Adversarial Augmentation) are crucial for ShieldLearner's effectiveness",
       "evidence": "Table 3 ablation on GPT-4o: removing Self Attack raises ASR from 11.81% to 13.76%, removing Pattern Retrieval raises FPR from 11.62% to 27.62%, removing Framework raises ASR from 11.81% to 22.36%.",
       "supported": "moderate"
     },
     {
       "claim": "The framework's defense capability improves monotonically with more training data",
       "evidence": "Figure 5 shows ASR decreasing for GPT-4o (65.52% → 14.29%) and GPT-3.5 (58.87% → 26.71%) as training data increases from 10 to 100 samples. However, FPR increases concurrently, especially for GPT-3.5 (20.48% → 30.48%).",
       "supported": "weak"
     },
     {
       "claim": "ShieldLearner represents 'a revolutionary breakthrough in security' and 'a new paradigm'",
       "evidence": "Stated in Section 1, but evidence is limited to evaluation on two OpenAI models with specific jailbreak attack types. No testing on open-source models, other languages, or real-world deployment scenarios.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Perfect scores on easy benchmarks are uninformative",
       "detail": "The paper's headline result — 0% ASR on standard benchmarks — is undermined by the authors' own admission that these benchmarks are 'somewhat outdated' and that even ablated versions of ShieldLearner 'nearly achieve a 100% defense rate.' This suggests the easy-mode results do not meaningfully differentiate defense methods."
     },
     {
       "flag": "Substantial overclaiming",
       "detail": "The paper describes itself as 'a revolutionary breakthrough in security' (Section 1) and 'A New Paradigm' (title), but experiments are limited to two proprietary OpenAI models, English-language prompts, and five specific attack methods. These broad claims are not proportionate to the evidence."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons between methods rely on single point estimates with no uncertainty quantification. Without error bars, variance, or significance tests, it is impossible to determine whether differences between methods are reliable or due to random variation."
     },
     {
       "flag": "Potential train-test data contamination from shared source",
       "detail": "Training data includes samples from WildJailbreak (300 benign + 100 concealed-intent prompts), and the hard test set also draws from WildJailbreak (483 + 210 prompts). No overlap analysis is conducted to ensure independence of training and test distributions."
     },
     {
       "flag": "No code or artifacts released",
       "detail": "Neither the code, the learned Pattern Atlas, the Meta-analysis Framework, nor the curated hard test set are released, making the results impossible to independently verify or reproduce."
     },
     {
       "flag": "Single-run results with no variance reporting",
       "detail": "LLM outputs are stochastic, but the paper appears to report single-run results without any measure of variability across runs. This is especially concerning given that the defense pipeline involves multiple LLM calls per prompt."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundation LLM whose safety vulnerabilities the paper seeks to defend against."
     },
     {
       "title": "Jailbreaking leading safety-aligned LLMs with simple adaptive attacks",
       "authors": ["Maksym Andriushchenko", "Francesco Croce", "Nicolas Flammarion"],
       "year": 2024,
       "arxiv_id": "2404.02151",
       "relevance": "SAA jailbreak attack method used as one of the five attack types in evaluation."
     },
     {
       "title": "Guide for Defense (G4D): Dynamic guidance for robust and balanced defense in large language models",
       "authors": ["He Cao", "Weidi Luo", "Yu Wang"],
       "year": 2024,
       "arxiv_id": "2410.17922",
       "relevance": "Primary baseline defense method using multi-agent guidance and external knowledge for jailbreak defense."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban"],
       "year": 2023,
       "arxiv_id": "2310.08419",
       "relevance": "PAIR iterative jailbreak attack method used in evaluation."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LLM-based safety classifier used for response-defense; Llama-Guard-3 is used as the ASR evaluation metric."
     },
     {
       "title": "Baseline defenses for adversarial attacks against aligned language models",
       "authors": ["Neel Jain", "Avi Schwarzschild", "Yuxin Wen"],
       "year": 2023,
       "arxiv_id": "2309.00614",
       "relevance": "Paraphrase defense baseline evaluated against ShieldLearner."
     },
     {
       "title": "WildTeaming at scale: From in-the-wild jailbreaks to (adversarially) safer language models",
       "authors": ["Liwei Jiang", "Kavel Rao", "Seungju Han"],
       "year": 2024,
       "arxiv_id": "2406.18510",
       "relevance": "WildJailbreak dataset used for both framework training and hard test set construction."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["Mantas Mazeika", "Long Phan", "Xuwang Yin"],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Standard evaluation benchmark used for easy-mode jailbreak defense testing."
     },
     {
       "title": "Intention analysis prompting makes large language models a good jailbreak defender",
       "authors": ["Yuqi Zhang", "Liang Ding", "Lefei Zhang", "Dacheng Tao"],
       "year": 2024,
       "arxiv_id": "2401.06561",
       "relevance": "Intent Analysis (IA) two-stage defense method used as a baseline."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG gradient-based jailbreak attack method and AdvBench dataset used in evaluation."
     },
     {
       "title": "Defending ChatGPT against jailbreak attack via self-reminders",
       "authors": ["Yueqi Xie", "Jingwei Yi", "Jiawei Shao"],
       "year": 2023,
       "relevance": "Self-Reminder defense baseline and early prompt-defense approach for jailbreak mitigation."
     },
     {
       "title": "Jailbreak and guard aligned language models with only few in-context demonstrations",
       "authors": ["Zeming Wei", "Yifei Wang", "Ang Li"],
       "year": 2023,
       "arxiv_id": "2310.06387",
       "relevance": "In-Context Demonstration (ICD) defense method used as a baseline."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The approach could interest LLM safety practitioners, but no code or artifacts are released, so no one can use it at work today."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The observation that standard jailbreak benchmarks are 'outdated' is mildly contrarian, but the self-learning defense paradigm is an incremental evolution of existing ideas."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Directly addresses jailbreak attacks on deployed LLMs, a significant and growing safety concern for production systems."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no criticism of specific companies or products, no dramatic claims beyond standard academic overclaiming."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, no demo, no repository — nothing for anyone to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates GPT-4o and GPT-3.5 (recognizable products), but the paper is from relatively unknown academic institutions."
     }
   }
 }