ai-research-survey

scan.json (31486B)

---

 {
   "paper": {
     "title": "Hijacking Large Language Models via Adversarial In-Context Learning",
     "authors": [
       "Xiangyu Zhou",
       "Yao Qiang",
       "Saleh Zare Zade",
       "Prashant Khanduri",
       "Dongxiao Zhu"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2311.09948",
     "doi": "10.48550/arXiv.2311.09948"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a greedy gradient-guided injection (GGI) attack that appends adversarial suffixes to in-context demonstrations, hijacking LLMs to produce targeted outputs or harmful responses. GGI achieves near-perfect attack success rates on classification tasks (collapsing negative accuracy to ~0%) and high jailbreak ASR (e.g., 97.2% on Vicuna-7b with 4-shots). The attack transfers across demo sets and datasets and maintains low perplexity (stealthy). A simple defense of appending clean demonstrations at inference time substantially mitigates the attack, with the prefix strategy recovering up to 86% of lost accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/xzhou98/Hijacking-LLMs-GGI, as stated in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: SST-2, Rotten Tomatoes, AG's News, and AdvBench. The paper references these standard benchmarks and does not use proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The appendix states 'All experiments are conducted on 2 NVIDIA H100 GPU cards in a single node' but provides no software dependencies, library versions, requirements.txt, or environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is linked but the paper itself does not include commands or a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 and 5-6 report point estimates only. Despite averaging over 5 repetitions of demo selection, no confidence intervals, error bars, or ± notation are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GGI achieves SOTA jailbreaking performance', 'GGI remains highly effective even on the SOTA LLMs') based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are presented with baseline context throughout. Tables show clean accuracy vs attack accuracy (e.g., negative accuracy drops from 93.8% to 2.0%), ASR values with and without defense, and PRR percentages quantifying recovery magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 2 training queries are used to learn adversarial suffixes for sentiment tasks, 4 for AG's News and jailbreak, or 1000 test queries. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 states 'we randomly select the demos from the training set and repeat this process 5 times, reporting the average accuracy over the repetitions,' but no standard deviation, interquartile range, or any spread measure is reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: Square Attack, Greedy Search, TextAttack (TA), and ICA for jailbreak. Defense baselines include Onion and a benign instruction approach. Details in Appendix B."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ICA (2023), Square Attack (2020), TextAttack (2020), and Onion (2020). ICA is the most closely related concurrent work. The older baselines are justified as representative of different attack paradigms (black-box, character-level, word-level)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the GGI attack components (e.g., removing the gradient guidance, varying top-k, removing the random subset sampling). The defense variants (Pre. vs Suf.) are compared but this is not an ablation of the attack method itself."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: per-class accuracy (positive and negative separately), Attack Success Rate (ASR) for jailbreak, Performance Recovery Rate (PRR) for defense evaluation, and perplexity scores for stealthiness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. The paper claims the attack is 'imperceptible' and 'stealthy' but assesses stealthiness only via automated perplexity scores (Figure 3), not human judgment of whether adversarial suffixes are detectable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training queries (2-4 examples per Table 4) used to learn adversarial suffixes are separate from the 1000 test queries used for evaluation. The split is clearly described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model, per dataset, per few-shot setting (2/4/8-shots), and per sentiment class (positive vs negative). Tables 1, 2, 3, 5, and 6 all provide detailed breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where attacks are less effective: baseline attacks fail on stronger models like LLaMA3.1-8b (Section 6.2), the Suf. defense fails on some models with PRR=0% (Table 2), and LLaMA3.1-8b-Instruct resists jailbreaking more effectively (Table 3, ASR remains low for baselines)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results include: Suf. defense showing PRR=0% on Vicuna-7b and LLaMA-13b for classification (Table 2), Onion defense underperforming across most settings, and the benign instruction defense failing to mitigate jailbreak attacks (Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about effectiveness (near-perfect hijacking in Tables 1, 5), stealthiness (low perplexity in Figure 3), transferability (Figure 2), and defense effectiveness (Tables 2, 3) are all supported by corresponding experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about adversarial suffixes causing misclassification and jailbreaking. The study design uses controlled comparisons (clean vs. adversarial demos, with vs. without defense) which adequately supports these claims. The attention analysis in Figure 6 provides a mechanistic explanation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Hijacking Large Language Models' generally, but experiments are limited to models up to 13B parameters from specific families (OPT, LLaMA, Vicuna, Mistral). The abstract says 'leveraging LLMs' and 'diverse LLMs' without bounding to the tested range. No discussion of whether results extend to larger models (70B+) or closed-source models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides one attention analysis example (Figure 6) as explanation for why GGI works but does not consider alternative explanations. For instance, it does not discuss whether the attack's effectiveness could be partially attributed to the small demo set size, model-specific vulnerabilities, or dataset characteristics."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy and ASR and claims exactly those things. ASR is defined precisely (absence of refusal phrases) and accuracy is standard. The stealthiness claim uses perplexity as an explicit proxy, though without fully discussing its limitations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model names are specified with sizes and versions: OPT-6.7b, Vicuna-7b-v1.5, LLaMA-13b (cited to LLaMA 1 paper), LLaMA3.1-8b, Mistral-7B-Instruct, LLaMA3-8b-Instruct, LLaMA3.1-8b-Instruct. For these open-source models, the name+size+citation identifies the exact HuggingFace checkpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with concrete examples are provided in Table 7 for SST-2/RT, AG's News, and AdvBench. The benign instruction for jailbreak defense is provided verbatim in Appendix A."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 defines parameters T (iterations), b (batch size), and k (top-k) but their actual values are not stated in the paper. LLM inference settings (temperature, top-p) are also not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach directly manipulates prompt tokens without any agent framework, tool use, or workflow orchestration."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 documents training/test splits per dataset. The ICL settings section describes demo selection (random from training set, 5 repetitions). Template structures are shown in Table 7. The AdvBench evaluation uses 200 randomly selected harmful queries from the benchmark."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' paragraph appears at the end of Section 8 (Conclusion), discussing the unexplored impact of adversarial token placement within demos."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies a specific threat: 'Our analysis has not yet examined the impact of adversarial token placement within the demos (e.g., at the prefix, middle, or suffix) on the effectiveness of the attack.' This is specific to their study design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of model size boundaries, closed-source model applicability, whether results generalize beyond English, or the limited task diversity (only classification and jailbreak)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-sample predictions, learned adversarial suffixes, intermediate optimization logs) is made available. Only aggregated results in tables are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Dataset sources are clearly identified (SST-2, Rotten Tomatoes, AG's News, AdvBench). Table 4 provides statistics on training and test queries. Selection procedures for demos and test queries are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks (SST-2, RT, AG's News, AdvBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: select training queries (Table 4) → learn adversarial suffixes via GGI (Algorithm 1) → construct adversarial demos → evaluate on 1000 test queries with random demo selection repeated 5 times. The defense pipeline (appending clean demos) is also clearly described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment, grants, or sponsorship information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Wayne State University and Oakland University. No evaluated product is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because funding is not disclosed. The absence of a funding disclosure makes this unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming/adversarial attack study testing defense robustness rather than evaluating pre-trained model knowledge on benchmarks. The core contribution is the attack mechanism, not model capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As an adversarial attack study, the concern is whether attacks can manipulate model behavior, not whether models have memorized benchmark answers. Contamination does not affect the validity of attack success measurements."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming study testing adversarial robustness of ICL, not evaluating model capabilities on benchmarks. Benchmark contamination is not a threat to the validity of the attack evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations are automated using benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper includes an Ethics Statement discussing responsible disclosure intent."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The attack requires gradient computation over the full vocabulary for each demo position across T iterations, which is computationally expensive, but this cost is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The appendix mentions '2 NVIDIA H100 GPU cards' as hardware but does not state total GPU hours, wall-clock time, or any compute budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper averages over 5 repetitions of random demo selection, it does not report variance across these runs or analyze seed sensitivity. Only averaged point estimates are shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (ICL Settings) states: 'we randomly select the demos from the training set and repeat this process 5 times, reporting the average accuracy over the repetitions.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The values for T (iterations), k (top-k), and b (batch size) in Algorithm 1 appear tuned but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the final attack configuration was selected. The paper does not explain how hyperparameters for GGI were chosen or whether multiple configurations were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparisons across 4+ models, 2-3 datasets, 3 few-shot settings, and 4+ attack methods without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own versions of baseline attacks (Greedy, Square) and compare against their proposed GGI without acknowledging potential self-comparison bias in baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GGI requires gradient computation over the full vocabulary (gradient-based) while baselines like Square are black-box. This compute asymmetry is not discussed or controlled for in comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ASR (absence of refusal phrases) as the jailbreak metric without discussing whether string-matching for refusal phrases adequately captures harmful content generation. No discussion of whether SST-2 accuracy adequately measures attack effectiveness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The attack operates directly on prompt tokens without any agentic framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SST-2, RT, or AG's News examples appeared in the training data of the evaluated models. These benchmarks predate all tested models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in realistic deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training queries used for learning adversarial suffixes and test queries, or whether structural similarities between datasets affect results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "GGI consistently hijacks LLMs to generate targeted positive sentiment, achieving near-perfect positive accuracy and collapsing negative accuracy to near 0% across most settings.",
       "evidence": "Table 1 shows GGI achieving 100% positive accuracy and 0-2% negative accuracy across OPT-6.7b, LLaMA-13b at 8-shots. Even on LLaMA3.1-8b (strongest model), negative accuracy drops to 0% at 8-shots.",
       "supported": "strong"
     },
     {
       "claim": "GGI achieves state-of-the-art jailbreaking performance across all models and settings.",
       "evidence": "Table 3 shows GGI achieving 76.1-97.2% ASR on Vicuna-7b, 90.5-98.9% on Mistral-7B-Instruct, and 28.6-62.0% on LLaMA3.1-8b-Instruct, consistently outperforming ICA, Square, and Greedy baselines.",
       "supported": "strong"
     },
     {
       "claim": "The Prefix defense method substantially recovers model performance against hijacking attacks, with PRR frequently exceeding 65%.",
       "evidence": "Table 2 shows PRR values of 58.8-81.1% for Pre. defense across OPT-6.7b, Vicuna-7b, LLaMA-13b on SST-2 and RT. Pre. consistently outperforms Onion baseline.",
       "supported": "moderate"
     },
     {
       "claim": "The Suffix defense consistently delivers the strongest jailbreak mitigation, reducing ASR to single digits.",
       "evidence": "Table 3 shows Suf. reducing ASR to 1.5-4.4% on Vicuna-7b and 4.9-8.6% on Mistral-7B-Instruct under GGI attack. However, results on LLaMA3.1-8b-Instruct are less dramatic (4.1-9.3%).",
       "supported": "moderate"
     },
     {
       "claim": "GGI exhibits transferability across different demo sets and across different datasets of the same task.",
       "evidence": "Figure 2 shows adversarial tokens learned from one demo set achieving high ASR when applied to three different demo sets on both SST-2 and RT. Tokens learned from RT transfer to SST-2.",
       "supported": "moderate"
     },
     {
       "claim": "GGI maintains strong stealthiness with perplexity scores comparable to clean inputs, making it difficult to detect via perplexity-based defenses.",
       "evidence": "Figure 3 shows average perplexity of 19.36 for GGI vs 13.8 for clean inputs on LLaMA-13b with RT, while TextAttack reaches 46.39. However, this is tested on only one model/dataset combination.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller LLMs are more vulnerable to adversarial hijacking attacks.",
       "evidence": "Table 6 compares OPT-2.7b vs OPT-6.7b and LLaMA-7b vs LLaMA-13b, showing smaller models achieve 100% attack success more easily. Figure 4 confirms with ASR comparison on AG's News.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification despite multiple runs",
       "detail": "The paper averages over 5 repetitions of demo selection but reports zero variance, standard deviation, or confidence intervals in any table. This hides potential instability in results, especially for settings with more moderate attack effectiveness."
     },
     {
       "flag": "No statistical significance tests for comparative claims",
       "detail": "All claims of superiority ('GGI achieves SOTA', 'outperforms all baselines') are based on comparing raw numbers without any statistical tests. Given the variance in ICL performance that the paper itself acknowledges, this is a notable omission."
     },
     {
       "flag": "Compute-unfair baseline comparison",
       "detail": "GGI uses gradient-based optimization (requiring full model access and backpropagation over the vocabulary), while Square Attack is a black-box method. The paper does not control for or discuss the compute asymmetry, making it unclear how much of GGI's advantage comes from compute rather than algorithmic innovation."
     },
     {
       "flag": "Stealthiness claim based on limited evaluation",
       "detail": "The 'imperceptible' and 'stealthy' claims rest entirely on perplexity scores from one model (LLaMA-13b) on one dataset (RT) in Figure 3. No human evaluation of detectability is conducted, and adversarial suffixes like 'NULL', 'Remove', 'For', and 'refresh Real result' shown in figures would be noticeable to a careful reader."
     },
     {
       "flag": "Missing hyperparameter details",
       "detail": "Key algorithm parameters (iterations T, top-k value, batch size b, and LLM temperature/sampling settings) are not reported, making the attack difficult to reproduce or fairly evaluate."
     },
     {
       "flag": "Jailbreak ASR metric limitations",
       "detail": "ASR is defined as absence of refusal phrases, but the paper does not verify whether the generated content is actually harmful or coherent. A model that generates gibberish without a refusal phrase would count as a successful jailbreak under this metric."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational gradient-based adversarial attack (GCG) on aligned LLMs; provides AdvBench benchmark used in this paper."
     },
     {
       "title": "Adversarial demonstration attacks on large language models",
       "authors": ["Jiongxiao Wang"],
       "year": 2023,
       "arxiv_id": "2305.14950",
       "relevance": "Most closely related prior work on adversarial attacks targeting ICL demonstrations; this paper's TextAttack baseline follows their approach."
     },
     {
       "title": "Backdoor attacks for in-context learning with language models",
       "authors": ["Nikhil Kandpal"],
       "year": 2023,
       "arxiv_id": "2307.14692",
       "relevance": "Proposes backdoor attacks against ICL by fine-tuning on poisoned training samples; addresses the security of in-context learning."
     },
     {
       "title": "Jailbreak and guard aligned language models with only few in-context demonstrations",
       "authors": ["Zeming Wei"],
       "year": 2023,
       "arxiv_id": "2310.06387",
       "relevance": "Shows that in-context demos alone can jailbreak aligned LLMs; serves as the ICA baseline and inspires the defense strategy in this paper."
     },
     {
       "title": "Universal vulnerabilities in large language models: Backdoor attacks for in-context learning",
       "authors": ["Shuai Zhao", "Meihuizi Jia", "Luu Anh Tuan", "Fengjun Pan", "Jinming Wen"],
       "year": 2024,
       "arxiv_id": "2401.05949",
       "relevance": "Introduces ICLAttack inserting backdoor triggers into ICL demos and queries; directly related to adversarial ICL security."
     },
     {
       "title": "Baseline defenses for adversarial attacks against aligned language models",
       "authors": ["Neel Jain"],
       "year": 2023,
       "arxiv_id": "2309.00614",
       "relevance": "Proposes perplexity-based defenses against adversarial prompts; this paper evaluates stealthiness against such defenses."
     },
     {
       "title": "Survey of vulnerabilities in large language models revealed by adversarial attacks",
       "authors": ["Erfan Shayegani"],
       "year": 2023,
       "arxiv_id": "2310.10844",
       "relevance": "Comprehensive survey of LLM adversarial vulnerabilities; provides broader context for the attack landscape."
     },
     {
       "title": "Test-time backdoor mitigation for black-box large language models with defensive demonstrations",
       "authors": ["Wenjie Mo"],
       "year": 2023,
       "arxiv_id": "2311.09763",
       "relevance": "Proposes test-time defense using clean demonstrations against backdoor attacks; directly inspires the defense strategy in this paper."
     },
     {
       "title": "Cold-attack: Jailbreaking LLMs with stealthiness and controllability",
       "authors": ["Xingang Guo"],
       "year": 2024,
       "arxiv_id": "2402.08679",
       "relevance": "Addresses stealthy jailbreak attacks on LLMs; related to the adversarial prompt generation space."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Aaron Grattafiori", "Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Describes the LLaMA 3 model family used as primary evaluation target in this paper's experiments."
     },
     {
       "title": "Mitigating fine-tuning jailbreak attack with backdoor enhanced alignment",
       "authors": ["Jiongxiao Wang"],
       "year": 2024,
       "arxiv_id": "2402.14968",
       "relevance": "Proposes alignment-based defense against jailbreak attacks from fine-tuning; related to LLM safety mechanisms."
     },
     {
       "title": "Safety alignment should be made more than just a few tokens deep",
       "authors": ["Xiangyu Qi", "Ashwinee Panda"],
       "year": 2024,
       "arxiv_id": "2406.05946",
       "relevance": "Analyzes the depth of safety alignment in LLMs, relevant to understanding why adversarial ICL can bypass safety mechanisms."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The defense strategy (appending clean demos) is immediately usable by practitioners deploying ICL-based systems, though the attack requires white-box model access."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Adversarial vulnerability of ICL extends known concerns about LLM robustness but doesn't overturn any widely-held belief."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates concrete jailbreaking of aligned models (97% ASR on Vicuna) via adversarial in-context demos, raising legitimate safety concerns about ICL deployment."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy or provocative framing; positioned as constructive red-teaming with accompanying defense."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with open-source models (LLaMA, Vicuna, Mistral), allowing reproduction with sufficient compute."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Wayne State and Oakland University; tests on well-known open-source models but not from a famous AI lab."
     }
   }
 }