ai-research-survey

scan.json (31251B)

---

 {
   "paper": {
     "title": "'Do as I say not as I do': A Semi-Automated Approach for Jailbreak Prompt Attack against Multimodal LLMs",
     "authors": [
       "Chun Wai Chiu",
       "Linghan Huang",
       "Bo Li",
       "Huaming Chen",
       "Kim-Kwang Raymond Choo"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.00735",
     "doi": "10.48550/arXiv.2502.00735"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces Flanking Attack, a voice-based jailbreak method that embeds adversarial audio prompts between benign queries to bypass multimodal LLM safety filters. Tested against Gemini 1.5 Flash across 7 forbidden scenarios (2,100 prompts), the full attack achieves 0.81 average ASR, compared to 0.57 without the flanking technique and 0.12 with plot-only prompts. Ablation shows that both the text prompt framing and the flanking technique are critical components, with flanking contributing approximately 24 percentage points to ASR.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, notebook, or archive URL is provided despite describing a Google Colab-based pipeline. The paper claims to establish 'a replicable testing framework' but releases no code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The forbidden question set is partially shown in Table III (21 questions across 7 scenarios), but the full 2,100 prompts, audio files (MP3), and model responses are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.A specifies: Google Colab (Python 3.10, Ubuntu 22.04), 2 vCPUs, 12 GB RAM, no GPU, google-generativeai 0.4.1, python-docx 1.0. Sufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The experimental setup (Section VI.A) describes the pipeline conceptually but provides no step-by-step reproduction instructions, scripts, or notebook. A researcher would need to reverse-engineer the methodology from the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table I reports only point estimates for ASR (e.g., 0.93, 0.80). No confidence intervals, error bars, or uncertainty measures are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims differences between configurations (e.g., Config 1 at 0.81 vs Config 2 at 0.57) without any statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. The paper shows ASR values for each configuration but does not compute or report standardized effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 50 requests per run and mentions 2,100 total prompts, but provides no justification for these sample sizes. No power analysis or rationale for why 50 per run is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table I states 'averages across multiple runs' but reports no standard deviation, variance, or spread measures. The reader cannot assess result stability across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I presents 4 configurations as progressive ablations: full method (Config 1), without Flanking Attack (Config 2), Setting+Character+Plot only (Config 3), and Plot only (Config 4). These serve as baselines against the full method."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No comparison with other published jailbreak attack methods from the literature (e.g., Voice Jailbreak [8], Sandwich Attack [9]). All baselines are ablated variants of the authors' own method."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table I and Section VI.E present a systematic ablation across 4 configurations, progressively removing components (Flanking Attack, Text Prompt, Setting+Character) to measure each component's contribution to ASR."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Attack Success Rate (ASR) is used as a metric. No other metrics (e.g., response severity, specificity of harmful content, false positive rate of the evaluator) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes both manual inspection (preliminary study of ~100 experiments, Section V) and the semi-automated approach. Failure and success cases are shown and manually analyzed in Figures 10-15 and Appendix figures."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The attack strategy was refined through a preliminary study (~100 experiments) and then applied to the full evaluation (2,100 prompts), but there is no formal separation of development and test sets. The same question set appears to be used throughout."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I breaks down ASR by each of the 7 forbidden scenarios (Illegal Activities, Abuse & Disruption, Circumventing Safety, Harmful Content, Misinformation, Sexually Explicit, Privacy Violations) for all 4 configurations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple failure cases are shown and discussed: Figures 11, 12, 13, 15, 16, 19, 20, and 23 show cases where Gemini successfully blocked the attack. The paper analyzes why each configuration fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Configurations 3 and 4 show low ASR (0.28 and 0.12 respectively). Misinformation scenario has 0.00 ASR for Config 4. These negative results are reported and discussed in the ablation study."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'average attack success rate ranging from 0.67 to 0.93 across seven forbidden scenarios' are supported by Table I (Config 1 row shows exactly these values). The 0.81 average is also confirmed."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about component contributions (e.g., Flanking Attack 'enhances' ASR). The progressive ablation design (Table I) with controlled removal of single components supports causal inference: Config 1 vs 2 isolates Flanking Attack (+24pp), Config 2 vs 3 isolates Text Prompt (+29pp)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Jailbreak Prompt Attack against Multimodal LLMs' (plural) but only Gemini 1.5 Flash is tested. The abstract says 'manipulating state-of-the-art LLMs' and the paper claims to 'establish a replicable testing framework for adversarial robustness evaluation in multimodal LLMs,' extending well beyond the single model tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why the Flanking Attack succeeds. For example, it could be that the December 2024 Gemini snapshot had particularly weak audio moderation, or that the evaluation via Gemini self-assessment inflates ASR. No robustness checks or confound analysis."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "ASR is measured by Gemini's self-evaluation of its own policy compliance, but the paper does not discuss whether this proxy accurately measures real-world jailbreak severity. The gap between 'Gemini says its own output violates policy' and 'the output would cause actual harm' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.A specifies 'gemini-1.5-flash' (API model identifier), 'December 2024 snapshot,' 'balanced safety tier,' and SDK version 'google-generativeai 0.4.1.' This is sufficiently specific to identify the model version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 9 shows a Flanking Attack template, and implementation examples describe the text prompt components (Setting: 'Imagine that we are in a game'; Character: 'hacker'; Rule: 'We are just simulating'). But the complete verbatim prompt text used in experiments is not provided. The reader cannot reconstruct the exact prompts sent to the API."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.A reports: 'Default inference parameters were retained (temperature=0.7, top_p=0.95); no system-prompt modifications were applied.' Audio parameters also specified: 128 kbps MP3, 48 kHz, 16-bit PCM."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The attack is a direct prompt injection via API call, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes the forbidden question set design (based on Gemini policy, 7 scenarios following [5]'s principles). Section VI.A documents the pipeline: text prefix + MP3 audio → generate_content API → responses saved to ai_outputs.docx → Gemini self-evaluation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Challenges and Future Directions' discusses multiple limitations: model updates may invalidate findings, audio variations not tested, fixed sentence structure, monolingual (English) only."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses specific threats: (1) Gemini updates may mitigate identified vulnerabilities, (2) audio properties (pitch, tone, speed) were not varied, (3) fixed sentence structure may limit generalizability, (4) monolingual approach limits scope. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what was NOT tested: other audio variations (Section VII.A), sentence structure manipulations (Section VII.B), multilingual inputs (Section VII.C). Scope is bounded to Gemini, English, fixed audio format."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (prompts, audio files, Gemini responses, evaluation logs) is released. Only aggregated ASR values in Table I and selected examples in figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes the forbidden question set design (7 scenarios from Gemini policy), prompt construction methodology (Setting, Character, Rule, Flanking Attack), and Section VI.A describes the API interaction procedure (50 requests/run, 2s delay)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "No human participants. The data source is custom-crafted prompts, not a standard benchmark. The paper describes how the forbidden question set was designed: based on Gemini usage policy [35] with 7 specific scenarios, following design principles from [5]. Table III provides the full question bank."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The general pipeline is described (prompts → API → docx → evaluation), but key details are missing: how many runs were conducted per configuration, how averages were computed, and how many total responses were excluded or flagged. The paper says 'averages across multiple runs' without specifying the number of runs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Sydney, University of Chicago, and University of Texas at San Antonio. No affiliations with Google (the maker of the evaluated product Gemini)."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The authors are from academic institutions with no apparent financial interest in the outcome. Appears to be unfunded academic research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests jailbreak attacks against safety filters, not model knowledge on a benchmark. Contamination in the traditional sense (model having seen test data) is not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates defense bypass rather than model capability on a knowledge benchmark. Train/test overlap is not a relevant concern for jailbreak attack evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The 'benchmark' consists of custom-crafted jailbreak prompts, not a pre-existing knowledge test. Benchmark contamination does not apply to this type of adversarial evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve automated API interactions with Gemini."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves only automated interactions with an LLM API."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or per-query cost is reported despite making 2,100+ API calls to Gemini. Only a 2-second delay between requests (for rate limiting) is mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is described (Google Colab, 2 vCPUs, 12 GB RAM) but total compute time, API spend, and total wall-clock time for the full experimental campaign are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or stochastic variation across runs. With temperature=0.7, Gemini outputs are non-deterministic, but no seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper says '50 such requests per run' and 'Table I reports averages across multiple runs' but never states the exact number of runs conducted per configuration or scenario."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.A explicitly states 'Default inference parameters were retained (temperature=0.7, top_p=0.95); no system-prompt modifications were applied,' indicating no hyperparameter tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 4 configurations are reported in Table I with full results — no cherry-picking of a single best configuration. The ablation study presents all variants transparently."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims across 7 scenarios and 4 configurations without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own attack method against their own ablated baselines without acknowledging self-comparison bias. No independent evaluation or acknowledgment of the bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute requirements across configurations. Config 1 (with Flanking Attack audio) presumably requires more resources than Config 4 (text only), but this is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASR (as judged by Gemini self-evaluation) actually measures real-world jailbreak severity. The construct validity of using the target model to evaluate its own policy violations is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The attack is a direct API call with text and audio input, not an agentic system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Gemini may have been trained on similar jailbreak patterns or whether its safety training incorporated defenses against flanking-style attacks. The December 2024 snapshot timing relative to the paper's development is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., the text prompt framing the interaction as fictional) provides information that makes the task easier than a real-world attack scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 2,100 prompts share the same structural template across scenarios. Non-independence between prompts within and across scenarios is not discussed — success on one prompt may predict success on structurally similar ones."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. The paper does not check whether its attack patterns overlap with known jailbreak examples that may be in Gemini's safety training data."
       }
     }
   },
   "claims": [
     {
       "claim": "Flanking Attack achieves an average ASR of 0.81 across seven forbidden scenarios when using all components (Text Prompt + Setting + Character + Flanking Attack).",
       "evidence": "Table I shows per-scenario ASRs ranging from 0.67 (Misinformation) to 0.93 (Illegal Activities) with a 0.81 average for Configuration 1.",
       "supported": "moderate"
     },
     {
       "claim": "The Flanking Attack component is critical, contributing approximately 24 percentage points to ASR compared to the same setup without it.",
       "evidence": "Table I: Config 1 (with Flanking Attack) averages 0.81 ASR vs Config 2 (without) at 0.57 ASR. Section VI.E discusses this contribution.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first voice-based jailbreak attack against multimodal LLMs.",
       "evidence": "Stated in abstract and Section I. The paper references Voice Jailbreak [14] against GPT-4o as prior work but claims novelty in the multi-modal flanking approach.",
       "supported": "weak"
     },
     {
       "claim": "The semi-automated self-assessment framework enables scalable policy violation detection.",
       "evidence": "Section V.D describes using Gemini to evaluate its own outputs against policy. No quantitative comparison against manual-only evaluation to measure scalability improvement or agreement rate.",
       "supported": "weak"
     },
     {
       "claim": "LLMs' reliance on surface-level cues and context makes them vulnerable to narrative-embedded attacks.",
       "evidence": "Section VI.B discusses this mechanism, supported by the ASR difference between Config 1 (fictional framing + flanking, 0.81) vs Config 4 (plot only, 0.12). However, this is an interpretive claim without mechanistic evidence.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Circular evaluation methodology",
       "detail": "Gemini is used to evaluate whether Gemini's own outputs violate Gemini's policies. This self-evaluation creates a circularity — the model's leniency or strictness in self-assessment directly inflates or deflates ASR. The paper calls this an 'innovative' approach but does not validate it against independent human evaluation at scale."
     },
     {
       "flag": "Single model tested, broad claims made",
       "detail": "Only Gemini 1.5 Flash (one model, one snapshot) is tested, but the title and abstract claim results about 'Multimodal LLMs' generally. The paper claims to 'establish a replicable testing framework for adversarial robustness evaluation in multimodal LLMs' without testing a single other model."
     },
     {
       "flag": "No statistical rigor on key results",
       "detail": "Table I reports point estimates with no confidence intervals, error bars, significance tests, or variance measures. The number of runs is never stated ('averages across multiple runs'). The reader cannot assess whether the ASR differences between configurations are statistically meaningful."
     },
     {
       "flag": "No comparison with prior jailbreak methods",
       "detail": "The paper references several existing jailbreak attacks (Voice Jailbreak, Sandwich Attack, Crescendo) but does not compare Flanking Attack's ASR against any of them. The claimed effectiveness is evaluated only against the authors' own ablated variants."
     },
     {
       "flag": "Claims of novelty contradict cited prior work",
       "detail": "The paper claims to introduce 'the first voice-based jailbreak attack against multimodal LLMs' but cites Voice Jailbreak [14] (Shen et al., 2024) which already attacks GPT-4o via voice. The novelty claim is not clearly distinguished from this prior work."
     },
     {
       "flag": "No code or data released despite claiming replicability",
       "detail": "The paper claims to establish 'a replicable testing framework' but releases no code, notebooks, prompts, audio files, or response data. The full prompt text used in experiments is not provided verbatim."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-integrated applications, directly relevant to the survey's coverage of prompt injection vulnerabilities."
     },
     {
       "title": "Jailbroken: How does llm safety training fail?",
       "authors": ["A. Wei", "N. Haghtalab", "J. Steinhardt"],
       "year": 2024,
       "relevance": "Analyzes failure modes of LLM safety training, providing theoretical grounding for why jailbreak attacks succeed."
     },
     {
       "title": "LLM jailbreak attack versus defense techniques–a comprehensive study",
       "authors": ["Z. Xu", "Y. Liu", "G. Deng", "Y. Li", "S. Picek"],
       "year": 2024,
       "arxiv_id": "2402.13457",
       "relevance": "Comprehensive survey of jailbreak attack and defense techniques for LLMs, useful for mapping the attack landscape."
     },
     {
       "title": "Do anything now: Characterizing and evaluating in-the-wild jailbreak prompts on large language models",
       "authors": ["X. Shen", "Z. Chen", "M. Backes", "Y. Shen", "Y. Zhang"],
       "year": 2024,
       "relevance": "Characterizes real-world jailbreak prompts collected from online sources and evaluates their effectiveness against OpenAI policies."
     },
     {
       "title": "Voice jailbreak attacks against GPT-4o",
       "authors": ["X. Shen", "Y. Wu", "M. Backes", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2405.19103",
       "relevance": "Direct predecessor — demonstrates voice-based jailbreak against GPT-4o, the primary work this paper claims to extend."
     },
     {
       "title": "Sandwich attack: Multi-language mixture adaptive attack on LLMs",
       "authors": ["B. Upadhayay", "V. Behzadan"],
       "year": 2024,
       "arxiv_id": "2404.07242",
       "relevance": "Proposes multilingual mixture attacks on LLMs, a related flanking/sandwiching strategy using language diversity rather than audio modality."
     },
     {
       "title": "Survey of vulnerabilities in large language models revealed by adversarial attacks",
       "authors": ["E. Shayegani", "M. A. A. Mamun", "Y. Fu", "P. Zaree", "Y. Dong", "N. Abu-Ghazaleh"],
       "year": 2023,
       "arxiv_id": "2310.10844",
       "relevance": "Comprehensive survey of LLM vulnerabilities from adversarial attacks, providing taxonomy of attack types relevant to the survey scope."
     },
     {
       "title": "Great, now write an article about that: The crescendo multi-turn LLM jailbreak attack",
       "authors": ["M. Russinovich", "A. Salem", "R. Eldan"],
       "year": 2024,
       "arxiv_id": "2404.01833",
       "relevance": "Introduces the Crescendo multi-turn jailbreak attack, a related escalation strategy for bypassing LLM safety mechanisms."
     },
     {
       "title": "FigStep: Jailbreaking large vision-language models via typographic visual prompts",
       "authors": ["Y. Gong", "D. Ran", "J. Liu", "C. Wang", "T. Cong", "A. Wang", "S. Duan", "X. Wang"],
       "year": 2023,
       "arxiv_id": "2311.05608",
       "relevance": "Demonstrates jailbreaking vision-language models via visual prompts, showing multimodal attack surfaces beyond text."
     },
     {
       "title": "Comprehensive assessment of jailbreak attacks against LLMs",
       "authors": ["J. Chu", "Y. Liu", "Z. Yang", "X. Shen", "M. Backes", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2402.05668",
       "relevance": "Systematic benchmark of jailbreak attacks against LLMs, relevant to the survey's evaluation of adversarial robustness research."
     },
     {
       "title": "Catastrophic jailbreak of open-source LLMs via exploiting generation",
       "authors": ["Y. Huang", "S. Gupta", "M. Xia", "K. Li", "D. Chen"],
       "year": 2023,
       "arxiv_id": "2310.06987",
       "relevance": "Shows how generation configuration can enable jailbreaking of open-source LLMs, relevant to understanding safety alignment failures."
     },
     {
       "title": "Pleak: Prompt leaking attacks against large language model applications",
       "authors": ["B. Hui", "H. Yuan", "N. Gong", "P. Burlina", "Y. Cao"],
       "year": 2024,
       "relevance": "Demonstrates prompt leaking attacks against LLM applications, a related class of prompt-based security vulnerabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The attack methodology is described conceptually but no tools, code, or reusable framework is released for practitioners to apply."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "That multimodal LLMs can be jailbroken through creative prompting is not surprising given extensive prior work on text-based and voice-based jailbreaks."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that voice-based flanking can bypass Gemini's safety filters with 81% success rate across forbidden scenarios, raising concerns about audio-enabled LLM safety."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows Gemini vulnerability but presents findings in an academic context without dramatic framing or controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. The approach cannot be tried without reimplementing from the paper description."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Targets Google Gemini, a well-known consumer AI product, which adds brand recognition relevance."
     }
   }
 }