ai-research-survey

scan.json (33572B)

---

 {
   "paper": {
     "title": "Multilingual Blending: LLM Safety Alignment Evaluation with Language Mixture",
     "authors": [
       "Jiayang Song",
       "Yuheng Huang",
       "Zhehua Zhou",
       "Lei Ma"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2407.07342",
     "doi": "10.48550/arXiv.2407.07342"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Multilingual Blending—mixing multiple languages at the token level in both queries and responses—significantly increases safety alignment bypass rates on LLMs (67.23% on GPT-3.5, 40.34% on GPT-4o), far exceeding single-language baselines (0–15.84%). The effect is strongest with ~4 languages and when mixing languages from different morphological types and language families. Lower-resource languages and cross-family combinations are more effective at bypassing safety mechanisms. Uncertainty analysis shows mixed-language responses exhibit ~2x higher first-token entropy than single-language counterparts.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided anywhere in the paper. No GitHub link, Zenodo archive, or supplementary materials link is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 120 samples from three public datasets (MultiJail, AdvBench, jailbreakHub) but does not release the specific curated subset or the generated mixed-language queries. The source datasets are public, but the specific selection is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model versions and some tools (Google Translate API, all-MiniLM-L6-v2 embedding model) but provides no requirements.txt, Dockerfile, or detailed environment setup section listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level in Section 3 but there are no scripts, commands, or README-style instructions to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (1–7) report only point estimate bypass rates with no confidence intervals, error bars, or uncertainty measures on the bypass rate metric itself."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., mixed vs. single language, different morphologies, different families) based solely on comparing raw percentages without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports bypass rates for both baselines and treatment conditions, allowing assessment of effect magnitude. E.g., English baseline 0% vs. mixed-language 40.34% on GPT-4o (Table 1 vs. Table 4), single-family 48.31% vs. mixed-family 57.46% (Table 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 120 malicious questions (20 per category) with no justification for why this number was chosen and no power analysis. The selection criteria are described in Appendix C but the sample size itself is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Temperature is set to 0 for LLM inference, but the random translation process introduces stochastic variation that is never quantified across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides single-language baselines across 12 languages for both GPT-3.5 and GPT-4o, against which all Multilingual Blending results are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use the same contemporary models (GPT-4o, GPT-3.5, Llama3, Mixtral, Qwen-1.5) with single-language queries, which are the appropriate comparison for evaluating the mixed-language attack vector."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Table 8) provides an ablation study separating the effects of mixed-language queries and mixed-language response instructions, showing that both components contribute to bypass rate and their combination achieves the best results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses bypass rate as the primary metric (Section 4.1) and Shannon entropy of the first token as a secondary uncertainty analysis metric (Section 3.4, Table 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Safety evaluation is entirely automated using Perspective API (Appendix F). No human evaluation of the LLM outputs was conducted to validate whether flagged responses are genuinely unsafe."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 4 as the optimal number of languages based on Table 2 results, then uses this configuration for all subsequent experiments (Tables 3–7), all on the same 120 questions. No held-out set separates the selection phase from the evaluation phase."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by resource level (Table 3), morphology type (Table 4), language family (Table 5), number of languages (Table 2), and model (Table 7). The dataset also spans 6 malicious question categories (20 each)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses conditions where Multilingual Blending is less effective: too few languages (models retain safety awareness), too many languages (models struggle to comprehend). Section 4.2 also notes Llama3-8B's limited multilingual ability reduces bypass rates."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: some combinations achieve low bypass rates (e.g., en+zh-cn only 4.17% on GPT-4o in Table 2), Llama3-8B shows limited bypass due to poor multilingual output ability, and some single-morphology combinations on GPT-4o have substantially reduced effectiveness (~58% drop)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 67.23% bypass on GPT-3.5 and 40.34% on GPT-4o, which match Table 4 (vi, fi, ko, he and th, da, hu, ar respectively). The claim that mixed-morphology and mixed-family combinations are more effective is supported by Tables 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'Multilingual Blending significantly amplifies the detriment of malicious queries' and attributes bypass to mixed-language format. However, confounds are not addressed: the translation process could introduce semantic artifacts, the back-translation evaluation pipeline could introduce errors, and the random translation itself could shift meaning in ways that trigger different model behaviors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames this as 'LLM Safety Alignment Evaluation' broadly, and conclusions state 'state-of-the-art LLMs' need multilingual safety measures. While 7 models and 55 languages were tested, the 120-question sample is small and results may not generalize to all safety scenarios, LLM architectures, or deployment configurations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly notes that models can understand mixed-language queries (Section 4.2, resource level discussion), ruling out pure incomprehension. However, it does not consider alternative explanations such as: Perspective API reliability on back-translated text, semantic drift during translation, whether the system prompt itself triggers different behavior, or whether the translation artifacts rather than the multilingual format per se cause the bypass."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'bypass rate' via Perspective API toxicity scores (threshold 0.3 on any of 10 attributes) and equates this with safety alignment failure. No discussion of the gap between Perspective API detection and genuine safety alignment compromise, or the limitations of using automated toxicity detection as a proxy for safety."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact API versions are provided: 'gpt-3.5-turbo-0125' and 'gpt-4o-2024-05-13' (Section 4.1). Open-source models are identified by specific releases: Llama3-70B, Llama3-8B, Mixtral-8x22B-instruct, Mixtral-8x7B-instruct, Qwen-1.5-72B."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides the actual system prompt text for all four experimental conditions: single-language baseline, Multilingual Blending, English-Query & Mixed-Response, and Mixed-Query & English-Query."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'The temperature is set to 0 for all experiments,' semantic similarity threshold of 0.9, embedding model all-MiniLM-L6-v2, and Perspective API threshold of 0.3 (Appendix F)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach involves direct API calls to LLMs with system prompts and queries."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the query transformation pipeline: word-based tokenization, random per-token translation to designated languages, back-translation, semantic similarity check (≥0.9 threshold). Appendix C describes the dataset curation criteria (120 samples with 'direct and explicit unsafe or harmful intentions' across 6 categories)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) contains a dedicated 'Limitation & Future Work' subsection that discusses specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: only 55 of many possible languages explored, only 60+ of many possible combinations tested, only token-level random translation used (other strategies could differ), and only morphology/family linguistic properties examined while others (e.g., syntactics) remain unstudied."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the limitations section lists things not tested, the paper does not explicitly state what the results do NOT show or what claims the authors are NOT making. The conclusions generalize broadly to 'LLMs' and 'safety alignment' without bounding claims to the specific models and conditions tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (the specific 120 questions selected, the mixed-language transformations, or the model outputs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix C describe the data source (three public datasets), selection criteria (120 samples with explicit harmful intent, 20 per category across 6 categories), and the taxonomy from Yu et al. [2024] used for categorization."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmark datasets (MultiJail, AdvBench, jailbreakHub)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The high-level pipeline is described (select questions → translate → check similarity → query LLM → back-translate → evaluate), but key quantitative details are missing: how many translations were rejected before finding valid ones, how many retries per query, what fraction of the claimed 300,000+ inference runs were for translation attempts vs. evaluation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Alberta, Canada and University of Tokyo, Japan. The authors are academic researchers evaluating commercial (OpenAI) and open-source models, with no disclosed conflicts."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding source is disclosed. The authors are at universities, but without explicit funding disclosure, independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming study testing safety alignment defenses, not evaluating model knowledge on a benchmark. Training data contamination is not the relevant concern; if models had seen these malicious questions during safety training, they would be better at refusing them."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: this paper tests safety alignment bypass, not model knowledge. Train/test overlap would work against the paper's attack vector, not for it."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for a red-teaming/safety evaluation study. Benchmark contamination concerns apply to capability evaluation, not adversarial safety testing."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve automated LLM queries and evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The ethics section (6.1) discusses broader impact but no IRB approval is needed or mentioned."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'over 300,000 million LLM inference runs' (likely a typo) but does not report actual API costs, cost per query, or latency for the Multilingual Blending pipeline including translation and evaluation costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The number of inference runs is mentioned but no dollar costs, GPU hours, or API spend is provided despite extensive use of commercial APIs (OpenAI, Google Translate, Perspective API)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for LLM inference, but the query generation involves random per-token translation to target languages. No analysis of sensitivity to this randomness is reported — different random translation assignments could yield different bypass rates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each configuration was run. Given the randomness in the translation process, multiple runs would be needed to assess stability, but it appears each configuration was evaluated once."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters include the semantic similarity threshold (0.9) and the number of languages (4, selected from Table 2). No search budget or justification for the threshold value is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of 4 languages as optimal is based on Table 2 results showing peak bypass rates around 4 languages. However, this selection is made on the same 120 questions used for all subsequent evaluation — no separate validation set is used."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable. The paper compares conditions by raw percentage differences only."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own Multilingual Blending method against baselines without acknowledging potential bias in their evaluation choices (e.g., specific language combinations selected, threshold values, dataset curation) that could favor their method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The mixed-language approach requires additional compute (Google Translate API calls, semantic similarity checks, back-translation) compared to single-language baselines. The paper notes that increasing languages 'consumes substantial computational time' but never quantifies this overhead or discusses performance-per-compute tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Perspective API with a 0.3 threshold across 10 toxicity attributes to define 'unsafe' output but does not discuss whether this automated measure is a valid proxy for genuine safety alignment failure. No validation of the evaluator against human judgments is performed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Direct API calls are used for all model evaluations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The malicious question datasets (MultiJail, AdvBench, jailbreakHub) are publicly available and could have been included in LLM safety training data. The paper does not discuss whether models may have been specifically trained to refuse these exact questions, which would make single-language bypass rates artificially low and inflate the relative effectiveness of Multilingual Blending."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The system prompt explicitly instructs models to respond in mixed-language format, which could interact with safety mechanisms in ways beyond the language mixing itself. This potential confound between instruction format and language mixing is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 120 questions drawn from three datasets may share thematic overlap or similar phrasing patterns. No analysis of independence between test examples or potential duplication across source datasets is provided."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. No analysis of whether the specific malicious questions appear in any model's safety training data."
       }
     }
   },
   "claims": [
     {
       "claim": "Multilingual Blending achieves bypass rates of 67.23% on GPT-3.5 and 40.34% on GPT-4o, far exceeding single-language baselines.",
       "evidence": "Table 4 shows mixed-morphology combinations (vi, fi, ko, he) achieving 67.23% on GPT-3.5 and (th, da, hu, ar) achieving 40.34% on GPT-4o. Table 1 shows single-language baselines ranging from 0% (English on GPT-4o) to 15.84% (Irish on GPT-3.5).",
       "supported": "moderate"
     },
     {
       "claim": "Combinations of approximately 4 languages are optimal for Multilingual Blending, with both fewer and more languages reducing effectiveness.",
       "evidence": "Table 2 shows bypass rates across 2–6 languages for three sets of combinations. Peak rates generally occur around 3–5 languages. The authors note that 2 languages leave models with some safety awareness while 6+ languages complicate query comprehension.",
       "supported": "weak"
     },
     {
       "claim": "Lower resource-level languages and mixed resource-level combinations produce higher bypass rates.",
       "evidence": "Table 3 shows high-resource combinations achieving 42.02–50.83% (GPT-3.5) vs. extremely-low-resource achieving 55.83–60.00%, and mixed-resource reaching 65.83% (GPT-3.5) and 34.17% (GPT-4o).",
       "supported": "moderate"
     },
     {
       "claim": "Mixed-morphology language combinations are more effective at bypassing safety alignment than single-morphology combinations.",
       "evidence": "Table 4 shows mixed morphology achieving the highest bypass rates (67.23% GPT-3.5, 40.34% GPT-4o) with higher averages than any single-morphology group, especially pronounced on GPT-4o.",
       "supported": "moderate"
     },
     {
       "claim": "Mixed language-family combinations achieve higher bypass rates than single-family combinations.",
       "evidence": "Table 5 shows mixed-family combinations reaching 57.46%/31.09% (GPT-3.5/GPT-4o) vs. single-family highest of 48.31%/17.95%. Germanic-only achieves only 8.33% on GPT-4o.",
       "supported": "moderate"
     },
     {
       "claim": "Uncertainty (Shannon entropy of first token) approximately doubles in mixed-language scenarios compared to single-language cases.",
       "evidence": "Table 6 shows safe single-language average uncertainties of 0.11–0.70 vs. safe mixed-language averages of 1.12–1.42. The paper states uncertainties 'increase by 120% for safe cases and 52% for bypassed cases.'",
       "supported": "moderate"
     },
     {
       "claim": "The vulnerability to Multilingual Blending generalizes beyond GPT models to Llama3, Mixtral, and Qwen.",
       "evidence": "Table 7 shows all five open-source models have substantially higher bypass rates with mixed-language combinations vs. single-language baselines: Mixtral-8x7B from 3.33% to 42.37%, Qwen-1.5-72B from 3.33% to 36.13%.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All claims about differences between conditions (language combinations, morphology types, resource levels) are based on raw percentage comparisons without any significance tests. With 120 questions and random translation, observed differences could be due to chance."
     },
     {
       "flag": "Uncontrolled randomness in translation",
       "detail": "The query transformation randomly assigns target languages per token, introducing uncontrolled stochastic variation. Results are reported as single-run point estimates without quantifying the variance introduced by different random translation assignments."
     },
     {
       "flag": "Sole reliance on automated safety evaluator",
       "detail": "Perspective API with a 0.3 threshold is the only evaluator. No human validation of whether flagged outputs are genuinely unsafe. The API's reliability on back-translated mixed-language text is unknown and could introduce systematic bias."
     },
     {
       "flag": "Small sample size for the claims being made",
       "detail": "120 malicious questions (20 per category) are used to evaluate 60+ language combinations across 7 models. With 20 questions per category, a single question flipping between safe/unsafe changes the bypass rate by 5 percentage points."
     },
     {
       "flag": "Configuration selection on test data",
       "detail": "The number of languages (4) is selected based on Table 2 results on the same 120 questions used for all subsequent experiments. This optimization on test data could inflate reported bypass rates."
     },
     {
       "flag": "No code or data released",
       "detail": "Despite the study involving specific curated datasets, translation pipelines, and evaluation code, nothing is released. The specific 120-question subset and mixed-language transformations cannot be independently verified."
     }
   ],
   "cited_papers": [
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2024,
       "relevance": "Systematic analysis of how LLM safety training can be circumvented, directly relevant to understanding jailbreak attack vectors."
     },
     {
       "title": "LLM jailbreak attack versus defense techniques–a comprehensive study",
       "authors": ["Zihao Xu", "Yi Liu", "Gelei Deng", "Yuekang Li", "Stjepan Picek"],
       "year": 2024,
       "arxiv_id": "2402.13457",
       "relevance": "Comprehensive assessment of jailbreak attacks and defenses, providing the attack landscape context for this study."
     },
     {
       "title": "Multilingual jailbreak challenges in large language models",
       "authors": ["Yue Deng", "Wenxuan Zhang", "Sinno Jialin Pan", "Lidong Bing"],
       "year": 2023,
       "arxiv_id": "2310.06474",
       "relevance": "Prior work on multilingual jailbreaking showing low-resource languages bypass safety, which this paper extends to mixed-language settings."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Foundational RLHF safety alignment work that the Multilingual Blending attacks are testing."
     },
     {
       "title": "The language barrier: Dissecting safety challenges of LLMs in multilingual contexts",
       "authors": ["Lingfeng Shen", "Weiting Tan", "Sihao Chen"],
       "year": 2024,
       "arxiv_id": "2401.13136",
       "relevance": "Closely related work examining multilingual safety challenges in LLMs, demonstrating language-dependent safety degradation."
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to!",
       "authors": ["Xiangyu Qi", "Yi Zeng", "Tinghao Xie"],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Demonstrates that fine-tuning can compromise safety alignment, relevant to understanding alignment fragility."
     },
     {
       "title": "GPT-4 is too smart to be safe: Stealthy chat with LLMs via cipher",
       "authors": ["Youliang Yuan", "Wenxiang Jiao", "Wenxuan Wang"],
       "year": 2023,
       "arxiv_id": "2308.06463",
       "relevance": "Cipher-based jailbreak attack on LLMs, an alternative encoding-based approach to bypassing safety alignment."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Prompt injection attacks on LLM applications, relevant to the broader landscape of LLM security threats."
     },
     {
       "title": "Certifying LLM safety against adversarial prompting",
       "authors": ["Aounon Kumar", "Chirag Agarwal", "Suraj Srinivas"],
       "year": 2023,
       "arxiv_id": "2309.02705",
       "relevance": "Defense-side work on certifying LLM safety against adversarial inputs, complementary to this attack-side study."
     },
     {
       "title": "How alignment and jailbreak work: Explain LLM safety through intermediate hidden states",
       "authors": ["Zhenhong Zhou", "Haiyang Yu", "Xinghua Zhang"],
       "year": 2024,
       "arxiv_id": "2406.05644",
       "relevance": "Mechanistic study of how safety alignment and jailbreaks affect LLM internal representations."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG adversarial attack framework; the AdvBench dataset used in this study originates from this work."
     },
     {
       "title": "Look before you leap: An exploratory study of uncertainty measurement for large language models",
       "authors": ["Yuheng Huang", "Jiayang Song", "Zhijie Wang"],
       "year": 2023,
       "arxiv_id": "2307.10236",
       "relevance": "Uncertainty measurement framework for LLMs that this paper's uncertainty analysis methodology builds upon."
     },
     {
       "title": "Llama guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LLM safety guardrail system representing the defense mechanisms that Multilingual Blending aims to bypass."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly useful for red-teamers and safety practitioners evaluating multilingual attack surfaces, though no released tools."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Extends known findings about low-resource language vulnerabilities to mixed-language settings; the mixed-language angle is novel but builds on expected patterns."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates a 40% bypass rate on GPT-4o's safety alignment using a straightforward technique that requires no crafted jailbreak templates."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques the single-language focus of safety alignment efforts at major labs, but framed academically rather than confrontationally."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released; the technique requires a custom translation pipeline to reproduce."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates GPT-4o, GPT-3.5, Llama3, and Mixtral — all well-known models — but the authors and labs are not household names."
     }
   }
 }