ai-research-survey

scan.json (27388B)

---

 {
   "paper": {
     "title": "Jailbreaking Leading Safety-Aligned LLMs with Simple Adaptive Attacks",
     "authors": ["Maksym Andriushchenko", "Francesco Croce", "Nicolas Flammarion"],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2404.02151"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Simple adaptive jailbreaking attacks achieve 100% attack success rate on all leading safety-aligned LLMs including Llama-2-Chat, Llama-3, Gemma, GPT-3.5/4o, Claude 3/3.5, and the adversarially trained R2D2. The key insight is that adaptivity—tailoring attack strategy to each model's specific vulnerabilities (logprobs, prefilling, in-context learning)—is far more effective than any single static attack. The authors also won 1st place in the SaTML'24 Trojan Detection Competition using restricted random search on embedding-distance-selected token pools.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code, logs, and jailbreak artifacts are released at https://github.com/tml-epfl/llm-adaptive-attacks, as stated in the abstract and Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available AdvBench dataset (50 harmful requests curated by Chao et al. 2023) and release jailbreak artifacts in JailbreakBench format."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment specifications in the paper. Hardware is mentioned only briefly (single A100 GPU in Appendix C.2)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to supplementary code but does not include step-by-step reproduction instructions in the paper itself. Algorithm 1 describes the random search but no end-to-end reproduction guide is provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point-estimate attack success rates (e.g., 100%, 96%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims its method outperforms baselines based solely on comparing ASR numbers (e.g., 100% vs 61%) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "ASR improvements are reported with baseline context, e.g., 'the previous best result on Claude 2.0 is 61% while we get 100%' (Section 4.4), providing clear magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 50 harmful requests from AdvBench without justifying why 50 is sufficient. No power analysis or discussion of whether this sample size supports the 100% ASR claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation across runs is reported. Figure 3 shows non-determinism in GPT logprobs but does not report variance of attack success rates across multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baseline comparisons are provided: GCG, PAIR, TAP, PAP, Persona Modulation, AutoDAN, and others across Tables 1-4 and Tables 21-22."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods: PAP (Zeng et al. 2024), HarmBench (Mazeika et al. 2024), TAP (Mehrotra et al. 2023), and evaluations on recent models like GPT-4o and Claude 3.5 Sonnet."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablations are provided: prompt alone vs. prompt+RS vs. prompt+RS+self-transfer (Tables 2-4), effect of suffix length (Appendix C.1, Figure 4), and ablations on Claude request structure (Tables 15-17)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four different judges are used: GPT-4, rule-based judge, Llama-3-70B, and Llama Guard 2 (Table 20 and Appendix C.6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors state 'we manually inspect all generations and flag cases with a significant number of false positives' (Section 3.1), providing human verification of the automated judge."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'all our suffixes are optimized to be request-specific and model-specific, i.e., there is no distinction between the training and test requests or models' (Section 3.2). For the trojan task, a validation set is used (Section 5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (Tables 1-4), per attack component (prompt alone, +RS, +self-transfer), and per judge type (Table 20)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "False positives of GPT-4 judge are discussed (Appendix C.5, Table 19). Cases where prompt alone fails (0% on Llama-2, GPT-4o) are reported. Claude 2.1 is flagged as producing false positives."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: prompt alone at 0% on Llama-2-Chat and GPT-4o, self-transfer ineffective on R2D2, transfer attack 0% on Claude 2.1 and Claude 3 Opus, longer suffixes hurting performance (Appendix C.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 100% ASR on listed models, which is supported by Tables 1-4. The trojan competition first place claim is supported by Table 5. All abstract claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about which components matter are supported by controlled ablations: each component (prompt, RS, self-transfer) is tested incrementally (Tables 2-4), constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states attacks are customized per model and that 'no single method can generalize across all target models' (Section 1). Results are bounded to the specific models and the 50 AdvBench requests tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses that a 10/10 judge score 'does not always imply that the generated content is actually beneficial for an attacker.' They discuss judge limitations, non-determinism effects (Figure 3), and potential defenses (Section 6, A.3)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges the gap between judge scores and actual harm: 'Even a perfect jailbreak score (10/10) from the GPT-4 judge does not always imply that the generated content is actually beneficial for an attacker' (Section 6). They use multiple judges to address this."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific API checkpoints are stated: 'gpt-3.5-turbo-1106, gpt-4-1106-preview, and gpt-4o-2024-05-13' (Section 4.3). Model sizes are given for open models (e.g., Llama-2-Chat-7B/13B/70B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided: main template in Figure 1, in-context prompt in Table 6, GPT-4o custom prompt in Table 7, judge prompt in Table 8, and all system prompts in Tables 9-14."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: 25-token suffix length, up to 10,000 iterations, up to 10 random restarts, temperature zero for GPT models, temperature one for Claude restarts. Algorithm 1 formalizes the procedure."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The attacks are direct prompt-based methods with random search optimization, not multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents using '50 harmful requests from AdvBench curated by Chao et al. (2023) that ensures distinct and diverse harmful requests' (Section 3.1). The trojan detection section documents the token selection pipeline in detail (Section 5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion, Recommendations, and Limitations' contains substantive discussion of limitations. Appendix A also covers ethics and additional discussion points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: GPT-4 judge false positives (especially on Claude 2.1), non-determinism of GPT models affecting random search signal (Figure 3), and that jailbreak judge scores don't guarantee harmful content utility (Section 6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: attacks were not tested against test-time defenses like SmoothLLM (Appendix A.3), results are specific to the 50 AdvBench behaviors, and that 'attacking SmoothLLM would require developing new adaptive attack' (Appendix A.3)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All jailbreak artifacts are released in JailbreakBench format at the GitHub repository, enabling independent verification of attack outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data source is clearly described: 50 harmful requests from AdvBench curated by Chao et al. (2023). The trojan competition setup references Rando & Tramèr (2024). Evaluation procedure is fully specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM models using a standard benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is clear: harmful request → prompt template → optional random search → LLM generation → judge evaluation. Algorithm 1 formalizes the optimization. The trojan detection pipeline is documented in Section 5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses: Google Fellowship, Open Phil AI Fellowship, unrestricted gift from Google, Swiss National Science Foundation grant 212111. OpenAI API credits and Anthropic evaluation access also disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with EPFL, clearly stated on page 1. They are independent of the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The paper is partially funded by Google (unrestricted gift) and Google Fellowship, and they receive OpenAI API credits and Anthropic evaluation access. These companies' models are being evaluated for security weaknesses, creating a potential tension — though the findings are critical of all evaluated models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attack effectiveness against safety alignment, not model knowledge on a benchmark. Contamination of the AdvBench requests in training data would not meaningfully affect attack success rates — the models are supposed to refuse these requests."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests jailbreaking defenses rather than model knowledge capabilities. The AdvBench requests are intended to be recognized as harmful and refused."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a security/red-teaming study testing defense robustness, not a capability benchmark where contamination would inflate scores."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 reports wall-clock time: '4000 iterations of random search on Llama-3-8B take 20.9 minutes on a single A100 GPU' and 'the total time of the whole experiment does not exceed a few hours.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-experiment time is given for one model, total API costs for GPT and Claude evaluations are not quantified. Hardware is mentioned (single A100) but total compute budget is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Figure 3 shows non-determinism in GPT logprobs but ASR results are not shown across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of restarts is stated throughout: 'up to 10 random restarts' (Section 3.2), and specific restart counts for Claude models in Tables 15-17 (1, 10, 100 restarts)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The prompt template was optimized on GPT-3.5 and the suffix length of 25 was selected via ablation (Figure 4), but the total search budget for template design is not quantified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows the ablation over suffix lengths with 25 performing optimally. The prompt template optimization process is described (maximizing logprob of 'Sure' on GPT-3.5). Selection criteria are transparent."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their attack against baselines but do not discuss the bias of implementing and tuning their own method while using reported numbers for baselines. Some baseline numbers come from different datasets/judges (marked with * in tables)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows convergence curves (ASR and logprob vs. iterations) for different models with and without self-transfer, explicitly showing performance as a function of compute (iterations)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses what ASR actually measures: 'Even a perfect jailbreak score does not always imply that the generated content is actually beneficial for an attacker' (Section 6). Multiple judges are used to validate construct validity (Table 20). False positive analysis is provided (Appendix C.5)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Attacks are direct prompt-based methods applied to model APIs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests attack robustness of safety alignment, not model knowledge. AdvBench requests are meant to be refused regardless of training data."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a red-teaming/attack study, not a capability benchmark where feature leakage would be meaningful."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the 50 AdvBench requests are a fixed test set for attack evaluation, not a train/test split scenario."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — contamination/leakage is not a meaningful concern for red-teaming studies testing defense robustness."
       }
     }
   },
   "claims": [
     {
       "claim": "Adaptive jailbreaking attacks achieve 100% attack success rate on all leading safety-aligned LLMs tested, including Llama-2-Chat, GPT-4o, and Claude 3.5 Sonnet.",
       "evidence": "Tables 1-4 show 100% ASR across 15+ models using GPT-4 as judge on 50 AdvBench behaviors. Table 22 provides comprehensive results.",
       "supported": "strong"
     },
     {
       "claim": "Self-transfer is key for query efficiency and high attack success rate on resistant models like Llama-2-Chat.",
       "evidence": "Table 2 shows Llama-2-Chat-7B goes from 50% ASR (prompt+RS) to 100% (prompt+RS+self-transfer). Figure 2 convergence curves show faster convergence with self-transfer initialization.",
       "supported": "strong"
     },
     {
       "claim": "The prefilling attack achieves 100% ASR on all Claude models without requiring any search.",
       "evidence": "Table 4 shows 100% ASR with prefilling on Claude 2.0, 2.1, 3 Haiku, 3 Sonnet, 3 Opus, and 3.5 Sonnet. Tables 16-17 provide ablations.",
       "supported": "strong"
     },
     {
       "claim": "No single attack method generalizes across all target models; adaptivity is crucial.",
       "evidence": "Table 2 shows prompt alone is 0% on Llama-2 but 100% on GPT-3.5 (Table 3). In-context prompt works on R2D2 (90%) but not Llama-2 (0%). Transfer attack works on Claude 3 Sonnet (100%) but not Claude 3 Opus (0%).",
       "supported": "strong"
     },
     {
       "claim": "Random search on restricted token sets won first place in the SaTML'24 Trojan Detection Competition.",
       "evidence": "Table 5 shows their method achieves best total score (-30.22 vs -29.21 for 2nd place). The embedding-distance token selection strategy is described in Section 5.",
       "supported": "strong"
     },
     {
       "claim": "The adversarially trained R2D2 model is vulnerable to in-context learning prompts, achieving 100% ASR.",
       "evidence": "Table 2 shows in-context prompt alone at 90% ASR, boosted to 100% with random search, vs. 61% best prior result from Mazeika et al. (2024). However, prior results used different request sets and judges.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Non-comparable baselines",
       "detail": "Many baseline numbers (marked with * in tables) come from different request sets, different judges (distilled GPT-4 vs GPT-4), or different experimental conditions, making direct ASR comparison imprecise. The paper acknowledges this with footnotes but still presents the numbers side by side."
     },
     {
       "flag": "GPT-4 judge false positives",
       "detail": "The paper acknowledges GPT-4 judge produces false positives, particularly on Claude 2.1 ('around 20%'). This means the reported 100% ASR on Claude 2.1 is inflated. The paper flags this but still reports 100%."
     },
     {
       "flag": "No variance reporting",
       "detail": "Attack success rates are reported as single numbers despite the stochastic nature of random search and LLM non-determinism (Figure 3). Without variance across runs, it is unclear how stable the 100% ASR claim is."
     },
     {
       "flag": "Small evaluation set",
       "detail": "All main results use only 50 harmful requests. While this is a standard benchmark, 100% ASR on 50 examples with no confidence intervals provides limited statistical power to distinguish, say, true 98% from true 100% ASR."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduced GCG attack and AdvBench benchmark — the primary baseline and evaluation dataset used in this paper."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban", "Hamed Hassani", "George J Pappas", "Eric Wong"],
       "year": 2023,
       "arxiv_id": "2310.08419",
       "relevance": "Introduced PAIR attack and the curated 50-behavior subset of AdvBench used throughout this paper."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou"],
       "year": 2024,
       "relevance": "Standardized jailbreak evaluation framework; source of R2D2 adversarially trained model and baseline numbers."
     },
     {
       "title": "On adaptive attacks to adversarial example defenses",
       "authors": ["Florian Tramèr", "Nicholas Carlini", "Wieland Brendel", "Aleksander Madry"],
       "year": 2020,
       "relevance": "Established the definition of adaptive attacks used in this paper — fundamental to the paper's methodology and argument."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Foundational RLHF safety alignment work that the jailbreaking attacks in this paper aim to circumvent."
     },
     {
       "title": "JailbreakBench: An open robustness benchmark for jailbreaking large language models",
       "authors": ["Patrick Chao", "Edoardo Debenedetti", "Alexander Robey", "Maksym Andriushchenko"],
       "year": 2024,
       "relevance": "Standardized jailbreaking benchmark; the paper's artifacts are released in JailbreakBench format."
     },
     {
       "title": "How johnny can persuade llms to jailbreak them: Rethinking persuasion to challenge AI safety by humanizing LLMs",
       "authors": ["Yi Zeng", "Hongpeng Lin", "Jingwen Zhang"],
       "year": 2024,
       "arxiv_id": "2401.06373",
       "relevance": "PAP attack achieving 92-94% ASR on Llama-2/GPT-3.5 — one of the strongest prior baselines."
     },
     {
       "title": "Improving alignment and robustness with circuit breakers",
       "authors": ["Andy Zou", "Long Phan", "Justin Wang"],
       "year": 2024,
       "relevance": "Representation-based defense approach discussed as a promising defense direction against adaptive attacks."
     },
     {
       "title": "Safety alignment should be made more than just a few tokens deep",
       "authors": ["Xiangyu Qi", "Ashwinee Panda", "Kaifeng Lyu"],
       "year": 2025,
       "relevance": "Subsequent work that reused the prefilling attack and jailbreak template from this paper."
     },
     {
       "title": "AgentHarm: A benchmark for measuring harmfulness of LLM agents",
       "authors": ["Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian"],
       "year": 2025,
       "relevance": "Extension of jailbreaking to the LLM agent setting; reused techniques from this paper."
     },
     {
       "title": "Refusal-trained LLMs are easily jailbroken as browser agents",
       "authors": ["Priyanshu Kumar", "Elaine Lau", "Saranya Vijayakumar"],
       "year": 2025,
       "relevance": "Demonstrates jailbreaking in agentic browser settings, building on techniques from this paper."
     },
     {
       "title": "Many-shot jailbreaking",
       "authors": ["Cem Anil", "Esin Durmus", "Mrinank Sharma"],
       "year": 2024,
       "relevance": "In-context learning jailbreaking approach from Anthropic; compared against in Appendix C.7."
     }
   ]
 }