ai-research-survey

scan.json (28743B)

---

 {
   "paper": {
     "title": "The Attacker Moves Second: Stronger Adaptive Attacks Bypass Defenses Against LLM Jailbreaks and Prompt Injections",
     "authors": [
       "Milad Nasr",
       "Nicholas Carlini",
       "Chawin Sitawarin",
       "Sander V. Schulhoff",
       "Jamie Hayes",
       "Michael Ilie",
       "Juliette Pluto",
       "Shuang Song",
       "Harsh Chaudhari",
       "Ilia Shumailov",
       "Abhradeep Thakurta",
       "Kai Yuanqing Xiao",
       "Andreas Terzis",
       "Florian Tramèr"
     ],
     "year": 2025,
     "venue": "Preprint (arXiv, under review)",
     "arxiv_id": "2510.09023"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. The paper describes attack methods in detail (Appendix A-D) but does not link to any public implementation."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HarmBench, AgentDojo, OpenPromptInject, and the Alpaca/Davinci dataset (with explicit HuggingFace URL in Section B.1). These are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements file, Dockerfile, or detailed environment specification is provided. The paper mentions using Gemini-2.5 Pro as LmMutator and various LLM APIs but does not provide library versions or reproducible environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The appendices describe the attack methods at a conceptual level, but there are no step-by-step commands, scripts, or README that would allow a researcher to reproduce the main results without significant guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2-7 and Figure 1 report only point estimates of attack success rates (ASR). No confidence intervals, error bars, or standard deviations are reported anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares ASRs between static/weak attacks and adaptive attacks (e.g., 0% vs. 99% for Spotlighting), but no statistical significance tests are performed. Differences are compared numerically without hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute ASR values for both static attacks and adaptive attacks, providing sufficient context for effect size (e.g., 'near-zero ASR' for static vs. '>90% ASR' for adaptive). Figure 1 and Table 7 make magnitude explicit."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 80 samples from AgentDojo (Slack, Travel, Workspace suites) without justifying this sample size. The human red-teaming study uses 500+ participants but does not discuss power or adequacy of the sample for drawing conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 reports 'Median Num. Queries' for search attacks but no standard deviation, IQR, or other spread measure. ASR figures are single point estimates across runs with no variance reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each defense is evaluated against both the original paper's static/weak attack (the baseline) and the authors' adaptive attacks, providing a clear before/after comparison. Figure 1 shows side-by-side ASR for static and adaptive attacks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 12 defenses published in 2024-2025 using their own evaluation protocols. The GCG attack used as one baseline is 2023 and still considered a standard reference attack. The authors note defenses used generic existing attacks like GCG, which is their point."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "There is no ablation study on the attack components themselves. The paper describes four attack families (gradient, RL, search, human) and uses them separately, but does not systematically ablate individual components of the search or RL attack to measure each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses multiple metrics: attack success rate (ASR), number of queries until first success (Table 7, Figure 6), utility of the defended system, and qualitative analysis of successful attack examples."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper runs a human red-teaming competition with 500+ participants and $20,000 in prizes (Section E). Human attacks are compared directly against automated attacks across 29 scenarios, making human evaluation a central component."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on fixed benchmark datasets (HarmBench, AgentDojo, OpenPromptInject) that are separate from anything used to design the attacks. The attacks are adaptive by design rather than tuned to a specific held-out set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by defense type (Prompting, Training, Filtering Model, Secret Knowledge) and by individual defense within each category. Table 7 further breaks down by model and defense combination."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses reward hacking as a failure mode (Appendix C.1, Figure 5) where the RL attacker finds spurious strategies that score well but do not constitute real attacks. PIGuard's lower ASR (71%) relative to other defenses is acknowledged."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that gradient-based attacks are 'generally unreliable' and recommends attacks in the text space instead (Section 4). The paper also notes in Appendix F that search attacks succeed on only 69% of scenarios vs. 100% for human red-teamers, acknowledging limitations of automated attacks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 12 defenses are bypassed with ASR above 90%. This is supported by Figure 1 and Tables 1-7, though some defenses (PIGuard at 71%, MELON human at 89%) fall slightly below the 'above 90%' threshold; the abstract accurately qualifies with 'above 90% for most.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim is that adaptive attackers cause high ASR compared to static attacks. This is supported by directly applying adaptive attacks to defended systems and measuring ASR. The controlled comparison (same defense, different attack strength) provides adequate causal identification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'robustness numbers are not necessarily comparable across defenses' and 'the goal of this section is not to provide a full evaluation of defenses across all attacks' (Section 5). Conclusions are bounded to the 12 tested defenses and specific benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F.1 discusses challenges in interpreting results, including that human ASR may be artificially lower due to inexperienced participants, and that comparison between human and automated attacks is complicated by differing query budgets and participant skill variance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Gemini-2.5 Pro', 'GPT-5', 'Llama-3.3-70B', and 'Gemini-2.5 Flash' but does not provide specific API versions or snapshot dates. Marketing names without version identifiers do not satisfy this criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows examples of successful attack triggers (e.g., the RL trigger against RPO in Section 5.1, the search trigger against Prompt Sandwiching), but these are outputs of the attack optimization process, not the prompts used to drive the experiments. The actual experimental prompts — the LmMutator system prompt, the critic LLM prompt, the scoring function prompts — are described only in natural language (e.g., 'consists of sections: broad context, the attacker's task, and other miscellaneous information' in Appendix D). Per the schema, 'If prompts are described only in natural language without the actual text, NO.' The LmMutator system prompt is stated to have been 'generated by another LLM and then manually edited' but its actual text is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The RL attack uses GRPO with 32 independent sessions and 5 rounds per session (Appendix A.2), but key hyperparameters like learning rate, batch size, temperature, and reward weights are not fully specified. Search attack uses Gemini-2.5 Pro with 'maximum thinking budget' (not a numerical value)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendices A-D describe the attack scaffolding in detail: the four-step PSSU loop (Propose, Score, Select, Update), the MAP Elites controller, LmMutator design, and scoring functions for each defense (Appendix C)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the benchmarks used. For AgentDojo, the paper specifies using 80 samples from Slack, Travel, and Workspace suites and explains why Banking was excluded. For OpenPromptInject, the source dataset and task pairs are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 6 ('Lessons and Discussion') and Appendix F.1 discuss challenges, but there is no section explicitly titled 'Limitations' or equivalent."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F.1 provides specific discussion of validity threats: human skill heterogeneity affecting ASR comparisons, differing query costs between human and automated attacks, and the fact that human collective success may not reflect individual performance."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states 'the goal of this section is not to provide a full evaluation of defenses across all attacks or to compare the effectiveness of multiple defenses.' The paper is scoped to demonstrating that adaptive attacks exist, not ranking defenses."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw attack outputs, successful trigger strings, and model response logs are not publicly available. Successful prompt examples are shown selectively in the paper but the full dataset of attack attempts is not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The human red-teaming competition setup is described in detail in Appendix E: online platform, 500+ participants, $20,000 prize structure, AgentDojo evaluation environment, scoring mechanism, and human judge adjudication for edge cases."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we run a human AI red-teaming competition' with 500+ participants but does not describe how participants were recruited, what channels were used, or whether the participant pool introduces selection bias toward more skilled attackers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes the benchmark setup, Appendix C describes scoring functions for each defense, and Appendix E.2 describes how human submissions are evaluated (automatic + human adjudication for appeals). The pipeline from attack submission to ASR is clear."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section disclosing grants, funding agencies, or corporate sponsors. The human competition offered $20,000 in prizes but the funding source for this is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are affiliated with OpenAI, Anthropic, Google DeepMind, HackAPrompt, Northeastern University, ETH Zürich, MATS, and AI Security Company. These affiliations are listed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors from Anthropic and Google DeepMind evaluate defenses against attacks and also use Gemini-2.5 Pro (Google's model) as the LmMutator and as the base model for some evaluations. OpenAI employees are on the team while 'GPT-5' models are evaluated. These are non-independent relationships."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial conflict-of-interest declaration is present in the paper. Authors at AI companies (OpenAI, Anthropic, Google) have potential financial interests but these are not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model knowledge on a benchmark — it evaluates security defenses against adversarial attacks. Contamination of training data is not relevant to measuring defense robustness to adaptive attacks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above: the paper tests whether attacks can bypass defenses, not whether models have memorized test examples. Contamination is not applicable to this evaluation design."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above: benchmark contamination is not relevant to adaptive attack evaluations of security defenses."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human red-teaming competition involved human participants making decisions, but no pre-registration link (OSF, AsPredicted, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics Statement (Section after Conclusion) mentions 'all participation was voluntary' and 'no personally identifiable or sensitive data were collected,' but no IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '500+ participants' in the competition but provides no demographic information (experience level, geographic distribution, professional background)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for competition participants are stated. The paper does not describe any screening or eligibility requirements."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human red-teaming is not an experimental study with random assignment to conditions — participants self-select which challenges to attempt. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states that 'the name of the model is replaced with a pseudonym, and participants are unaware of defenses potentially being deployed,' constituting partial blinding of participants."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 500+ total participants but notes 'not every participant attempts every scenario.' The dropout or participation rate per scenario is not systematically reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Gemini-2.5 Pro with 'maximum thinking budget' as LmMutator, runs 32 independent sessions per attack instance, and tests across 12 defenses with multiple benchmarks, but no API costs or token counts are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is stated. The competition prizes ($20,000) are mentioned but the infrastructure and compute costs of running the attacks are not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Adaptive attacks bypass 12 recent LLM defenses with attack success rate above 90% for most, compared to near-zero rates reported in original papers.",
       "evidence": "Figure 1 shows side-by-side ASR for static attacks vs. adaptive attacks across all 12 defenses. Table 7 provides per-defense, per-model breakdown with specific ASR values.",
       "supported": "strong"
     },
     {
       "claim": "Human red-teaming succeeds on all tested scenarios (100% ASR) while search-based attacks succeed on 69% of the 29 overlapping scenarios.",
       "evidence": "Figure 6 and Appendix F.2 compare human and automated attack success rates on 29 AgentDojo scenarios. The human collective ASR curve reaches 100% while search attack plateaus at 69%.",
       "supported": "strong"
     },
     {
       "claim": "Prompting-based defenses (Spotlighting, Prompt Sandwiching) can be bypassed with ASR above 95% using search-based adaptive attacks.",
       "evidence": "Section 5.1 reports >95% ASR for both Spotlighting and Prompt Sandwiching using search-based attacks on AgentDojo. Human red-teaming generated 265 and 178 successful attacks respectively.",
       "supported": "strong"
     },
     {
       "claim": "Training-based defenses (Circuit Breakers, StruQ, MetaSecAlign) offer no meaningful security against adaptive attacks, with RL attacks achieving 96-100% ASR.",
       "evidence": "Section 5.2 reports 100% ASR against Circuit Breakers, success on every StruQ test case, and 96% ASR against MetaSecAlign using RL-based attacks.",
       "supported": "strong"
     },
     {
       "claim": "Current LLM defense evaluations are flawed because they rely on static attack sets or computationally weak attacks not adapted to the defense.",
       "evidence": "Section 1 and Section 6 (Lesson #1) argue that static datasets cause false sense of security; supported by the empirical finding that defenses claiming near-zero ASR on static attacks fail under adaptive attacks.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "The paper demonstrates that 12 recent LLM defenses against jailbreaks and prompt injections, which reported near-zero attack success rates (ASR) in their original evaluations, are all bypassed with ASR above 90% when evaluated against adaptive attacks using gradient descent, reinforcement learning, evolutionary search, and human red-teaming. Human red-teaming in a competition with 500+ participants achieved 100% ASR across all tested scenarios while the best automated search attack reached 69%. The paper argues that defense evaluations must use adaptive attackers that explicitly counter the defense design, and that static benchmark evaluations create a false sense of security in the LLM safety literature.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All ASR results are point estimates with no confidence intervals, error bars, or standard deviations reported. Given that ASR is computed over finite sample sizes (e.g., 80 AgentDojo examples), sampling variability could affect conclusions, especially for defenses near the 90% threshold."
     },
     {
       "flag": "Conflict of interest: authors evaluate competitors' systems",
       "detail": "Authors from Anthropic, Google DeepMind, and OpenAI participate in evaluating defenses, and the search attack uses Gemini-2.5 Pro (Google's model). The paper evaluates defenses partly against models from the same companies as the authors, creating a potential non-independence issue."
     },
     {
       "flag": "No code or full data release",
       "detail": "The adaptive attack framework described is novel and central to the paper's claims, but no code is released. This limits reproducibility and verification of results."
     },
     {
       "flag": "Human participant recruitment not described",
       "detail": "The 500+ competition participants' recruitment method is not described, potentially introducing selection bias toward more skilled attackers that inflates the perceived 100% human ASR relative to what an average red-teamer would achieve."
     },
     {
       "flag": "Incomplete model version specification",
       "detail": "The paper uses marketing names like 'Gemini-2.5 Pro', 'GPT-5', and 'Llama-3.3-70B' without specific API version dates or snapshot identifiers, making it impossible to know exactly which model versions were evaluated."
     },
     {
       "flag": "Missing limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. While some limitations are discussed in the lessons and appendix, there is no systematic enumeration of what the results do not show."
     }
   ],
   "cited_papers": [
     {
       "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal",
       "authors": [
         "Mantas Mazeika",
         "Long Phan",
         "Xuwang Yin",
         "Andy Zou"
       ],
       "year": 2024,
       "relevance": "Core benchmark used for evaluating jailbreak defenses; directly relevant to LLM safety evaluation methodology."
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents",
       "authors": [
         "Edoardo Debenedetti",
         "Jie Zhang",
         "Mislav Balunovic",
         "Luca Beurer-Kellner",
         "Marc Fischer",
         "Florian Tramèr"
       ],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Core benchmark for prompt injection evaluation in agentic settings; evaluates robustness of LLM agent defenses."
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": [
         "Andy Zou",
         "Zifan Wang",
         "Nicholas Carlini",
         "Milad Nasr",
         "J. Zico Kolter",
         "Matt Fredrikson"
       ],
       "year": 2023,
       "relevance": "GCG attack paper; key baseline attack method used to evaluate LLM defenses."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": [
         "Kai Greshake",
         "Sahar Abdelnabi",
         "Shailesh Mishra",
         "Christoph Endres",
         "Thorsten Holz",
         "Mario Fritz"
       ],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Foundational work on prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "Improving Alignment and Robustness with Circuit Breakers",
       "authors": [
         "Andy Zou",
         "Long Phan",
         "Justin Wang"
       ],
       "year": 2024,
       "relevance": "One of the 12 defenses evaluated; training-based jailbreak defense using representation engineering."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": [
         "Sizhe Chen",
         "Julien Piet",
         "Chawin Sitawarin",
         "David Wagner"
       ],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "One of the 12 defenses evaluated; adversarial training approach for prompt injection defense."
     },
     {
       "title": "Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples",
       "authors": [
         "Anish Athalye",
         "Nicholas Carlini",
         "David Wagner"
       ],
       "year": 2018,
       "relevance": "Foundational work on adaptive attacks against adversarial example defenses; the conceptual basis for the paper's argument."
     },
     {
       "title": "Ignore this title and HackAPrompt: Exposing Systemic Vulnerabilities of LLMs through a Global Prompt Hacking Competition",
       "authors": [
         "Sander Schulhoff",
         "Jeremy Pinto",
         "Anaum Khan"
       ],
       "year": 2023,
       "relevance": "Previous human red-teaming competition for LLMs; directly relevant to human-based adversarial evaluation."
     },
     {
       "title": "Tensor Trust: Interpretable Prompt Injection Attacks from an Online Game",
       "authors": [
         "Sam Toyer",
         "Olivia Watkins",
         "Ethan Adrian Mendes"
       ],
       "year": 2023,
       "arxiv_id": "2311.01011",
       "relevance": "Human-based prompt injection evaluation through gamification; relevant to human red-teaming methodology."
     },
     {
       "title": "Jailbreaking Leading Safety-Aligned LLMs with Simple Adaptive Attacks",
       "authors": [
         "Maksym Andriushchenko",
         "Francesco Croce",
         "Nicolas Flammarion"
       ],
       "year": 2024,
       "arxiv_id": "2404.02151",
       "relevance": "Prior work on adaptive jailbreak attacks; directly related to the paper's argument for stronger evaluation."
     },
     {
       "title": "DataSentinel: A game-theoretic detection of prompt injection attacks",
       "authors": [
         "Yupei Liu",
         "Yuqi Jia",
         "Jinyuan Jia",
         "Dawn Song",
         "Neil Zhenqiang Gong"
       ],
       "year": 2025,
       "relevance": "One of the 12 defenses evaluated; secret-knowledge based prompt injection defense."
     },
     {
       "title": "MELON: Provable Defense Against Indirect Prompt Injection Attacks in AI Agents",
       "authors": [
         "Kaijie Zhu",
         "Xianjun Yang",
         "Jindong Wang",
         "Wenbo Guo",
         "William Yang Wang"
       ],
       "year": 2025,
       "relevance": "One of the 12 defenses evaluated; dual-run defense for agentic prompt injection."
     },
     {
       "title": "On Adaptive Attacks to Adversarial Example Defenses",
       "authors": [
         "Florian Tramèr",
         "Nicholas Carlini",
         "Wieland Brendel",
         "Aleksander Madry"
       ],
       "year": 2020,
       "relevance": "Framework for adaptive attack evaluation that this paper extends to the LLM domain."
     }
   ]
 }