ai-research-survey

calibration.json (19122B)

---

 {
   "paper_slug": "adaptive-attacks-bypass-defenses-2025",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "setup_transparency",
       "question": "prompts_provided",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The paper shows examples of successful attack triggers (e.g., the RL trigger against RPO in Section 5.1, the search trigger against Prompt Sandwiching), but these are outputs of the attack optimization process, not the prompts used to drive the experiments. The actual experimental prompts — the LmMutator system prompt, the critic LLM prompt, the scoring function prompts — are described only in natural language (e.g., 'consists of sections: broad context, the attacker's task, and other miscellaneous information' in Appendix D). Per the schema, 'If prompts are described only in natural language without the actual text, NO.' The LmMutator system prompt is stated to have been 'generated by another LLM and then manually edited' but its actual text is not provided.",
       "sonnet_justification": "The paper provides actual prompt examples used in attacks (e.g., the successful RL trigger against RPO in Section 5.1, the search trigger against Prompt Sandwiching, the RL trigger against Data Sentinel in Appendix C.1). These are the actual prompts sent to models, not just templates.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided anywhere in the paper. The attack methods are described in detail in Appendices A-D but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HarmBench, AgentDojo, OpenPromptInject, and the Alpaca/Davinci dataset (with an explicit HuggingFace URL in Section B.1). These are standard public benchmarks that readers can obtain independently."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements file, Dockerfile, or detailed environment specification is provided. The paper mentions using Gemini-2.5 Pro as LmMutator and various LLM APIs but does not provide library versions or reproducible environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The appendices describe attack methods at a conceptual level (PSSU loop, MAP Elites controller, LmMutator design) but there are no step-by-step commands, scripts, or README to allow a researcher to reproduce results without significant guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All ASR results in Tables 2-7 and Figure 1 are point estimates. No confidence intervals, error bars, or standard deviations are reported anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares ASRs between static/weak attacks and adaptive attacks (e.g., 0% vs. 99% for Spotlighting) but no statistical significance tests (p-values, bootstrap tests, etc.) are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute ASR values for both static attacks and adaptive attacks with clear baselines (e.g., 'near-zero ASR' for static vs. '>90% ASR' for adaptive). Figure 1 and Table 7 show both baseline and attack ASR values, providing sufficient context to assess the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 80 samples from AgentDojo (Slack, Travel, Workspace suites) without justifying this sample size. No power analysis is discussed for any of the experiments."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 reports 'Median Num. Queries' for search attacks but no standard deviation, IQR, or other spread measure. ASR figures are single point estimates with no variance reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each defense is evaluated against both the original paper's static/weak attack (baseline) and the authors' adaptive attacks. Figure 1 shows side-by-side ASR comparison for all 12 defenses."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 12 defenses evaluated are all from 2024-2025 (e.g., Circuit Breakers 2024, StruQ 2024, MetaSecAlign 2025, Data Sentinel 2025, MELON 2025). The baseline attacks used are those from the original defense papers."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents four attack families (gradient, RL, search, human) but does not systematically ablate individual components within each attack. For instance, the search attack's controller, mutator, and scorer components are not individually removed to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses multiple metrics: attack success rate (ASR), number of queries until first success (Table 7, Figure 6), and utility of the defended system (Table 7 'Utility' column)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper runs a human red-teaming competition with 500+ participants and $20,000 in prizes (Appendix E). Human attacks are directly compared against automated attacks across 29 scenarios."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard benchmark datasets (HarmBench, AgentDojo, OpenPromptInject) that are separate from the attack design process. The attacks are adaptive to defenses, not tuned to specific test examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by defense type (Prompting, Training, Filtering Model, Secret Knowledge), by individual defense, and by base model within each defense (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Reward hacking is discussed as a failure mode (Appendix C.1, Figure 5) where the RL attacker finds strategies that score well but do not constitute real attacks. PIGuard's lower ASR (71%) relative to other defenses is also acknowledged."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Gradient-based attacks are described as 'generally unreliable' with a recommendation to use text-space attacks instead (Section 4). The search attack reaches only 69% ASR vs. 100% for human red-teamers (Appendix F)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 12 defenses are bypassed with ASR 'above 90% for most.' This is supported by Figure 1 and Tables 1-7. The qualification 'for most' is accurate since PIGuard reaches 71% and a few other settings fall below 90%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim is that adaptive attacks (vs. static/weak attacks) cause higher ASR against the same defenses. This is demonstrated through controlled comparisons: same defense, same benchmark, different attack method. This is adequate causal identification for this claim type."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states 'the goal of this section is not to provide a full evaluation of defenses across all attacks or to compare the effectiveness of multiple defenses.' Results are scoped to the 12 tested defenses on specific benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F.1 discusses specific challenges in interpreting results: human skill heterogeneity may artificially lower average ASR, differing query budgets complicate human-vs-automated comparison, and collective success may not reflect individual performance. Reward hacking (Appendix C.1) is discussed as an alternative explanation for apparent attack success."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names like 'Gemini-2.5 Pro', 'GPT-5', 'Llama-3.3-70B', and 'Gemini-2.5 Flash' without specific API version dates or snapshot identifiers. Per the schema, marketing names without a snapshot date or API version do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows examples of successful attack triggers (outputs of the optimization), but the actual experimental setup prompts are not provided. The LmMutator system prompt is described only in natural language in Appendix D ('consists of sections: broad context, the attacker's task, and other miscellaneous information') — the actual text is not given. The critic LLM prompts and scoring function prompts are also described conceptually. Per the schema, natural language descriptions without actual prompt text are NO."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Some parameters are stated (32 independent sessions, 5 rounds for RL; 8 candidates per mutation for search) but key hyperparameters are missing: temperature, learning rate, batch size for RL training, reward weights. The LmMutator uses 'maximum thinking budget' which is not a precise numerical value."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the four-step PSSU loop. Appendices A-D provide detailed descriptions of each attack family's scaffolding: the MAP Elites controller, LmMutator design, island-based database, scoring mechanisms, and feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes benchmark details: AgentDojo subset of 80 samples from Slack, Travel, and Workspace suites with explanation for why Banking was excluded. OpenPromptInject task pairs and the Alpaca/Davinci dataset source URL are provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Section 6 is titled 'Lesson and Discussion' and Appendix F.1 is 'Challenges.' While limitations are discussed throughout, there is no dedicated section for them."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F.1 identifies specific threats: human skill heterogeneity affecting average ASR, differing costs between human and automated queries complicating comparison, and the caveat that collective human success does not reflect individual performance. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'the goal of this section is not to provide a full evaluation of defenses across all attacks or to compare the effectiveness of multiple defenses.' The paper explicitly scopes its contribution to demonstrating that adaptive attacks exist and are effective."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw attack outputs, successful trigger strings, and model response logs are not publicly available. Only selected examples of successful triggers are shown in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The human red-teaming competition setup is described in Appendix E: online platform, 500+ participants, $20,000 prize structure, AgentDojo evaluation environment, scoring mechanism with token penalty, and human judge adjudication for edge cases."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we run a human AI red-teaming competition' with 500+ participants but does not describe how participants were recruited, what channels or communities were used, or whether the participant pool introduces selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from attack submission to ASR is documented: Appendix B describes benchmark setup, Appendix C describes scoring functions for each defense, and Appendix E.2 describes how human submissions are evaluated (automatic + human adjudication for appeals)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section listing grants, funding agencies, or corporate sponsors. The $20,000 competition prize funding source is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on page 1: OpenAI, Anthropic, Google DeepMind, HackAPrompt, Northeastern University, ETH Zurich, AI Security Company, and MATS."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors from Google DeepMind use Gemini-2.5 Pro as the LmMutator and evaluate defenses on Google models. OpenAI employees participate while GPT-5 is evaluated. Anthropic employees participate. These are non-independent relationships between authors/funders and outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial conflict-of-interest declaration is present. Authors at OpenAI, Anthropic, and Google have potential financial interests that are not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense robustness against adversarial attacks, not pre-trained model knowledge on benchmarks. Contamination of training data is not relevant to measuring whether adaptive attacks can bypass defenses."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above: the paper tests whether attacks can bypass defenses, not whether models have memorized test examples. Train/test overlap is not applicable to this evaluation design."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not relevant to adaptive attack evaluations of security defenses. The paper tests defense mechanisms, not model knowledge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human red-teaming competition involved 500+ human participants, but no pre-registration link (OSF, AsPredicted, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics Statement mentions voluntary participation and anonymized data collection, but no IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '500+ participants' but provides no demographic information about them — no experience level, geographic distribution, or professional background."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for competition participants are stated. There is no description of eligibility requirements or screening."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human red-teaming is a competition where participants self-select which challenges to attempt, not an experimental study with random assignment to conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states 'the name of the model is replaced with a pseudonym, and participants are unaware of defenses potentially being deployed,' constituting partial blinding of participants to both model identity and defense presence."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 500+ total participants and notes 'not every participant attempts every scenario,' but the number of participants per scenario and dropout/completion rates are not systematically reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite using Gemini-2.5 Pro with 'maximum thinking budget' as LmMutator, running 32 independent sessions per attack instance, and testing across 12 defenses with multiple benchmarks, no API costs, token counts, or latency figures are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is stated. The competition prizes ($20,000) are mentioned but the infrastructure and compute costs of running the attacks are not quantified."
       }
     }
   }
 }