ai-research-survey

calibration.json (22801B)

---

 {
   "paper_slug": "agentdojo-dynamic-environment-2024",
   "calibration_date": "2026-02-28",
   "calibrator": "opus",
   "total_questions": 50,
   "agreement_count": 45,
   "disagreement_count": 5,
   "agreement_rate": 0.90,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "held_out_test_set",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "This paper introduces a new benchmark framework and evaluates pre-trained LLMs via API on it. There is no training involved and no dev/test split. The concept of a 'held-out test set' is structurally inapplicable because the LLMs are not tuned on any portion of the AgentDojo data. The benchmark IS the evaluation set. Sonnet treated the benchmark as a held-out test set, but the schema asks about separation of dev/test splits which does not apply here."
     },
     {
       "category": "setup_transparency",
       "question": "model_versions_specified",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The code in Figure 12 shows 'gpt-4o-2024-05-13' for one model, but the paper text refers to models by marketing names only: 'Claude 3 Sonnet', 'Claude 3 Opus', 'Claude 3.5 Sonnet', 'GPT-3.5 Turbo', 'GPT-4 Turbo', 'GPT-4o', 'Gemini 1.5 Flash', 'Gemini 1.5 Pro', 'Llama 3 70B', 'Command R+'. The schema requires 'specific model names with version (e.g., gpt-4-0613)' and states 'marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count.' One snapshot date in a code figure for one model out of ten does not satisfy this criterion. Most models lack version/snapshot identifiers."
     },
     {
       "category": "contamination",
       "question": "training_cutoff_stated",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "The schema states NA if 'the paper does not evaluate a pre-trained model's capability on any benchmark' and includes 'studies that test defenses/tools rather than model knowledge.' AgentDojo evaluates prompt injection attacks and defenses — it tests security properties and task-completion utility in a stateful tool-calling environment, not model knowledge or capability on a benchmark in the traditional sense. The tasks are newly designed by the authors and require dynamic interaction, not recall of training data. Contamination is largely irrelevant here."
     },
     {
       "category": "contamination",
       "question": "train_test_overlap_discussed",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning as training_cutoff_stated. The paper evaluates security properties (prompt injection robustness) in a dynamic tool-calling environment, not model knowledge on a static benchmark. Train/test overlap is not a meaningful concern for a security benchmark where tasks require multi-step stateful tool interaction. The NA rule states 'if the paper does not evaluate a pre-trained model on any benchmark (same NA rule as training_cutoff_stated).' While one could argue this is a benchmark evaluation, the nature of the tasks (dynamic tool calling, security testing) makes contamination structurally irrelevant."
     },
     {
       "category": "contamination",
       "question": "benchmark_contamination_addressed",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning as the other contamination items. The benchmark tasks were newly created in 2024, involve dynamic stateful tool interactions, and test security properties rather than model knowledge. Even if a model had 'seen' similar task descriptions, this would not meaningfully inflate its scores on the actual benchmark since performance depends on correct multi-step tool orchestration in a specific environment state. The contamination concern is structurally inapplicable."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released under MIT license at https://github.com/ethz-spylab/agentdojo (Section 1, Appendix E.1). A Zenodo DOI (10.5281/zenodo.12528188) is also provided in Appendix E.6."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks, tools, and environment state are released via the GitHub repository. Model outputs and conversations are released as JSON files on Google Drive (Appendix E.3). The benchmark is installable via pip."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 explicitly states: 'we additionally include a requirements.txt file that can be used to install the exact dependencies we used for the experimental results in the paper.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states the GitHub README provides 'extensive documentation on how to use our framework (including how to run the existing benchmark, create new tools, task, etc.)' and includes Jupyter Notebooks to reproduce all figures and tables."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for all main results using statsmodels.stats.proportion.proportion_confint. Tables 3, 4, and 5 all include CI values. The NeurIPS checklist confirms this explicitly."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper that reports performance metrics across systems. No formal claims of statistical superiority are made that would require significance tests. The paper reports CIs but does not claim one model is statistically significantly better than another."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute and relative differences are reported throughout. Table 2 shows percentage changes from baseline (e.g., '-22.6%'). Table 5 shows targeted ASR dropping from 57.69% to 6.84% with tool filter. These provide sufficient context for assessing practical magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 97 user tasks and 629 security test cases, but no power analysis or explicit justification for why these numbers are sufficient is provided. The tasks were designed for scenario coverage, not statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for all major results in Tables 3, 4, and 5, providing spread information. This is a deterministic benchmark (no random seeds involved in the benchmark itself), and CIs are computed from the binomial proportion, which is appropriate."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple attack baselines are compared: 'Ignore previous instructions', 'TODO', InjecAgent, and 'Important message' (Section 4.2, Figure 8, Table 4). Multiple defense baselines are compared (Section 4.3, Figure 9, Table 5). Multiple LLMs serve as agent baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods from 2023-2024: InjecAgent (2024), Spotlighting (2024), ProtectAI (2024). LLM evaluations include Claude 3.5 Sonnet, GPT-4o, Gemini 1.5 Pro — all state-of-the-art at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 presents ablations on attacker knowledge (Table 2) showing the impact of knowing user name and model name on attack success rate. Section 4.3 ablates multiple defense strategies independently. These are controlled single-variable manipulations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are explicitly defined in Section 3.4: Benign Utility, Utility Under Attack, and Targeted Attack Success Rate (ASR). Results are reported across all three throughout the paper."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses deterministic utility functions to evaluate agent task completion. Human evaluation is not relevant here — in fact, the paper explicitly argues against LLM-based evaluation because the injection might fool the evaluator too. Deterministic checks are more appropriate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This paper introduces a benchmark framework and evaluates pre-trained LLMs via API. No model training or tuning is performed on any portion of the data. There is no dev/test split because there is no development phase using the benchmark data. The concept of a held-out test set is structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task suite (Workspace, Slack, Travel, Banking) in Figure 7 showing attack success rates per injection task and suite. Figure 20 shows utility by suite for different attacks. Figure 6 breaks down by model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Specific failure cases are discussed: travel task 6 succeeds in 0% of cases because the injection requires two unrelated actions. The Slack suite's 92% success rate is analyzed. Defense failure modes are discussed in Section 4.3 (tool filter fails when planning cannot be done in advance)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: incorrect attacker knowledge weakens attacks significantly (-22.6%, Table 2). Prompt injection detector has too many false positives and degrades utility (Section 4.3). Denial-of-service attacks are less effective than targeted attacks (Figure 20, Appendix D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: '97 realistic tasks' and '629 security test cases' confirmed in Table 1. 'LLMs fail at many tasks' confirmed by benign utility scores in Table 3. 'Attacks break some security properties but not all' confirmed by per-task ASR breakdowns in Figure 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about the effect of attacker knowledge (Table 2) and defense mechanisms (Table 5) are made via controlled single-variable manipulations. The ablation design is adequate: one variable is changed at a time while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported for specific models on the AgentDojo benchmark. The paper explicitly notes 'settings which AgentDojo does not cover yet' (Section 4.3, Section 5). Claims are bounded to the tested environments and attacks. The title accurately names the specific framework rather than claiming general results."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Alternative explanations are discussed: the inverse scaling finding is explained by 'models with low utility often fail at correctly executing the attacker's goal, even when the prompt injection succeeds.' The high Slack ASR is attributed to the attacker controlling a significant fraction of tool output (Figure 21b). Tool filter failures are explained by task structure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to most models by marketing names only: 'Claude 3 Sonnet', 'GPT-4o', 'Gemini 1.5 Pro', etc. Only one code figure (Figure 12) shows 'gpt-4o-2024-05-13'. The schema explicitly states 'marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count as specified versions.' Nine out of ten models lack snapshot dates or API version identifiers in the paper text."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B: the default system prompt (Figure 14), additional Claude Sonnet prompt (Figure 15), Llama 3 70B prompt (Figure 16), defense prompts (Figures 17-18), and all four attack prompts (Figure 19) are reproduced verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, max tokens, or other sampling hyperparameters for any of the LLM API calls. Section 4 states models are queried 'following the respective documentation' but specific settings are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Section 3.2 explains the pipeline interface, Figure 11 shows the base component, Figure 12 shows the pipeline combining LLM with injection detector, and Section 3.3 explains how attacks integrate with the scaffold. Code examples are provided."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how environment state data was generated: 'manually or assisted by GPT-4o and Claude 3 Opus, by providing the models with the expected schema of the data and a few examples. For LLM-generated test data we manually inspected all outputs to ensure high quality.' The cross-product method for generating security test cases is explained in Section 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Conclusion) contains substantive discussion of limitations with five explicitly labeled avenues for improvement. Section 4.3 discusses specific failure modes of defenses. This goes beyond a single sentence."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: tool filter defense fails when task planning cannot be done before observing untrusted data; defense fails when needed tools also suffice for attacks (17% of test cases); multi-session attack scenarios not yet covered; limitations of manual task specification for scaling. These are specific to THIS study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'settings which AgentDojo does not cover yet' including multi-session scenarios (Section 4.3). It notes attacks are 'general-purpose and not designed specifically for any given tasks or security scenarios' (Section 1). It acknowledges tasks are synthetically generated, not from real deployments."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All model outputs and conversations are released as JSON files on Google Drive (Appendix E.3). The benchmark code and environment state data are on GitHub under MIT license. Zenodo DOI 10.5281/zenodo.12528188 provides archival access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Task creation process is described in Section 3.1: tasks were manually designed to cover diverse scenarios, dummy data was generated manually or via GPT-4o/Claude 3 Opus with manual inspection. The four environments and their contents are described in detail in Table 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or crowd-sourced contributors are involved. The benchmark uses synthetically generated data and tasks designed by the paper authors. The data card confirms 'No real data. Only dummy data.' The NeurIPS checklist marks human subjects items as N/A."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task design to evaluation is documented: task design process (Section 3.1), cross-product combination to obtain security test cases (Section 3), environment state management (Section 3.1), and deterministic utility function evaluation (Section 3.1, with code examples in Figures 4-5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'E.D. is supported by armasuisse Science and Technology. J.Z. is funded by the Swiss National Science Foundation (SNSF) project grant 214838.' The data card (Appendix F.2.3) adds: 'No institution provided explicit funding for the creation of this benchmark.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed on the title page: ETH Zurich (all authors) and Invariant Labs (Balunovic, Beurer-Kellner, Fischer). The data card (Appendix F.2.2) repeats these affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors (Balunovic, Beurer-Kellner, Fischer) are affiliated with Invariant Labs, a company building AI security tools including prompt injection defenses. They evaluate defense concepts that are closely related to Invariant Labs' commercial interests. While not direct funders, the commercial affiliation creates a non-independent interest in the outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper. Three authors are affiliated with Invariant Labs (a commercial AI security company), but this financial interest is not explicitly declared beyond the affiliation listing. The absence of a competing interests declaration is NO per the schema."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "AgentDojo tests prompt injection attack/defense robustness in a dynamic tool-calling environment, not model knowledge on a static benchmark. The schema states applies=false for 'studies that test defenses/tools rather than model knowledge.' The tasks require multi-step stateful tool interaction where contamination is not a meaningful concern — knowing the task description would not help a model execute the correct tool calls in the correct order on the specific environment state."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated. This is a security benchmark testing defenses and attack robustness, not model knowledge. Train/test overlap is structurally irrelevant for evaluating whether a model follows injected instructions versus resisting them."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as the other contamination items. The benchmark was newly created in 2024 and tests dynamic security properties in stateful environments. Even theoretical contamination would not meaningfully affect the measured outcomes (attack success rates, defense effectiveness)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The NeurIPS checklist marks human subjects items as N/A."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Only synthetically generated data and automated LLM evaluations."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D explicitly states: 'We estimate that running the full suite of 629 security test cases on GPT-4o costs around US$35, and running the suite of 97 utility test cases costs US$4.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total API costs are reported ($35 for the full security test suite on GPT-4o, $4 for utility cases). No GPU compute is required as only API-accessed models are evaluated (except Llama 3 70B, but no GPU details are given for it)."
       }
     }
   }
 }