ai-research-survey

calibration.json (20041B)

---

 {
   "calibration": {
     "paper_slug": "agentvigil-generic-blackbox-2025",
     "date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "model": "opus",
     "agreement_rate": 0.98,
     "total_questions": 50,
     "agreements": 49,
     "disagreements": 1,
     "disagreement_details": [
       {
         "category": "statistical_methodology",
         "question": "effect_sizes_reported",
         "sonnet": {"applies": true, "answer": false},
         "opus": {"applies": true, "answer": true},
         "direction": "opus_generous",
         "opus_justification": "The paper reports '71% success rate' vs '38% baseline' and states 'nearly a 100% improvement over the baseline attacks' (Section 5.1). This provides percentage improvement with baseline context, which the schema description explicitly lists as sufficient for YES: 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper matches this pattern across both benchmarks.",
         "sonnet_justification": "While the paper reports raw performance numbers, it does not report formal effect sizes (Cohen's d, odds ratios, etc.). The claim of 'nearly doubling performance' is contextually described but not accompanied by a formal effect size measure.",
         "resolution_note": "Borderline case. The schema description lists 'percentage improvement with baseline context' as an acceptable form of effect size alongside Cohen's d and odds ratios. The paper provides relative improvement ('nearly 100% improvement') with explicit baseline and result numbers. Sonnet applies a stricter interpretation requiring formal statistical effect sizes. Opus reads the schema's YES example literally. Reasonable people could disagree."
       }
     ],
     "summary": "Very high agreement (98%) between Sonnet and Opus on this paper. The single disagreement is on effect_sizes_reported, where Opus reads the schema's example of 'percentage improvement with baseline context' as matching the paper's reporting pattern, while Sonnet applies a stricter standard requiring formal effect size measures like Cohen's d. Both evaluations otherwise align on all 49 remaining questions, including the large number of false answers for statistical methodology, the strong evaluation design scores, and the complete absence of limitations discussion.",
     "opus_checklist": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. Algorithms are described in pseudocode (Appendix C) but no implementation is released."
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper evaluates on two public benchmarks: AgentDojo (Debenedetti et al., 2024) and VWA-adv (Wu et al., 2024b), both publicly available. The real-world case study uses open-source WebArena/magento2. No novel dataset was created that requires separate release."
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, conda environment, or dependency specification is provided. Appendix A lists model checkpoints but not the software environment needed to run experiments."
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions, README, or reproduction scripts are provided. Algorithms 1-3 in Appendix C describe logic in pseudocode but are not runnable instructions."
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 1-5 and Figures 3-4 are reported as point estimates (e.g., '71% success rate'). No confidence intervals, error bars, or ± notation appears anywhere in the paper."
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used. Comparative claims ('nearly doubling performance', 'outperforms the baseline') are made by directly comparing point-estimate success rates without p-values, t-tests, or any formal testing."
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports percentage improvement with baseline context: '71% success rate' vs '38% baseline' with 'nearly a 100% improvement over the baseline attacks' (Section 5.1). Similar reporting for VWA-adv (70% vs 36%). Per the schema description, 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper matches this pattern."
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses 142/173 task splits for AgentDojo and 99/100 for VWA-adv. The splits are described as 'randomly dividing' but no justification for these sizes or power analysis is provided."
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results are single-run point estimates. No standard deviation, variance, IQR, or any spread measure is reported across experimental runs. Given the stochastic nature of fuzzing (random seed sampling, random mutation selection), variance across runs could be substantial."
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Handcrafted adversarial prompts from AgentDojo and VWA-adv serve as baselines (Sections 5.1, 5.2). Additional baselines from OpenPromptInjection and InjecAgent are included in Appendix B.2 (Table 4)."
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines are from 2024: AgentDojo (2024), VWA-adv (2024), OpenPromptInjection (2024), InjecAgent (2024). These represent the current state of the art in indirect prompt injection benchmarking."
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 presents an ablation study isolating contributions of three components: (1) initial corpus, (2) adaptive seed scoring, and (3) MCTS-based seed selection. Results shown in Figure 3."
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports attack success rate (ASR) as the primary metric and coverage as an additional metric (Figures 3 and 4). Utility scores for open-source models are also reported in Appendix B.1."
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "This is an adversarial security paper evaluating automated attack success rates on benchmarks with programmatic success criteria. Human evaluation of outputs is not relevant to the core claims about attack effectiveness."
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly separates tasks into fuzzing sets (used for optimization) and test sets (held-out for evaluation): 142/173 for AgentDojo and 99/100 for VWA-adv. Transferability is evaluated on the test set."
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B.3, Table 5 provides per-scenario breakdowns: AgentDojo by suite (Slack, Workspace, Travel, Banking) and VWA-adv by attack goal (Illusioning, Goal misdirection). Per-model results appear throughout Table 1."
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper discusses failure against Claude-3.5-Sonnet (Section 5.1: 'both the baseline and AGENTVIGIL's prompts are ineffective against Claude-3.5-Sonnet'). Performance degradation against defenses is discussed in Sections 5.1 and 5.2."
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that AGENTVIGIL's optimized prompts do not transfer to Claude-3.5-Sonnet (0.03-0.04 ASR, worse than baseline's 0.08-0.12). It also reports that in VWA-adv with defenses, 'AGENTVIGIL's performance declines and converges with the baseline' (Section 5.2)."
         }
       },
       "claims_and_evidence": {
         "abstract_claims_supported": {
           "applies": true,
           "answer": true,
           "justification": "Abstract claims of 71% and 70% success rates are supported by Tables 1-3. 'Nearly doubling' the baseline (38% to 71%, 36% to 70%) is accurate. Transferability claims are supported by test set results in Table 1. 'Promising results against defenses' is a hedged claim supported by Tables 2-3."
         },
         "causal_claims_justified": {
           "applies": true,
           "answer": true,
           "justification": "The ablation study (Section 5.3) uses controlled single-variable manipulation to support causal claims about which components contribute to performance. Replacing the initial corpus while keeping other components constant, and removing MCTS/scoring while keeping other components, constitutes adequate ablation design."
         },
         "generalization_bounded": {
           "applies": true,
           "answer": false,
           "justification": "The title claims 'Generic Black-Box Red-teaming' and the abstract claims applicability to 'diverse LLM agents', but testing is limited to two benchmarks (personal assistants in AgentDojo, web agents in VWA-adv) plus one real-world case study. Only English-language text-based injection is tested. Only specific types of injection goals are covered. The broad claims outstrip the tested scope."
         },
         "alternative_explanations_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No alternative explanations for results are discussed. The speculation about Claude's resistance ('Claude is more vulnerable to simpler adversarial prompts') is presented without evidence or alternative considerations. No threats-to-validity section addresses confounds or alternative interpretations."
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A explicitly lists all model checkpoints with version IDs: o3-mini (o3-mini-2024-12-17), GPT-4o-mini (gpt-4o-mini-2024-07-18), GPT-4o (gpt-4o-2024-08-06), Claude-3.5-Sonnet (claude-3-5-sonnet-20241022), Gemini-2-flash-exp (gemini-2.0-flash-exp)."
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "The adversarial prompt templates use 'placeholders to accommodate different variables' (Section 4.2) but the actual template text is not provided. Mutation methods (Shorten, Expand, Rephrase, Crossover, GenerateSimilar) are described in natural language only (Section 4.3) without the actual prompt text sent to the helper LLM."
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "LLM inference hyperparameters (temperature, top-p, max tokens) are not reported for any model. Fuzzing parameters (10 iterations, 3 or 10 mutations per iteration, top-5 seeds for transferability) are stated, but LLM API parameters are absent."
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "AGENTVIGIL attacks third-party agents (AgentDojo's agent, VWA-adv's agent, WebArena's default agent) that are black-box systems. The authors cannot describe the internal scaffolding of these external agents. AGENTVIGIL itself is an attack framework, not an agentic scaffold."
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "The initial corpus collection is described qualitatively as being 'from human heuristics, online resources, existing prompt injection research' (Section 4.2) with two citations. The number of initial templates, specific filtering criteria, and exact sources are not documented in sufficient detail to reproduce the corpus."
         }
       },
       "limitations_and_scope": {
         "limitations_section_present": {
           "applies": true,
           "answer": false,
           "justification": "There is an 'Impact Statement' section before References that discusses dual-use concerns, but no dedicated Limitations or Threats to Validity section exists. The Impact Statement does not discuss methodological limitations of the evaluation."
         },
         "threats_to_validity_specific": {
           "applies": true,
           "answer": false,
           "justification": "No threats-to-validity section exists. The Impact Statement contains generic statements about ongoing research but does not address specific threats such as benchmark selection bias, limited number of injection task types, stochastic evaluation via random task sampling, or model version sensitivity."
         },
         "scope_boundaries_stated": {
           "applies": true,
           "answer": false,
           "justification": "The threat model (Section 3) defines attacker capabilities and exclusions, but this is not the same as bounding the results. The paper does not explicitly state what the results do NOT show, such as whether the approach would generalize to non-English agents, non-text modalities, or agent architectures beyond the two tested benchmarks."
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Generated adversarial prompts and raw per-task attack success logs are not released. Only aggregate success rates appear in tables. Independent verification of underlying results is not possible."
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The primary data comes from two public benchmarks. Section 4.2 describes corpus collection from 'human heuristics, online resources, existing prompt injection research' with specific citations. Benchmark task splits are described with counts (142/173 for AgentDojo, 99/100 for VWA-adv). Random division procedure is stated."
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. All evaluation uses automated benchmarks with pre-defined tasks."
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The fuzzing pipeline is documented through Algorithms 1-3 (Appendix C) and Figure 2. The flow from corpus collection to seed selection to mutation to scoring to iteration is clearly described with pseudocode."
         }
       },
       "conflicts_of_interest": {
         "funding_disclosed": {
           "applies": true,
           "answer": false,
           "justification": "No funding source, acknowledgments section, or grant information appears in the paper. As a preprint from three universities (UC Berkeley, Washington University, UC Santa Barbara), research funding would be expected but is not disclosed."
         },
         "affiliations_disclosed": {
           "applies": true,
           "answer": true,
           "justification": "Author affiliations are clearly listed on the first page: UC Berkeley (1), Washington University, Saint Louis (2), and UC Santa Barbara (3), with each author's affiliation number indicated."
         },
         "funder_independent_of_outcome": {
           "applies": true,
           "answer": false,
           "justification": "No funding is disclosed, making it impossible to assess funder independence. Per schema conventions, absence of disclosure is treated as NO."
         },
         "financial_interests_declared": {
           "applies": true,
           "answer": false,
           "justification": "No competing interests statement, patent disclosure, or financial interests declaration appears in the paper."
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies model checkpoints (Appendix A) but does not state training data cutoff dates for any of the LLMs used (o3-mini, GPT-4o, Claude-3.5-Sonnet, Gemini-2-flash-exp). This matters because benchmark tasks could appear in training data."
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether AgentDojo or VWA-adv benchmark tasks appeared in the training data of the evaluated LLMs. Both benchmarks are from 2024 and some models' training data may include descriptions of these benchmarks."
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "AgentDojo (arXiv:2406.13352, June 2024) and VWA-adv (2024) were published before some model checkpoints. GPT-4o (gpt-4o-2024-08-06) may have trained on data including these benchmark descriptions. This contamination risk is not discussed."
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. All evaluation is automated using benchmark frameworks."
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The paper discusses ethical considerations for the real-world case study (using a local copy) but this is not an IRB-reviewed human subjects study."
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants."
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants."
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants."
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants."
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants."
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Extensive LLM API usage (GPT-4o, o3-mini, GPT-4o-mini for helper, multiple models for evaluation, across hundreds of tasks and 10 fuzzing iterations) but no API costs, token counts, or latency figures are reported. The paper mentions excluding some defenses due to 'high computational costs' without quantifying its own costs."
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget is stated. The paper describes 10 fuzzing iterations with 3 or 10 mutations per iteration across 142+ tasks, implying substantial API usage, but no total cost, GPU hours, or compute budget is reported."
         }
       }
     }
   }
 }