ai-research-survey

calibration.json (18367B)

---

 {
   "calibration_metadata": {
     "paper_slug": "agenttypo-adaptive-typographic-2025",
     "calibration_date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "calibrator_model": "opus",
     "agreement_rate": 0.96,
     "total_questions": 50,
     "agreements": 48,
     "disagreements": 2
   },
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "held_out_test_set",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet says 'there is no indication that tasks were used for tuning — hyperparameter selection appears to be done separately (Section VI.G explores lambda and step budget).' However, Section VI.G explicitly tunes hyperparameters (lambda, step budget, number of retrieved examples) by measuring ASR on the same 77 VWA-Adv tasks shown in Figures 6, 7, and 8. There is no mention of a separate dev split for tuning vs. a held-out split for final evaluation. The hyperparameters were selected based on performance on the same data that the main results are reported on. Per the schema: 'If unclear whether the reported numbers are on data used for any selection decisions, NO.'"
     },
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet sets applies=false, reasoning that no funder is identifiable so NA applies. Opus sets applies=true because the paper does not explicitly state it is unfunded — the schema says 'NA if unfunded' but absence of funding disclosure is not the same as confirmed absence of funding. Since the paper could have disclosed funding (or its absence) and did not, the criterion is applicable and the answer is false. However, this is a genuine applies-boundary case since the schema does provide NA for unfunded work."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub repository URL, Zenodo archive, or any code link is provided anywhere in the paper. The ATPI algorithm, optuna-based TPE implementation, and multi-LLM scaffolding are described abstractly but no code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available VWA-Adv benchmark (referenced with links to VisualWebArena). No proprietary data was collected — all experiments use this existing public benchmark of 77 tasks across three domains."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only 'eight NVIDIA RTX 3090 GPUs' and the 'optuna' library are mentioned. No requirements.txt, Dockerfile, conda environment, or detailed list of library versions is provided. This is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 provides pseudocode for ATPI, but no step-by-step reproduction instructions, README, or runnable scripts are included. A competent researcher could not reproduce results without substantial implementation effort and guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III and IV are point estimates (e.g., '0.45', '0.68'). No confidence intervals, error bars, or uncertainty measures are reported despite three trials per task being run."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'AgentTypo significantly outperforms' baselines repeatedly but uses no statistical significance tests — no p-values, t-tests, bootstrap tests, or any other formal test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw ASR values are reported (e.g., 0.23 vs. 0.45). While both baseline and improved numbers are given, no standardized effect sizes (Cohen's d, odds ratios, relative risk) are provided. The schema asks for more than raw differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The VWA-Adv benchmark has 77 tasks. No justification is given for whether this sample size provides adequate statistical power for the comparative claims. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section VI.A states 'each prompt is executed with three independent trials' but only aggregate ASR is reported. No standard deviation, variance, interquartile range, or any spread measure across trials is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: Raw Prompt Injection, InjecAgent, AdvAgent, and AgentAttack, covering both text-based and image-based attack methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include AgentAttack (ICLR 2025), AdvAgent (ICML 2025), and InjecAgent (ACL 2024), all contemporary to the October 2025 submission date and representing current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.E (Table IV) presents ablation across four configurations: ATPI alone, Strategy Library alone, Strategy+RAG, and full AgentTypo-pro, clearly isolating each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Attack Success Rate (ASR) is used as the evaluation metric in the main results tables. LPIPS stealthiness is shown only in the hyperparameter sweep (Figure 6), not as a standard comparison metric across all methods. No secondary metrics like query cost, latency, or defense evasion rate are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Manual human review is mentioned only to validate the LLM-based scorer's accuracy, not as a systematic evaluation of attack outputs, agent behavior, or visual stealthiness. No human study is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Section VI.G tunes hyperparameters (lambda, step budget, number of retrieved examples) by measuring ASR on the same 77 VWA-Adv tasks (Figures 6, 7, 8). No separate dev/test split is mentioned. The reported numbers are on data used for hyperparameter selection decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table III provides per-domain breakdowns (Classifieds, Shopping, Reddit) and per-model breakdowns (GPT-4V, GPT-4o, GPT-4o-mini, Gemini-1.5-Pro, Claude-3-Opus) for both image+text and image-only settings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII discusses the stealth-effectiveness trade-off limitation. Section VI.D discusses why AgentAttack fails on complex tasks (0% ASR on email). The paper acknowledges where its method requires conspicuous text."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table IV shows Strategy Library alone yields lower ASR than the full system. Section VI.G shows ASR degrades when retrieving more than 5 examples. These are genuine negative findings about component configurations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'raises the success rate from 23% to 45%' on GPT-4o image-only, and '68% ASR in image+text settings.' Table III confirms both numbers for GPT-4o. Quantitative claims in the abstract match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims through ablation (e.g., 'Incorporating the Strategy Library shows a substantial improvement in ASR'). While the ablation design controls single variables, the absence of statistical tests over only 77 tasks means observed differences may not be distinguishable from noise. The causal claims are not adequately supported."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim AgentTypo 'poses a practical and potent threat to multimodal agents' without bounding to the tested setting of three simulated websites from VWA-Adv. The title says 'Black-box Multimodal Agents' generically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the observed improvements. The paper does not discuss whether improvements over AgentAttack could be due to specific task characteristics, benchmark artifacts, or other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are cited as 'GPT-4V', 'GPT-4o', 'GPT-4o-mini', 'Gemini-1.5-Pro', and 'Claude-3-Opus' without snapshot dates or API version strings. The attacker/scorer/summarizer LLMs are described only as 'GPT-4' without version. Per the schema, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "System prompts for the Attacker LLM, Scorer LLM, and Summarizer LLM are described in natural language only (e.g., 'formulates a hijacking prompt according to a specified adversarial goal'). Table V shows example output prompts, not the system prompts used to generate them. The actual prompt text sent to these LLMs is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.C reports: stealthiness weight lambda=10.0, maximum 20 optimization steps, top-k=5 retrieval examples, success threshold=0.8. Table I lists ATPI parameter search ranges (font size 10-150, color 0-255, position, etc.)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes AgentTypo-pro's multi-LLM scaffolding in detail: Attacker LLM (Eq. 7), Scorer LLM (Eq. 8), RAG module (Eq. 9), Summarizer LLM (Eq. 10), with Figure 4 showing the full pipeline, interaction logic, and feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A and VI.A describe the data setup: JavaScript annotation of interactable elements, SoM parsing, captioning by the same LVLM. The VWA-Adv benchmark's 77 tasks across three websites are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII 'Limitations' is a dedicated section discussing the stealth-effectiveness trade-off and restricted benchmark scope (three websites)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The two limitations discussed (stealth trade-off and limited benchmark) are partially specific to this study, but the section does not address key validity threats: the 77-task sample size, the reliability and potential bias of the LLM-based scorer, hyperparameter tuning on the test set, or whether results depend on specific VWA-Adv task characteristics."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The benchmark limitation is acknowledged but no specific claims are retracted or bounded. No statement like 'these results do not imply effectiveness against text-only agents or real-world web deployments.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (individual task outcomes, LLM scorer responses, agent action logs, generated adversarial images) are not released. Only aggregate ASR tables are published."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The VWA-Adv benchmark is a described public dataset. Section VI.A documents its composition: 77 adversarial tasks across Classifieds, Shopping, and Reddit domains, built on VisualWebArena."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study is a purely automated benchmark evaluation with LLM-based agents. Recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: task selection from VWA-Adv → ATPI image modification (Algorithm 1) → agent execution with 3 trials → LLM-based scoring → ASR calculation. Sections IV-VI describe each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section and no mention of funding sources anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors (Yanjie Li, Yiming Cao, Dong Wang, Bin Xiao) are identified as affiliated with the Computing Department, Hong Kong Polytechnic University, listed in the author footnote on page 1."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. The paper does not state it is unfunded, so we cannot confirm the 'NA if unfunded' exception applies. Absence of funding disclosure means independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present anywhere in the paper. Per the schema, absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4V, GPT-4o, GPT-4o-mini, Gemini-1.5-Pro, and Claude-3-Opus as target agents but does not state training data cutoff dates for any model. This matters for assessing whether models could have seen VWA-Adv/VisualWebArena tasks during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether VWA-Adv benchmark tasks or VisualWebArena data could appear in the training data of the evaluated LVLMs. VisualWebArena was published in 2024 and models trained after that could have seen it."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VWA-Adv is built on VisualWebArena (2024). Several evaluated models may have training data including this benchmark. The paper does not discuss contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a purely automated benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Randomization of participant assignment is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper calls GPT-4o, GPT-4V, and other commercial APIs extensively (3 trials × 77 tasks × up to 20 optimization steps per task × multiple models) but provides no cost estimates, token counts, or API expenditure figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only 'eight NVIDIA RTX 3090 GPUs' is mentioned for running local models (ATPI captioners). Total wall-clock time, GPU hours, or API budget for the full experimental suite are not reported."
       }
     }
   }
 }