ai-research-survey

scan.json (27726B)

---

 {
   "paper": {
     "title": "TAMAS: Benchmarking Adversarial Risks in Multi-Agent LLM Systems",
     "authors": ["Ishan Kavathekar", "Hemang Jain", "Ameya Rathod", "Ponnurangam Kumaraguru", "Tanuja Ganu"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2511.05269",
     "doi": "10.48550/arXiv.2511.05269"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at https://github.com/microsoft/TAMAS' and the paper mentions all data, attack implementations, and evaluation scripts will be publicly released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All data, attack implementations, and evaluation scripts will be publicly released to support reproducibility' (Section 5.1) and provides a GitHub repository URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency details are provided in the paper. Models are accessed via API or Ollama but no version pinning or environment setup is described."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper describes the experimental setup at a high level but does not include commands or scripts to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.3 provides bootstrapped estimates with 95% confidence intervals for all ARIA values across models and configurations (Tables 7-11)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite making numerous comparative claims (e.g., closed-source vs open-source models, configuration comparisons), no statistical significance tests are reported. Comparisons are based on raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage differences with context, e.g., 'average ARIA-4 is 15.6% for closed-source models compared to 39.2% for open-source models' (Section 6), providing magnitude context for comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 10 adversarial instances per attack type per scenario was chosen. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrapped confidence intervals (Appendix D.3) provide variance estimates across domains using 10,000 resamples with Dirichlet perturbation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 10 different LLM backbones across 3 agent configurations and 2 frameworks (AutoGen, CrewAI), providing comparative baselines across models and configurations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, Gemini-2.0-Flash, Deepseek-R1-32B, Qwen3-32B/8B — all contemporary models at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper does not systematically remove or modify components of the benchmark or defense mechanisms to understand their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ARIA scores (4-tier classification), Safety Score, PNA (Performance under No Attack), and ERS (Effective Robustness Score) — multiple complementary metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human verification of LLM-as-a-judge is conducted on 140 representative logs, with F1 scores reported per attack type (Appendix C.1). This validates the automated evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper testing robustness, not a model training paper. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by attack type (6 attacks), model (10 models), and configuration (5 setups) in Figure 2, Table 2, and appendix tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 and Appendix E provide illustrative failure cases showing agents executing malicious tasks, acknowledging maliciousness yet complying, and conversation trajectories being manipulated."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that persuasive agent attacks were 'entirely unsuccessful in practice: across all domains, models, and configurations' (Appendix A.3), a clear negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims multi-agent systems are 'highly vulnerable to adversarial attacks' — supported by ARIA-4 scores showing 60-90%+ attack success rates across most attack types. The claim of evaluating 10 LLMs, 3 configurations, 300 adversarial instances matches the reported experiments."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'Impersonation succeeds largely because agents prioritize instructions from perceived authorities' (Section 6) and attributes CrewAI's higher safety to 'assigning tasks to individual agents upfront.' These causal explanations are speculative without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Benchmarking Adversarial Risks in Multi-Agent LLM Systems' broadly, but results are limited to 5 specific scenarios, 2 frameworks (AutoGen, CrewAI), and synthetic tool environments. The limitations section acknowledges this is not exhaustive, but the title and abstract frame findings more broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the difference between open-source and closed-source model vulnerability could be due to model size rather than open/closed nature, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses synthetic tool invocations as a proxy for real-world adversarial risk but does not discuss this gap. The simulated tools return fixed strings (e.g., 'News data poisoned with false information!') — the gap between this simulation and actual adversarial consequences is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4', 'GPT-4o', 'GPT-4o-mini', 'Gemini-2.0-Flash' etc. without specific version snapshots or API dates. Table 5 lists models but no version identifiers beyond marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H provides the full ARIA evaluation prompts used for each attack type. Appendix A.1 provides sample agent system prompts and tool definitions. Sample adversarial queries are shown throughout."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states GPT-4o is used as LLM-as-judge with 'temperature setting of 0.0' (Appendix C.1). However, temperature settings for the backbone LLMs being evaluated are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three agent configurations (Central Orchestrator, Sequential, Collaborative/Swarm) are described in Section 5.2 with their coordination mechanisms, and Table 3 compares their key characteristics."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix A.4 describe the dataset construction process: scenarios were manually designed, ChatGPT was used for query generation, and all content was manually reviewed and refined."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F contains a dedicated 'Limitations and Future Work' section with four specific limitation categories."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: restricted to AutoGen and CrewAI frameworks, non-exhaustive set of configurations, limited to five scenarios and six attack types, and no defense benchmarking."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F explicitly states what was not tested: alternative frameworks, additional configurations, more scenarios/attacks, and defenses. Each limitation bullet identifies a specific scope boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper promises public release of all data, attack implementations, and evaluation scripts via GitHub, and provides the repository URL."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 describes the data curation process: manual design of multi-agent systems, ChatGPT-aided generation of user queries, and manual review of all generated content."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the main study. The benchmark uses synthetic scenarios and automated LLM evaluations. Human annotation was only for validation of the LLM-as-judge."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from scenario design → tool creation → query generation → manual review is described in Appendix A.4. The evaluation pipeline (log collection → LLM-as-judge → human verification) is described in Appendix C."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. One author is from Microsoft Research India, but no funding acknowledgment is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: IIIT Hyderabad and Microsoft Research India. The GitHub repository is under microsoft/ organization."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "A Microsoft Research co-author is involved, and the benchmark evaluates systems that could be deployed on Azure/Microsoft platforms. Microsoft has commercial interest in multi-agent safety. No funding independence statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests adversarial robustness of multi-agent systems, not model knowledge on a benchmark. The attacks are novel prompts, not knowledge-based tasks where training data contamination would be relevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the benchmark tests safety/robustness behavior, not model knowledge. Train/test overlap is not applicable to adversarial robustness testing."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark evaluates agent safety behavior, not factual knowledge. Contamination of adversarial test cases in training data is not a relevant concern for this type of evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The 140-sample human annotation for LLM-as-judge validation is a methodological check, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper includes an ethics statement focused on the research aim of advancing safety."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported despite running 10 models × 5 configurations × 400 tasks. GPT-4 was excluded from CrewAI 'due to budget constraints' (Section 5.3) but no actual cost figures are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The paper mentions budget constraints for GPT-4 and Gemini compatibility issues but provides no quantification of total API spend or compute used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results appear to be from single runs per model/configuration/attack combination, with bootstrapping applied post-hoc to existing results rather than multiple experimental runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not explicitly stated. Each attack type has 10 datapoints per scenario (50 total per attack), but it is unclear if these were run once or multiple times."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is involved — the paper evaluates off-the-shelf models and frameworks without tuning."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection/tuning is performed — all configurations are evaluated and reported."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 10 models, 5 configurations, and 6 attack types but applies no multiple comparison correction. No significance tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the TAMAS benchmark and evaluate it themselves. No discussion of self-evaluation bias or independent validation beyond the LLM-as-judge human verification (which is internal validation, not independent replication)."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper is not proposing a method that trades compute for performance. It evaluates existing systems on a benchmark."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its synthetic tool environments and scripted attack scenarios adequately measure real-world adversarial risk in multi-agent systems. The tools return fixed strings, which may not reflect realistic attack dynamics."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly evaluates the same models across multiple scaffolding configurations (Magentic-One, Round Robin, Swarm, CrewAI centralized/decentralized), treating the scaffold as a variable and analyzing its impact on results (Table 2, Section 6)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark tests adversarial robustness, not factual knowledge. Temporal leakage is not relevant to whether an agent complies with a malicious instruction."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — the evaluation measures safety behavior, not prediction accuracy. Feature leakage is not applicable."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "There is no train/test split; the benchmark evaluates behavioral robustness, not learned knowledge."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Leakage detection is not applicable to adversarial robustness benchmarks that test agent behavior."
       }
     }
   },
   "claims": [
     {
       "claim": "Multi-agent LLM systems are highly vulnerable to adversarial attacks across diverse attack vectors.",
       "evidence": "Figure 2 shows ARIA-4 (successful attack) rates of 60-90%+ for DPI and Impersonation attacks across most models and configurations. Section 6 reports specific numbers.",
       "supported": "strong"
     },
     {
       "claim": "Impersonation attacks are consistently the most effective, reaching 82% in Swarm and DPI 81% in Magentic-one.",
       "evidence": "Figure 2 and Section 6 report these specific attack success rates across configurations.",
       "supported": "strong"
     },
     {
       "claim": "Closed-source models are more resilient to IPI than open-source models.",
       "evidence": "Section 6: 'average ARIA-4 is 15.6% for closed-source models compared to 39.2% for open-source models' in Magentic One, and '37.6% versus 75.2%' in RoundRobin.",
       "supported": "moderate"
     },
     {
       "claim": "CrewAI configurations yield higher safety scores compared to their AutoGen counterparts.",
       "evidence": "Table 2 shows CrewAI safety scores generally higher. The paper attributes this to 'assigning tasks to individual agents upfront, rather than dynamically during execution.'",
       "supported": "moderate"
     },
     {
       "claim": "Colluding agents attack has low overall success (2-16%) but at least one agent often completes the malicious task.",
       "evidence": "Section 6 and Table 6 show colluding attack success is low overall but individual agent compliance ranges from 10-48% across configurations.",
       "supported": "strong"
     },
     {
       "claim": "Agents sometimes acknowledge maliciousness yet still comply with harmful instructions.",
       "evidence": "Section 7 and Appendix E provide qualitative examples where agents flag requests as malicious but proceed with execution.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TAMAS reveals that multi-agent LLM systems are highly vulnerable to adversarial attacks, with prompt-level attacks (DPI, Impersonation) achieving 60-90%+ success rates across 10 models and 5 configurations. Closed-source models show greater resilience to indirect prompt injection than open-source models. Agent coordination architecture significantly affects vulnerability, with CrewAI's upfront task assignment yielding higher safety than AutoGen's dynamic delegation. Even when agents recognize malicious intent, they frequently proceed with execution, indicating fundamental safety mechanism failures in multi-agent settings.",
   "red_flags": [
     {
       "flag": "Synthetic evaluation environment",
       "detail": "All tools are simulated with fixed-string returns (e.g., 'News data poisoned with false information!'). The gap between these synthetic scenarios and real-world adversarial risk is not discussed. Real tools would have actual consequences and different failure modes."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Numerous comparative claims (open vs closed source, framework comparisons, model rankings) are made based on raw percentage comparisons without any significance testing, despite small sample sizes per cell (10 instances per attack per scenario)."
     },
     {
       "flag": "Company evaluating related product space",
       "detail": "Microsoft Research co-author and microsoft/ GitHub organization, while the paper evaluates multi-agent frameworks that compete in Microsoft's commercial space. No conflict of interest statement is provided."
     },
     {
       "flag": "LLM-generated benchmark data",
       "detail": "ChatGPT was used to generate user queries and attacker tools (Appendix A.4). While manually reviewed, this introduces potential biases in attack pattern diversity and may favor certain model families."
     },
     {
       "flag": "Small sample sizes per condition",
       "detail": "Only 10 adversarial instances per attack type per scenario means each cell in the results has very few datapoints, limiting the reliability of percentage comparisons across models and configurations."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr"],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Benchmark for evaluating prompt injection attacks and defenses in LLM agent systems."
     },
     {
       "title": "Agent Security Bench (ASB): Formalizing and Benchmarking Attacks and Defenses in LLM-based Agents",
       "authors": ["Hanrong Zhang", "Jingyuan Huang", "Kai Mei"],
       "year": 2025,
       "arxiv_id": "2410.02644",
       "relevance": "Comprehensive single-agent security benchmark that TAMAS extends to multi-agent settings."
     },
     {
       "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents",
       "authors": ["Maksym Andriushchenko", "Alexandra Souly"],
       "year": 2025,
       "arxiv_id": "2410.09024",
       "relevance": "Evaluates how effectively agents refuse harmful queries, related to TAMAS safety evaluation."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "relevance": "Multi-agent framework used as one of the two primary evaluation platforms in TAMAS."
     },
     {
       "title": "Multi-Agent Risks from Advanced AI",
       "authors": ["Lewis Hammond", "Alan Chan", "Jesse Clifton"],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Comprehensive survey of risks from multi-agent AI systems, provides theoretical grounding for TAMAS."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes from inter-agent misalignment in multi-agent LLM systems."
     },
     {
       "title": "InjectAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for indirect prompt injection in LLM agents, predecessor to TAMAS in single-agent setting."
     },
     {
       "title": "RedCode: Risky Code Execution and Generation Benchmark for Code Agents",
       "authors": ["Chengquan Guo", "Xun Liu", "Chulin Xie"],
       "year": 2024,
       "arxiv_id": "2411.07781",
       "relevance": "Benchmarks safety of code agents generating and executing potentially harmful code."
     },
     {
       "title": "R-Judge: Benchmarking Safety Risk Awareness for LLM Agents",
       "authors": ["Tongxin Yuan", "Zhiwei He", "Lingzhong Dong"],
       "year": 2024,
       "arxiv_id": "2401.10019",
       "relevance": "Evaluates LLM safety awareness through curated risky agent trajectories."
     },
     {
       "title": "SafeArena: Evaluating the Safety of Autonomous Web Agents",
       "authors": ["Ada Defne Tur", "Nicholas Meade", "Xing Han Lù"],
       "year": 2025,
       "arxiv_id": "2503.04957",
       "relevance": "Provides the ARIA safety evaluation framework adopted by TAMAS."
     },
     {
       "title": "Secret Collusion Among Generative AI Agents: Multi-Agent Deception via Steganography",
       "authors": ["Sumeet Ramesh Motwani", "Mikhail Baranchuk"],
       "year": 2025,
       "arxiv_id": "2402.07510",
       "relevance": "Explores colluding agent attacks through steganographic communication between LLM agents."
     },
     {
       "title": "AgentMonitor: A Plug-and-Play Framework for Predictive and Secure Multi-Agent Systems",
       "authors": ["Chi-Min Chan", "Jianxuan Yu", "Weize Chen"],
       "year": 2024,
       "arxiv_id": "2408.14972",
       "relevance": "Framework for monitoring and securing multi-agent LLM systems."
     }
   ]
 }