ai-research-survey

scan.json (25969B)

---

 {
   "paper": {
     "title": "MaMa: A Game-Theoretic Approach for Designing Safe Agentic Systems",
     "authors": ["Jonathan Nöther", "Adish Singla", "Goran Radanovic"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.04431"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MaMa, a Stackelberg security game framework for automated design of safe agentic systems, produces systems significantly safer than manually designed alternatives and systems optimized only for quality, across four environments. Safety improvements transfer to stronger adversaries, different LLMs, targeted attacks, and indirect prompt injection attacks, while maintaining comparable task quality. Judge models validated with Cohen's Kappa (κ=0.88 safety, κ=0.80 quality) on 50 manually annotated episodes.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'we included the code for each final agentic system in the supplementary material' (Section 5.4) but provides no repository URL or persistent archive link. Supplementary material attached to a preprint is not the same as a released codebase."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the BAD-ACTS benchmark environments (Nöther et al., 2025) which appear to be publicly available, but the paper does not release the specific designed systems, attack archives, or episode logs generated during experiments."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 specifies compute infrastructure: 2x AMD EPYC 9555 CPUs, 24x 96GB memory, 2x Nvidia H200 141GB NVL GPUs. GPT-5.1 and GPT-5-mini accessed via OpenAI API. Open-source models (Llama-3.3:70b, Qwen3:32b) run locally. AutoGen used for execution."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Hyperparameters are listed in Table 7, but there are no scripts, commands, or README to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-4 report standard deviations in parentheses. Figure 2 shows shaded regions indicating standard deviation over three distinct evaluations."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MaMa systems are 'significantly safer' but uses no formal statistical significance tests. Comparisons are made by visually comparing means and standard deviations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute scores (1-5 scale) with baselines, allowing direct effect size computation. E.g., Travel Planning safety goes from 1.66 (attacked baseline) to 3.81 (MaMa), and ASR drops from 0.5646 to 0.1700 (Table 3)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for using 3 runs, 25 attacks per generation, 25 generations, or 50 episodes for judge validation. These numbers appear chosen pragmatically without power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported over three distinct runs in Tables 1-4 and as shaded regions in Figure 2."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include: Least-Privilege Tool Gating + Tool Filters, Guardian-Agents (Nöther et al., 2025), Delimiters (Hines et al., 2024), and a quality-only MaMa variant (AFlow-like). Section 5.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are from 2024-2025 (Debenedetti et al. 2024, Nöther et al. 2025, Hines et al. 2024), all recent and relevant to the multi-agent safety domain."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The quality-only variant (no Meta-Adversary) serves as an ablation of the adversarial component. Appendix A.4 analyzes the influence of the initial archive (empty vs. seeded). Figure 6 shows MaMa with empty initial archive."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics: quality score (1-5) and safety score (1-5), measured independently. Additionally, Attack Success Rate (ASR) is used for targeted and tool injection attacks (Tables 3-4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Judge models validated against manual annotation of 50 randomly sampled episodes. Cohen's Weighted Kappa reported (κ=0.88 safety, κ=0.80 quality). Section 5.1."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation between development/tuning tasks and evaluation tasks is described. The same environments are used for both the iterative design process and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per environment (Travel Planning, Personal Assistant, Financial Article Writing, Code Generation) in all tables and figures."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes specific attack successes: creating Ricin recipes, sending money to bank accounts, planning travel to disaster areas, injecting stock recommendations, deleting user files. Section 5.3 and Figure 3 show where safety degrades with >3 adversarial agents."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 Code Generation shows MaMa (0.0968) does not outperform Delimiters (0.0787) or Guardian Agents (0.0939). Figure 3 shows safety drops steeply when adversary controls >3 agents."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of improved safety while maintaining quality are supported by Figure 2 and Tables 1-4. Claims of generalization to stronger adversaries and different LLMs are supported by Table 2, Figure 3, and Tables 3-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('MaMa produces safer systems') supported by controlled comparisons: same environments, same models, same evaluation, with and without the adversarial component. The ablation (quality-only variant) isolates the Meta-Adversary's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Designing Safe Agentic Systems' generally, but results are limited to four specific environments from BAD-ACTS, specific LLMs (Llama-3.3, GPT-5.1, Qwen3:32b), and LLM-based judge evaluation. The paper does not bound claims to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, the improvements could be partly due to the iterative optimization process itself (more compute) rather than the adversarial framing. The quality-only ablation partially addresses this but is not discussed as such."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Safety and quality are measured via LLM judges on a 1-5 scale. While judges are validated against human annotations (κ=0.88/0.80), the paper does not discuss the gap between judge scores and actual safety/quality outcomes. A system rated 'safe' by a judge could still cause harm in deployment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'GPT-5.1', 'GPT-5-mini', 'Qwen3:32b', 'Gemma3:27b', 'Llama-3.3:70b'. No API versions, snapshot dates, or specific model IDs are provided. Marketing names without versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts for Meta-Agent, Meta-Adversary, all judge models, and environment descriptions are provided in Appendix B.4 (pages 14-28)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 lists all hyperparameters: generations=25, clean iterations=10, attacks per generation=25, k=5, ε=1, λ=0.3, γ=3, η=1. However, LLM sampling parameters (temperature, top-p) are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic system structure is formally defined in Section 3.1 as tuple (A, T, G). Communication graphs, tools, agent roles are described. AutoGen framework used for execution. Figure 1 and Figure 4 illustrate system architectures."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The environments come from BAD-ACTS benchmark. Attack validation procedure described (checking manipulated agent exists, self-reflection until valid). Episode budget limited to 50 messages/tool executions. Section 5.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Conclusion) includes substantive discussion of limitations: computational cost requiring regression models, focus on safety-only adversaries, and future directions for zero-sum settings and manipulation of additional system aspects."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in Section 6 are forward-looking research directions rather than specific threats to the validity of the current results. No discussion of: judge reliability beyond kappa, LLM-as-judge biases, overfitting to specific environments, or generalizability concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound findings to the four tested environments or discuss what kinds of agentic systems or attack settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw episode logs, agent interactions, or judge annotations are made available. Only aggregated scores are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection procedure is clear: iterative generation of systems and attacks over 25 generations, each system evaluated over 10 clean and 25 adversarial iterations, judged by LLM judges. Section 5.1 and Appendix B."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 50 episodes for judge validation were 'randomly sampled' but this is a benchmark evaluation, not a human study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Algorithms 1-2: propose system → evaluate clean → attack system → store results → select best. Top-5 attack selection criteria and archive sampling rule (equation in Section 4.2) are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources disclosed. Authors are at Max Planck Institute for Software Systems but no grants or funding are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as Max Planck Institute for Software Systems, Saarbrücken, Germany. No product being evaluated is affiliated with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a system design method, not a pre-trained model's knowledge on benchmarks. The models are used as components, not evaluated for learned capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not evaluating model knowledge on benchmarks — evaluating an automated system design algorithm."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The BAD-ACTS environments test system safety under adversarial conditions, not model knowledge. Contamination is not relevant."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 states total API cost of $72.15 for GPT-5.1 and GPT-5-mini. Table 8 reports wall-clock runtimes per environment (ranging from ~2 days to ~4.7 days for 25 iterations)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified (2x H200 GPUs, 2x EPYC 9555). API cost $72.15. Per-environment runtimes in Table 8. Sufficient to understand computational requirements."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results reported over 'three distinct runs' with standard deviations in all tables. Figure 2 shows shaded regions for variance across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'three distinct runs' (Tables 1-4), 25 attacks per generation, 10 clean iterations (Table 7)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The values in Table 7 appear to be fixed choices with no discussion of how they were selected or whether alternatives were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 states: 'we select the system that performed best according to the unweighted sum of quality and safety.' The selection criterion is explicit. Algorithm 2 selects the archive maximum of sum of safety and quality."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 4 environments, multiple baselines, and multiple transfer settings without any correction for family-wise error rate."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own MaMa system against baselines they implemented. No acknowledgment of potential bias from re-implementing baselines or tuning their own method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MaMa runs 25 generations of iterative optimization (2-5 days per environment) while baselines are static defenses. This massive compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether BAD-ACTS environments and LLM-based judge scoring actually measure real-world agentic system safety. The gap between simulated safety judgments and deployment safety is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "MaMa designs the entire scaffold as the intervention — the scaffold IS the thing being compared. Same underlying models used across all conditions. Table 2 explicitly varies both the adversary model and agent model to isolate contributions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not testing model knowledge on benchmarks. The environments are interactive task-completion settings, not static benchmark problems that could be memorized."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — not a benchmark evaluation testing learned knowledge."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no train/test split of benchmark problems."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — testing system design, not model memorization."
       }
     }
   },
   "claims": [
     {
       "claim": "The Meta-Adversary can significantly decrease the safety of agentic systems with common defenses.",
       "evidence": "Table 1: Safety drops from 4.89→1.66 (Travel), 4.31→1.53 (Personal Assistant), 4.29→2.46 (Financial), 4.65→3.20 (Code Gen) when attacked. Specific attacks described: Ricin recipe creation, unauthorized money transfers, malicious code execution.",
       "supported": "strong"
     },
     {
       "claim": "MaMa-designed systems are significantly safer than baselines while maintaining comparable quality.",
       "evidence": "Figure 2 shows safety improvements across all 4 environments over 25 generations, with quality maintained or improved. Table 3 shows ASR reductions vs LPTG+TF and Guardian Agents across all environments.",
       "supported": "strong"
     },
     {
       "claim": "MaMa-designed systems generalize to stronger adversaries and different LLMs.",
       "evidence": "Table 2 shows similar safety scores across Qwen3:32b and Gemma3:27b adversaries with Llama3.3 and GPT-5-mini agents. Figure 3 shows safety maintained for up to 3 adversarial agents (designed for ε=1).",
       "supported": "moderate"
     },
     {
       "claim": "MaMa-designed systems defend against indirect prompt injection attacks.",
       "evidence": "Table 4: MaMa achieves lowest ASR in 3/4 environments for tool injection attacks. In Code Generation, performance is comparable to baselines (0.0968 vs 0.0787 for Delimiters).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based judges achieve strong agreement with human annotations.",
       "evidence": "Section 5.1: Cohen's Weighted Kappa κ=0.88 (safety) and κ=0.80 (quality) on 50 randomly sampled episodes.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Compute budget asymmetry with baselines",
       "detail": "MaMa uses 2-5 days of iterative optimization per environment while baselines are static defenses. The improvement could partly be due to the additional compute and optimization iterations, not the adversarial game formulation specifically."
     },
     {
       "flag": "LLM-as-judge reliability",
       "detail": "Safety and quality are measured entirely by LLM judges (validated on only 50 episodes). LLM judges can have systematic biases, and 50 validation examples may be insufficient to characterize judge reliability across diverse attack types."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper repeatedly claims 'significant' improvements but uses no formal statistical tests. With only 3 runs and overlapping standard deviations in some settings, some claimed differences may not be statistically significant."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The title and abstract claim general applicability to 'safe agentic systems' but results are limited to 4 specific simulated environments with specific LLMs and judge-based evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated design of agentic systems",
       "authors": ["S. Hu", "C. Lu", "J. Clune"],
       "year": 2024,
       "relevance": "Foundational work on automated agentic system design that MaMa extends with safety considerations."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": ["J. Zhang", "J. Xiang", "Z. Yu"],
       "year": 2025,
       "relevance": "MCTS-based agentic system design method that MaMa directly builds upon."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["E. Debenedetti", "J. Zhang", "M. Balunovic"],
       "year": 2024,
       "relevance": "Benchmark for evaluating prompt injection attacks against LLM agents; MaMa evaluates against their tool injection attack."
     },
     {
       "title": "Benchmarking the robustness of agentic systems to adversarially-induced harms",
       "authors": ["J. Nöther", "A. Singla", "G. Radanovic"],
       "year": 2025,
       "relevance": "BAD-ACTS benchmark providing the evaluation environments used in MaMa; also introduces Guardian-Agents baseline."
     },
     {
       "title": "Magentic-One: A generalist multi-agent system for solving complex tasks",
       "authors": ["A. Fourney", "G. Bansal", "H. Mozannar"],
       "year": 2024,
       "relevance": "Multi-agent system demonstrating safety risks in handcrafted agentic architectures."
     },
     {
       "title": "AgentSquare: Automatic LLM agent search in modular design space",
       "authors": ["Y. Shang", "Y. Li", "K. Zhao"],
       "year": 2024,
       "relevance": "Structured approach to automated agentic system design that MaMa draws on."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2023,
       "relevance": "Multi-agent framework used to execute the designed agentic systems in MaMa experiments."
     },
     {
       "title": "Among Us: A sandbox for measuring and detecting agentic deception",
       "authors": ["S. Golechha", "A. Garriga-Alonso"],
       "year": 2025,
       "relevance": "Studies LLM agent deception in social deduction games, related to multi-agent safety."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "relevance": "Analyzes failure modes in multi-agent LLM systems including miscoordination and hallucinations."
     },
     {
       "title": "OS-Harm: A benchmark for measuring safety of computer use agents",
       "authors": ["T. Kuntz", "A. Duzan", "H. Zhao"],
       "year": 2025,
       "relevance": "Benchmark for evaluating safety of computer-using agents."
     },
     {
       "title": "AgentHarm: A benchmark for measuring harmfulness of LLM agents",
       "authors": ["M. Andriushchenko", "A. Souly", "M. Dziemian"],
       "year": 2024,
       "relevance": "Benchmark for measuring harmful behaviors of LLM agents."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["K. Hines", "G. Lopez", "M. Hall"],
       "year": 2024,
       "relevance": "Defense mechanism (Delimiters) against indirect prompt injection, used as baseline in MaMa."
     }
   ]
 }