ai-research-survey

scan-v5.json (25508B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "MaMa: A Game-Theoretic Approach for Designing Safe Agentic Systems",
     "authors": [
       "Jonathan Nöther",
       "Adish Singla",
       "Goran Radanovic"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.04431",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that MaMa consistently defends against worst-case attacks while maintaining quality are supported by Figure 2 and Tables 1–4 across all four environments. Generalization claims are supported by Section 5.3 transferability experiments.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares MaMa against concrete baselines (Guardian Agents, LPTG+TF, quality-only AFlow) in controlled environments with fixed threat models, and an ablation of the initial archive is provided; the design supports causal claims within the bounded experimental setting.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Results are explicitly bounded to the four BAD-ACTS environments and the specific threat model (ε compromised agents overwriting instructions); the conclusion explicitly identifies future extensions needed for zero-sum and stronger adversaries.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether safety improvements stem from generally more design iterations rather than adversarial feedback specifically; no ablation isolates the adversarial signal from sheer iterative search.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "LLM judges are used as proxies for safety and quality, but the paper validates both judges against 50 manually annotated episodes, reporting Cohen's kappa of 0.88 and 0.80 respectively, partially acknowledging the proxy gap.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the conclusion mentions computational cost and future directions but methodological limitations are not systematically addressed.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed; the paper acknowledges computational cost as a practical constraint but does not discuss evaluation reliability threats such as LLM-judge bias or environmental coverage.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The threat model scope is implicitly bounded but not explicitly framed as 'what results do NOT show'; the paper does not state that conclusions are limited to the four evaluated environments or to the specific agent-compromise threat model.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or grant acknowledgment appears anywhere in the paper, including abstract, conclusion, or impact statement.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are disclosed as affiliated with Max Planck Institute for Software Systems, Saarbrücken, Germany, with a correspondence email provided.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patents, equity, or consulting relationships are declared anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are formally defined: agentic system as a tuple (A, T, G), safety as a function s: T→R, quality as q: T→R, and the Stackelberg security game is formally stated in Section 3.3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are listed in the introduction under 'Framework,' 'Methodology,' and 'Evaluation' headings, clearly distinguishing this paper's additions from prior work.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages substantively with both automated design of agentic systems (AFlow, ADAS, AgentSquare) and adversarial attack literature (AgentDojo, InjecAgent), explicitly positioning MaMa as combining both streams.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper states final system code is in 'supplementary material' and prompts are in Appendix B.4, but no public code repository is linked and the MaMa algorithm implementation itself is not explicitly released.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The BAD-ACTS benchmark environments (Nöther et al., 2025) are used but no public URL or dataset release is provided; the benchmark is from the same lead author and its public availability is unconfirmed.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Compute hardware is specified (AMD EPYC 9555, H200 GPUs) and model names are given, but no requirements.txt, Dockerfile, or dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Pseudocode algorithms and hyperparameter tables are provided, but no end-to-end step-by-step reproduction guide exists; critical setup steps (AutoGen configuration, BAD-ACTS environment instantiation) are not described.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviations are reported parenthetically in all result tables (e.g., '1.66(0.11)') and as shaded regions in Figure 2, computed over three distinct evaluation runs.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, ANOVA, etc.) are applied to comparative claims despite multiple baselines and only three runs providing limited statistical power.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute score differences are visible (e.g., attacked safety rising from ~1.66 to ~3.81 in Travel Planning), and Attack Success Rate is reported numerically in Tables 3–4 with baseline comparisons.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 25 iterations, 25 attacks per generation, and 3 evaluation runs is mentioned only as a cost constraint, with no power analysis or justification for sample adequacy.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviation is reported across three distinct runs for all main results in tables and as shaded regions in figures.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three baselines are used: Least-Privilege Tool Gating + Tool Filters, Guardian Agents (Nöther et al., 2025), and a quality-only AFlow variant; Delimiters (Hines et al., 2024) is added for the injection attack evaluation.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines are drawn from recent work (2024–2025), including Debenedetti et al. (2024) for LPTG/TF and Nöther et al. (2025) for Guardian Agents.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.4 ablates the initial archive (empty vs. initialized), showing faster convergence and better results with initialization; however, no ablation directly isolates the Meta-Adversary contribution from iterative design alone.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both safety score (1–5 Likert) and quality score (1–5 Likert) are reported, along with Attack Success Rate (ASR) for targeted and injection attacks in Sections 5.3.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "Evaluation relies on LLM judges, not human annotators; 50 manually annotated episodes are used only to calibrate/validate the judge, not as the primary evaluation mechanism.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "No formal held-out test set is used; the Meta-Agent is optimized and evaluated on the same four environments. Transferability tests in Section 5.3 use different attack settings but not a separate held-out environment.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "All results are broken down per environment (Travel Planning, Personal Assistant, Financial Article Writing, Code Generation) in all tables and figures.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Figure 3 explicitly shows safety degradation when the adversary controls more than 3 agents; Table 4 shows MaMa performs no better than baselines in Code Generation under injection attacks.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 explicitly reports that MaMa does not outperform Delimiters in the Code Generation environment, and this result is acknowledged without dismissal.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model versions are named: GPT-5.1 (OpenAI, 2025) with a citation URL, Qwen3:32b (Yang et al., 2025), Llama-3.3:70b (Grattafiori et al., 2024), and GPT-5-mini for transfer experiments.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full system prompts for Meta-Adversary, Meta-Agent, and all quality/safety judges for all four environments are included in Appendix B.4 within the paper text.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 7 lists all hyperparameters: 25 generations, 10 clean iterations, 25 attacks per generation, k=5, ε=1, λ=0.3, γ=3, η=1.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "AutoGen (Wu et al., 2023) is specified as the multi-agent execution framework, and the message/tool execution episode structure is described in Section 5.1.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": false,
           "answer": false,
           "justification": "No dataset preprocessing is applicable; the environments generate episodes dynamically and no preprocessing pipeline exists.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw episode data, attack archives, or evaluation outputs are released publicly.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The process for generating episodes (running the agentic system, collecting tool executions and messages, scoring with judges) is described through Algorithms 1 and 2 and Section 5.1.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; the evaluation uses automated LLM-judge scoring of synthetic agentic episodes.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from system design → adversarial attack generation → episode execution → scoring → archiving is documented through pseudocode in Algorithms 1 and 2.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper does not evaluate model capabilities on held-out benchmarks; it evaluates iteratively designed system architectures, making training-cutoff contamination not the primary concern.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "NA — the evaluation is of system designs in simulated environments, not of model benchmark performance.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "NA — BAD-ACTS environments are task simulation environments, not NLP benchmark datasets susceptible to training-data contamination.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "API cost for GPT-5.1 and GPT-5-mini is reported as $72.15 total; per-environment runtimes are reported in Table 8 (2–5 days per environment over 25 iterations).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Compute infrastructure is specified (2x AMD EPYC 9555 CPUs, 24x 96GB RAM, 2x Nvidia H200 141GB GPUs), and per-environment runtimes are provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "The Meta-Adversary can discover effective attacks against systems with standard defenses (LPTG + Tool Filters), dramatically reducing safety scores.",
       "evidence": "Table 1 shows safety dropping from ~4.3–4.9 (clean) to 1.53–3.20 (attacked) across all four environments.",
       "supported": "strong"
     },
     {
       "claim": "MaMa-designed systems are significantly safer under adversarial attack than both initial designs and prior safety mechanisms (Guardian Agents).",
       "evidence": "Figure 2 shows consistent safety improvement over 25 iterations; Table 3 shows MaMa ASR (0.05–0.25) substantially below Guardian Agents (0.05–0.52) and LPTG+TF (0.09–0.57).",
       "supported": "strong"
     },
     {
       "claim": "Safety improvements from MaMa do not cause a significant quality reduction compared to systems optimized only for quality.",
       "evidence": "Figure 2 shows quality scores for MaMa equal to or higher than the quality-only AFlow baseline across all environments.",
       "supported": "moderate"
     },
     {
       "claim": "Systems designed by MaMa generalize to stronger adversaries controlling up to three agents.",
       "evidence": "Figure 3 shows safety scores remain relatively stable for 1–3 adversarial agents before a steep drop at 4+ agents.",
       "supported": "moderate"
     },
     {
       "claim": "MaMa systems generalize to different adversary and agent LLMs not used during design.",
       "evidence": "Table 2 shows similar safety scores across Qwen3:32b, Gemma3:27b adversaries and Llama3.3/GPT-5-mini agent models.",
       "supported": "moderate"
     },
     {
       "claim": "MaMa outperforms all baselines on indirect prompt injection attacks except in Code Generation.",
       "evidence": "Table 4 shows MaMa achieves lowest ASR in Travel Planner (0.0068) and Personal Assistant (0.2448) but not Code Generation (0.0968 vs. Delimiters 0.0787).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "theoretical",
     "benchmark-eval"
   ],
   "key_findings": "MaMa frames automated agentic system design as a Stackelberg security game, iteratively pitting a Meta-Agent (designer) against a Meta-Adversary (attacker) to produce systems robust under worst-case agent compromise. Empirical evaluation across four simulated environments (Travel Planning, Personal Assistant, Financial Article Writing, Code Generation) shows MaMa-designed systems achieve safety scores of ~3.7–4.6 under attack versus ~1.5–3.2 for systems with standard defenses, without sacrificing quality. The designed systems generalize to different adversary LLMs, stronger adversaries controlling up to three agents, and alternative attack types including indirect prompt injection. A notable failure mode is found in Code Generation under injection attacks, where MaMa does not outperform the Delimiters baseline.",
   "red_flags": [
     {
       "flag": "Self-authored benchmark",
       "detail": "The evaluation uses BAD-ACTS (Nöther et al., 2025), where the lead author of this paper is also the first author — no independent benchmark is used."
     },
     {
       "flag": "LLM-as-judge primary metric",
       "detail": "Safety and quality are measured by LLM judges validated on only 50 manually annotated episodes (κ=0.88, 0.80); LLM judges are known to have systematic biases that may inflate results."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparative claims are made across baselines with only 3 runs and standard deviation, but no t-tests or similar tests; some differences may not be statistically significant."
     },
     {
       "flag": "Missing key ablation",
       "detail": "No ablation isolates the contribution of adversarial feedback specifically from simply running more design iterations — the quality-only AFlow baseline differs in both adversarial signal and overall search dynamics."
     },
     {
       "flag": "Proprietary model dependency",
       "detail": "GPT-5.1 (OpenAI, 2025) is used for the Meta-Agent; this proprietary model may not be accessible to researchers attempting to reproduce results, and its training data is unknown."
     },
     {
       "flag": "No code repository",
       "detail": "The MaMa implementation is not released in a public repository; final designed system code is only in supplementary material, making full reproduction difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "AFlow: Automating Agentic Workflow Generation",
       "relevance": "Direct baseline; MaMa builds upon AFlow's Meta-Agent design loop, adding the adversarial second player."
     },
     {
       "title": "Benchmarking the robustness of agentic systems to adversarially-induced harms (BAD-ACTS)",
       "relevance": "Provides the evaluation environments and Guardian Agents baseline used throughout the paper."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "relevance": "Establishes the indirect prompt injection threat model used in Section 5.3 and inspires the LPTG/TF baselines."
     },
     {
       "title": "Automated Design of Agentic Systems (ADAS)",
       "relevance": "Prior work on Meta-Agent driven automated system search; MaMa extends this with a second adversarial player."
     },
     {
       "title": "Magentic-One: A generalist multi-agent system for solving complex tasks",
       "relevance": "Documents safety risks in manually designed agentic systems without adversarial safeguards, motivating this work."
     },
     {
       "title": "AgentSquare: Automatic LLM agent search in modular design space",
       "relevance": "Closely related automated design approach; MaMa adapts its structured agent representation."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "relevance": "Provides the Delimiters baseline used in the injection attack comparison in Table 4."
     },
     {
       "title": "Generative adversarial nets",
       "relevance": "Conceptual predecessor for two-player adversarial optimization framework that MaMa adapts for system design."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The framework directly addresses real multi-agent system safety, but requires 2–5 days of H200 GPU time per environment and proprietary model access, limiting near-term practitioner adoption."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that adversarial training improves safety without quality loss is somewhat counterintuitive; the game-theoretic framing of system design is novel in this space."
     },
     "fear_safety": {
       "score": 3,
       "justification": "The paper demonstrates concrete harmful outcomes (Ricin recipe generation, $600% stock fraud, device-wiping malware) achievable by compromising a single agent, directly addressing AI safety risks."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Adversarial agent compromise with real-world harm examples (financial fraud, dangerous travel planning) creates a compelling threat narrative, though framed constructively."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Requires days of compute on H200 GPUs and access to GPT-5.1; no demo environment or interactive version is available."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Max Planck Institute is a respected research institution but lacks the AI hype cachet of major industry labs; no famous product or celebrity researcher association."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }